refactor: make cross-API bridges call native executors instead of re-entering acompletion and aresponses
Some checks failed
Cost map pins recheck / rerun-cost-map-guard (push) Has been cancelled
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled

Each client surface (/v1/chat/completions, /v1/responses, /v1/messages) now
decides the provider's native API format once with litellm/native/route_plan.py,
owns logging for the request, and hands the translated request to a native
executor (execute_chat, execute_responses, execute_messages) instead of calling
back into a public entry point.

- Messages to Responses, Messages to chat, Responses to chat and chat to
  Responses bridges call the native executors
- Responses-mode models on providers with no Responses API stay on chat
- Bridges price the provider reply before translating it, keep its headers,
  map provider errors and run JSON-schema validation as re-entry did
- Bridged streams are logged once, under the client's call type, when the
  client stream ends
- execute_chat and the chat provider helpers move from main.py to
  litellm/native/, and a lint check keeps litellm.native from importing
  litellm.main
This commit is contained in:
narmstrong-berri 2026-10-29 21:30:08 +00:00
parent 60601ca995
commit 0ba4e85496
No known key found for this signature in database
98 changed files with 10674 additions and 6616 deletions

View file

@ -424,6 +424,7 @@ jobs:
tests/unit/images
tests/unit/interactions
tests/unit/messages
tests/unit/native
tests/unit/ocr
tests/unit/passthrough
tests/unit/rag
@ -715,6 +716,7 @@ jobs:
test-path: |-
tests/unit/responses/litellm_completion_transformation
tests/unit/responses/test_additional_tools.py
tests/unit/responses/test_bridge_executor.py
tests/unit/responses/test_custom_tool_call.py
tests/unit/responses/test_metadata_codex_callback.py
tests/unit/responses/test_no_duplicate_spend_logs.py

View file

@ -164,6 +164,7 @@ lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
# Linting targets
lint-ruff: $(LINT_DEP_INSTALL)
$(UV_RUN) python scripts/check_mcp_operation_boundary.py
$(UV_RUN) python scripts/check_native_imports.py
cd litellm && $(UV_RUN) ruff check . && cd ..
$(UV_RUN) ruff check --config ruff-tests.toml tests

View file

@ -11,8 +11,10 @@ from typing import (
cast, # noqa: TID251 # optional_params reaches the bridge as an untyped dict
)
from pydantic import TypeAdapter, ValidationError
from typing_extensions import TypedDict
import litellm
from litellm.types.llms.openai import ResponsesAPIResponse
if TYPE_CHECKING:
@ -36,6 +38,60 @@ def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str:
return f"{custom_llm_provider}/{model}"
_TOOLS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
_BRIDGED_REQUEST: Final = TypeAdapter(dict[str, object])
_TOOLS_ONLY_RESPONSES_SERVES: Final = frozenset({"mcp", "file_search"})
def _needs_responses_surface(request_data: Mapping[str, object]) -> bool:
"""`responses()` also checks the cache and dispatches MCP and emulated file_search tools, which the executor skips."""
if litellm.cache is not None or request_data.get("use_chat_completions_api"):
return True
model: Final = request_data.get("model")
if not isinstance(model, str) or model.startswith("openai/chat_completions/"):
return True
try:
tools: Final = _TOOLS.validate_python(request_data.get("tools") or ())
except ValidationError:
return True
return any(tool.get("type") in _TOOLS_ONLY_RESPONSES_SERVES for tool in tools)
def _executor_logging_obj(
request_data: Mapping[str, object], logging_obj: "LiteLLMLoggingObj"
) -> "LiteLLMLoggingObj | None":
from litellm.responses.bridge_executor import has_native_responses_config, native_logging_obj
model: Final = request_data.get("model")
if _needs_responses_surface(request_data) or not isinstance(model, str):
return None
if not has_native_responses_config(model, request_data):
return None
return native_logging_obj(logging_obj, request_data, request_data)
def _logged_by_client_stream(_completed: object) -> None:
return None
def _hand_completion_to_client_stream(result: object, executor_logging: "LiteLLMLoggingObj | None") -> None:
"""The chat stream wrapping the executor's Responses stream logs the request once, in the shape the client got."""
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
if executor_logging is not None and isinstance(result, BaseResponsesAPIStreamingIterator):
result.hand_completion_to(_logged_by_client_stream)
def _retagged_as_responses(
request_data: Mapping[str, object], logging_obj: "LiteLLMLoggingObj"
) -> Mapping[str, object]:
"""`responses()` logs the Responses object it returns, so the shared logging object takes its call type."""
from litellm.types.utils import CallTypes
setattr(logging_obj, "call_type", CallTypes.responses.value)
return request_data
class ResponsesToCompletionBridgeHandler:
def __init__(self):
from .transformation import LiteLLMResponsesTransformationHandler
@ -172,6 +228,7 @@ class ResponsesToCompletionBridgeHandler:
from litellm import responses
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.responses.bridge_executor import native_responses
validated_kwargs: Final = self.validate_input_kwargs(kwargs)
model: Final = validated_kwargs["model"]
@ -199,11 +256,16 @@ class ResponsesToCompletionBridgeHandler:
# `sanitized_litellm_params` already carries both, so passing them again
# would raise a duplicate-keyword TypeError.
request_data["custom_llm_provider"] = custom_llm_provider
request_data["model"] = _restore_routing_prefix(model, custom_llm_provider)
routed_model: Final = _restore_routing_prefix(model, custom_llm_provider)
request_data["model"] = routed_model
if kwargs.get("cache") is not None:
request_data["cache"] = kwargs["cache"]
result: Final = responses(
**request_data,
bridged_request: Final = _BRIDGED_REQUEST.validate_python(request_data)
executor_logging: Final = _executor_logging_obj(bridged_request, logging_obj)
result: Final = (
responses(**_retagged_as_responses(request_data, logging_obj))
if executor_logging is None
else native_responses(bridged_request, routed_model, bridged_request, executor_logging)
)
from litellm.types.utils import ModelResponse
@ -253,6 +315,7 @@ class ResponsesToCompletionBridgeHandler:
return self._apply_post_stream_processing(result, model, custom_llm_provider)
from .transformation import is_structured_output_request
_hand_completion_to_client_stream(result, executor_logging)
completion_stream: Final = self.transformation_handler.get_model_response_iterator(
streaming_response=result,
sync_stream=True,
@ -273,6 +336,7 @@ class ResponsesToCompletionBridgeHandler:
async def acompletion(self, *args, **kwargs) -> Union["ModelResponse", "CustomStreamWrapper"]:
from litellm import aresponses
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.responses.bridge_executor import anative_responses
validated_kwargs: Final = self.validate_input_kwargs(kwargs)
model: Final = validated_kwargs["model"]
@ -302,12 +366,16 @@ class ResponsesToCompletionBridgeHandler:
# `sanitized_litellm_params` already carries both, so passing them again
# would raise a duplicate-keyword TypeError.
request_data["custom_llm_provider"] = custom_llm_provider
request_data["model"] = _restore_routing_prefix(model, custom_llm_provider)
routed_model: Final = _restore_routing_prefix(model, custom_llm_provider)
request_data["model"] = routed_model
if kwargs.get("cache") is not None:
request_data["cache"] = kwargs["cache"]
result: Final = await aresponses(
**request_data,
aresponses=True,
bridged_request: Final = _BRIDGED_REQUEST.validate_python(request_data)
executor_logging: Final = _executor_logging_obj(bridged_request, logging_obj)
result: Final = (
await aresponses(**_retagged_as_responses(request_data, logging_obj), aresponses=True)
if executor_logging is None
else await anative_responses(bridged_request, routed_model, bridged_request, executor_logging)
)
from litellm.types.utils import ModelResponse
@ -357,6 +425,7 @@ class ResponsesToCompletionBridgeHandler:
return self._apply_post_stream_processing(result, model, custom_llm_provider)
from .transformation import is_structured_output_request
_hand_completion_to_client_stream(result, executor_logging)
completion_stream: Final = self.transformation_handler.get_model_response_iterator(
streaming_response=result,
sync_stream=False,

View file

@ -1195,10 +1195,6 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
api_model: Final = model
from litellm.types.utils import CallTypes
setattr(litellm_logging_obj, "call_type", CallTypes.responses.value)
sanitized_litellm_params: Final = self._build_sanitized_litellm_params(litellm_params)
request_data: Final = {

View file

@ -41,13 +41,13 @@ from litellm.llms.black_forest_labs.image_generation.handler import bfl_image_ge
from litellm.llms.openai.workload_identity import resolve_openai_workload_identity_config
from litellm.main import (
azure_chat_completions,
base_llm_aiohttp_handler,
base_llm_http_handler,
bedrock_image_edit,
bedrock_image_generation,
openai_chat_completions,
openai_image_variations,
)
from litellm.native.chat_providers.handlers import base_llm_aiohttp_handler
###########################################
from litellm.secret_managers.main import get_secret_str

View file

@ -79,6 +79,7 @@ from litellm.litellm_core_utils.core_helpers import (
get_provider_response_headers_from_hidden_params,
is_expected_client_error,
native_messages_reported_cost,
process_response_headers,
provider_reported_usage_cost,
proxy_stamped_kubernetes_pod_routing,
proxy_stamped_used_client_oauth_token,
@ -645,6 +646,10 @@ class Logging(LiteLLMLoggingBaseClass):
baseline_observation: "CapturedBaselineObservation | None" = None
_on_failure_payload_built: "Callable[[], None] | None" = None
def hold_stream_completion(self, completion: tuple[object, ...]) -> None:
"""Parks stream-end logging for the proxy to release once its end-of-stream guardrails finish."""
setattr(self, "_deferred_stream_complete_args", completion) # noqa: B010 # declaring the slot here turns the proxy's reads of it into private-usage findings
@property
def _defer_async_logging(self) -> bool:
return self.defer_async_logging
@ -4944,6 +4949,29 @@ class Logging(LiteLLMLoggingBaseClass):
row is built from this response rather than re-parsed from the served body."""
self.bridged_model_response = response
def price_bridged_provider_response(self, response: ModelResponse | ResponsesAPIResponse) -> None:
"""Stamps the cost and headers of the provider response a bridge translates into its client's shape.
The client shape has no field for some of the usage the price depends on (built-in tool calls,
the 1h cache-write split), and the translation carries the stamped cost over, so the request is
billed for what the provider returned. The headers get the round of processing the call that made
the response gave them, which also names the provider request id `llm_provider-request-id`."""
set_hidden_param(response, "response_cost", self.response_cost_calculator(result=response))
provider_headers: Final = (get_hidden_params(response) or {}).get("additional_headers")
if isinstance(provider_headers, dict):
set_hidden_param(
response,
"additional_headers",
process_response_headers(provider_headers, preserve_litellm_internal_headers=True),
)
def price_bridged_provider_stream(self, stream: object) -> None:
"""Prices a provider stream a bridge wraps for its client, as the call that opened it did.
Nothing has streamed yet, so this only records the zero cost breakdown the proxy builds the
response headers from; the request is billed when the stream ends."""
self.response_cost_calculator(result=stream)
def record_bridged_responses_api_response(self, response: ResponsesAPIResponse) -> None:
self.record_bridged_model_response(self._translate_responses_api_response_to_model_response(response))

View file

@ -305,6 +305,7 @@ def _provider_hidden_params(
class CustomStreamWrapper:
_earlier_rounds_spend: "BilledRoundsSpend | None" = None
_on_complete: Callable[[object], None] | None = None
@property
def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation
@ -1755,7 +1756,7 @@ class CustomStreamWrapper:
setattr(e, "message", str(e))
raise self._map_stream_exception(e)
def set_logging_event_loop(self, loop):
def set_logging_event_loop(self, loop: asyncio.AbstractEventLoop) -> None:
"""
import litellm, asyncio
@ -1907,11 +1908,19 @@ class CustomStreamWrapper:
if not cache_hit and self.logging_obj.llm_caching_handler is not None:
await self.logging_obj.llm_caching_handler.add_streaming_response_to_cache(processed_chunk)
def hand_completion_to(self, on_complete: Callable[[object], None]) -> None:
"""A bridge whose own client stream logs the request takes the assembled response instead of the logger."""
self._on_complete = on_complete
def take_completion_back(self) -> None:
"""The logger gets the assembled response again, as before a bridge took it."""
self._on_complete = None
def run_success_logging_and_cache_storage(self, processed_chunk, cache_hit: bool):
"""
Runs success logging in a thread and adds the response to the cache
"""
if litellm.disable_streaming_logging is True:
if litellm.disable_streaming_logging is True or self._on_complete is not None:
"""
[NOT RECOMMENDED]
Set this via `litellm.disable_streaming_logging = True`.
@ -2174,13 +2183,18 @@ class CustomStreamWrapper:
processed_chunk=_cache_copy,
cache_hit=cache_hit,
)
executor.submit(
self.logging_obj.success_handler,
_log_copy,
None,
None,
cache_hit,
)
if self._on_complete is not None:
self._on_complete(_log_copy)
else:
executor.submit(
self.logging_obj.success_handler,
_log_copy,
None,
None,
cache_hit,
)
elif self._on_complete is not None:
self._on_complete(response)
else:
executor.submit(
self.logging_obj.success_handler,
@ -2415,7 +2429,9 @@ class CustomStreamWrapper:
"_on_deferred_stream_complete",
None,
)
if _deferred_cb is not None:
if self._on_complete is not None:
self._on_complete(complete_streaming_response)
elif _deferred_cb is not None:
# Proxy has post-call guardrails. Store the assembled
# response so the outer streaming consumer
# (ProxyLogging.async_post_call_streaming_iterator_hook)
@ -2423,10 +2439,7 @@ class CustomStreamWrapper:
# end-of-stream blocks complete. Scheduling here via
# create_task would race with unified_guardrail's
# end-of-stream block for short-stream providers.
self.logging_obj._deferred_stream_complete_args = (
complete_streaming_response,
cache_hit,
)
self.logging_obj.hold_stream_completion((complete_streaming_response, cache_hit))
else:
# prefer_async_handlers routes CustomLogger to async_success_handler
# when consumers use ``async for`` on sync-SDK streams. Legacy string

View file

@ -24,6 +24,7 @@ from litellm.llms.anthropic.messages.context_management import (
apply_context_management,
)
from litellm.llms.anthropic.messages.convertible_blocks import reject_malformed_tool_result_text
from litellm.llms.anthropic.messages.held_stream_completion import hold_completion_of
from litellm.llms.anthropic.messages.safeguards import (
SafeguardsEvaluator,
build_safeguards_evaluator,
@ -39,6 +40,7 @@ from litellm.llms.anthropic.messages.utils import (
proxy_sub_call_logging_kwargs,
proxy_sub_call_trace_id,
)
from litellm.native.chat_plan import aexecute_planned_chat, execute_planned_chat, plan_chat
from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
@ -797,8 +799,13 @@ class LiteLLMMessagesToCompletionTransformationHandler:
extra_kwargs=kwargs,
)
completion_response: Final = await litellm.acompletion(**completion_kwargs)
logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs)
planned_chat: Final = None if logging_obj is None else plan_chat(completion_kwargs, logging_obj)
completion_response: Final = (
await litellm.acompletion(**completion_kwargs)
if planned_chat is None
else await aexecute_planned_chat(planned_chat)
)
if stream:
transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming(
@ -810,6 +817,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
completion_request=completion_kwargs,
litellm_logging_obj=logging_obj,
safeguards_evaluator=safeguards_evaluator,
held_completion=None if planned_chat is None else hold_completion_of(completion_response, logging_obj),
)
if transformed_stream is not None:
return transformed_stream
@ -944,8 +952,11 @@ class LiteLLMMessagesToCompletionTransformationHandler:
extra_kwargs=kwargs,
)
completion_response: Final = litellm.completion(**completion_kwargs)
logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs)
planned_chat: Final = None if logging_obj is None else plan_chat(completion_kwargs, logging_obj)
completion_response: Final = (
litellm.completion(**completion_kwargs) if planned_chat is None else execute_planned_chat(planned_chat)
)
if stream:
transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming(
@ -956,6 +967,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
is_async=False,
completion_request=completion_kwargs,
litellm_logging_obj=logging_obj,
held_completion=None if planned_chat is None else hold_completion_of(completion_response, logging_obj),
)
if transformed_stream is not None:
return transformed_stream

View file

@ -53,6 +53,7 @@ from litellm.types.utils import AdapterCompletionStreamWrapper, ChatCompletionDe
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
from litellm.llms.anthropic.messages.held_stream_completion import HeldStreamCompletion
from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator
from litellm.types.utils import ModelResponseStream
@ -637,6 +638,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
safeguards_evaluator: "SafeguardsEvaluator | None" = None,
completion_request: Mapping[str, object] | None = None,
held_completion: "HeldStreamCompletion | None" = None,
):
# Wrap the upstream stream so chunks that carry both content and a
# finish_reason (fake-streamed providers) are split into two — see
@ -651,6 +653,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
litellm_logging_obj.record_streamed_anthropic_message_id(self._message_id)
self.litellm_logging_obj = litellm_logging_obj
self.safeguards_evaluator: Final = safeguards_evaluator
self._held_completion: Final = held_completion
# Mapping of truncated tool names to original names (for OpenAI's 64-char limit)
self.tool_name_mapping = tool_name_mapping or {}
# Polyfill applied_edits on final message_delta.
@ -1350,14 +1353,18 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
This wrapper ensures dict chunks are SSE formatted with both event and data lines.
"""
for chunk in self:
if isinstance(chunk, dict):
event_type: str = str(chunk.get("type", "message"))
payload = f"event: {event_type}\ndata: {json.dumps(chunk)}\n\n"
yield payload.encode()
else:
# For non-dict chunks, forward the original value unchanged
yield chunk
try:
for chunk in self:
if isinstance(chunk, dict):
event_type: str = str(chunk.get("type", "message"))
payload = f"event: {event_type}\ndata: {json.dumps(chunk)}\n\n"
yield payload.encode()
else:
# For non-dict chunks, forward the original value unchanged
yield chunk
finally:
if self._held_completion is not None:
self._held_completion.log(is_async=False)
async def aclose(self) -> None:
await self._combined_chunk_splitter.aclose()
@ -1379,6 +1386,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
else:
yield anthropic_sse_frame(await safeguard_results.observe(_SSE_EVENT.validate_python(chunk)))
except Exception as e: # noqa: BLE001 # boundary before the socket: any upstream failure becomes an Anthropic error event
if self._held_completion is not None:
self._held_completion.discard()
verbose_logger.exception("Anthropic Adapter - mid-stream error: %s", e)
logging_obj: Final = self.litellm_logging_obj
provider_error: Final = _provider_error(e)
@ -1402,6 +1411,12 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
failure_handler_error,
)
yield _mid_stream_error_sse_event(e)
else:
if self._held_completion is not None:
self._held_completion.log(is_async=True)
finally:
if self._held_completion is not None:
self._held_completion.log(is_async=True, client_detached=True)
def _increment_content_block_index(self):
self.current_content_block_index += 1

View file

@ -213,6 +213,7 @@ from .streaming_iterator import AnthropicSSEStream, AnthropicStreamWrapper
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
from litellm.llms.anthropic.messages.held_stream_completion import HeldStreamCompletion
from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator
from litellm.types.llms.anthropic import ContentBlockContentBlockDict
@ -326,6 +327,7 @@ class AnthropicAdapter:
litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
safeguards_evaluator: "SafeguardsEvaluator | None" = None,
completion_request: Mapping[str, object] | None = None,
held_completion: "HeldStreamCompletion | None" = None,
) -> AsyncIterator[bytes] | Iterator[bytes] | None:
"""
Translate OpenAI streaming response to Anthropic format.
@ -355,6 +357,7 @@ class AnthropicAdapter:
litellm_logging_obj=litellm_logging_obj,
safeguards_evaluator=safeguards_evaluator,
completion_request=completion_request,
held_completion=held_completion,
)
# Return the SSE-wrapped version for proper event formatting.
if is_async:
@ -1620,7 +1623,8 @@ class LiteLLMAnthropicMessagesAdapter:
else translated_finish_reason
)
# extract usage
usage: Final[Usage] = getattr(response, "usage")
# A provider reply with no usage block leaves the response without one; it reads as zero tokens.
usage: Final[Usage] = getattr(response, "usage", None) or Usage()
message_usage: Final = self.translate_openai_usage_to_anthropic_usage(usage)
polyfill_iterations: Final = polyfill_result.iterations_usage if polyfill_result is not None else None
anthropic_usage: Final[AnthropicUsage] = (

View file

@ -0,0 +1,64 @@
from collections.abc import Callable
from typing import TYPE_CHECKING, Final, Protocol, runtime_checkable
from litellm.responses.streaming_iterator import dispatch_stream_success
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@runtime_checkable
class _HandsOffCompletion(Protocol):
def hand_completion_to(self, on_complete: Callable[[object], None]) -> None: ...
class _NothingHeld:
pass
_NOTHING_HELD: Final = _NothingHeld()
class HeldStreamCompletion:
"""The executor stream's assembled response, logged by the client-facing Messages stream when it ends.
A stream that ended with no content hands over ``None``; it is still logged, as the success
handler logs an empty stream when it is not held.
"""
def __init__(self, logging_obj: "LiteLLMLoggingObj") -> None:
self._logging_obj: Final = logging_obj
self._response: object = _NOTHING_HELD
def take(self, response: object) -> None:
self._response = response
def _release(self) -> object:
response: Final = self._response
self._response = _NOTHING_HELD
return response
def discard(self) -> None:
self._release()
def log(self, *, is_async: bool, client_detached: bool = False) -> None:
response: Final = self._release()
if response is _NOTHING_HELD:
return
dispatch_stream_success(
self._logging_obj,
response,
start_time=None,
end_time=None,
cache_hit=None,
is_async=is_async,
client_detached=client_detached,
)
def hold_completion_of(stream: object, logging_obj: "LiteLLMLoggingObj | None") -> HeldStreamCompletion | None:
if logging_obj is None or not isinstance(stream, _HandsOffCompletion):
return None
held: Final = HeldStreamCompletion(logging_obj)
stream.hand_completion_to(held.take)
return held

View file

@ -16,7 +16,6 @@ from pydantic import TypeAdapter
import litellm
from litellm.anthropic_beta_headers_manager import aload_beta_headers_config
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.common_utils import (
@ -29,23 +28,17 @@ from litellm.llms.base_llm.anthropic_messages.transformation import (
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.native.messages import execute_messages
from litellm.native.route_plan import NativeChat, NativeResponses, native_messages_config, plan_messages_route
from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway
from litellm.router_utils.reasoning_effort_capability import (
RequestedReasoningEffort,
deployment_reasoning_effort_levels,
keeps_effort,
model_group_from_metadata,
resolve_deployment_reasoning_effort,
with_adaptive_thinking_as_effort,
)
from litellm.types.llms.anthropic import AllAnthropicToolsValues
from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import CallTypes, LlmProviders
from litellm.utils import ProviderConfigManager, client
from litellm.types.utils import CallTypes
from litellm.utils import client
from ..adapters.handler import LiteLLMMessagesToCompletionTransformationHandler, request_safeguards_evaluator
from ..responses_adapters.handler import LiteLLMMessagesToResponsesAPIHandler
@ -56,69 +49,14 @@ from ..safeguards import (
with_safeguard_results,
)
from .interceptors import get_messages_interceptors
from .utils import AnthropicMessagesRequestUtils, mock_response, prepare_native_messages
from .utils import mock_response, prepare_native_messages
__all__ = ("anthropic_messages", "anthropic_messages_handler")
# Providers that are routed directly to the OpenAI Responses API instead of
# going through chat/completions.
_RESPONSES_API_PROVIDERS: Final = frozenset({"openai"})
_SAFEGUARD_KWARGS: Final = TypeAdapter(Mapping[str, object])
_SAFEGUARD_MESSAGES: Final = TypeAdapter(Sequence[Mapping[str, object]])
def _bridges_to_responses_api(model: str, custom_llm_provider: str, deployment_model_info: object) -> bool:
from litellm.responses.bridge_check import responses_api_bridge_check, str_keyed_mapping
model_info, _ = responses_api_bridge_check(
model=model,
custom_llm_provider=custom_llm_provider,
deployment_model_info=str_keyed_mapping(deployment_model_info),
)
return model_info.get("mode") == "responses"
def _responses_mode_is_lost_by_prefix_strip(
requested_model: str, resolved_model: str, custom_llm_provider: str, deployment_model_info: object
) -> bool:
"""Whether a Responses-only deployment stops looking like one once its provider prefix is stripped.
``litellm.completion`` re-derives the Responses bridge from the stripped id alone, so a
deployment id such as ``perplexity/perplexity/sonar`` (mode ``responses``) is shadowed by the
chat entry ``perplexity/sonar`` and would otherwise be sent to chat/completions.
"""
if requested_model == resolved_model:
return False
return _bridges_to_responses_api(
requested_model, custom_llm_provider, deployment_model_info
) and not _bridges_to_responses_api(resolved_model, custom_llm_provider, deployment_model_info)
def _should_route_to_responses_api(
custom_llm_provider: str | None,
requested_model: str | None = None,
resolved_model: str | None = None,
deployment_model_info: object = None,
stop_sequences: Sequence[str] | None = None,
) -> bool:
"""Return True when the request should use the Responses API path.
Set ``litellm.use_chat_completions_url_for_anthropic_messages = True`` to
opt out and route OpenAI/Azure requests through chat/completions instead.
"""
if litellm.use_chat_completions_url_for_anthropic_messages:
return False
if stop_sequences:
return False
if custom_llm_provider in _RESPONSES_API_PROVIDERS:
return True
if custom_llm_provider is None or requested_model is None or resolved_model is None:
return False
return _responses_mode_is_lost_by_prefix_strip(
requested_model, resolved_model, custom_llm_provider, deployment_model_info
)
async def _with_native_safeguard_results(
pending: Coroutine[object, object, AnthropicMessagesResponse | AsyncIterator[bytes]],
proxy_classifies: Callable[[], Awaitable[bool]],
@ -146,88 +84,6 @@ def _native_safeguards_evaluator(kwargs: Mapping[str, object], messages: object)
return request_safeguards_evaluator(kwargs, _SAFEGUARD_MESSAGES.validate_python(messages))
def _deployment_passes_through_anthropic_messages(model_info: object) -> bool:
"""Whether the deployment opted into forwarding /v1/messages untranslated.
The opt-in is ``model_info.supported_endpoints`` containing ``"/v1/messages"``,
declared per deployment in config.yaml and plumbed here as ``kwargs["model_info"]``
by the router.
"""
if not isinstance(model_info, dict):
return False
supported_endpoints: Final = model_info.get("supported_endpoints")
return isinstance(supported_endpoints, (list, tuple)) and "/v1/messages" in supported_endpoints
def _with_output_config(params: Mapping[str, object], output_config: RequestedReasoningEffort) -> Mapping[str, object]:
rest: Final = {key: value for key, value in params.items() if key != "output_config"}
return rest if output_config is None else {**rest, "output_config": output_config}
def _with_deployment_output_effort(
params: Mapping[str, object],
*,
model: str,
custom_llm_provider: str,
kwargs: Mapping[str, object],
drop_params: bool,
) -> Mapping[str, object]:
deployment_model_info: Final = kwargs.get("model_info")
def resolve(requested: RequestedReasoningEffort) -> RequestedReasoningEffort:
return resolve_deployment_reasoning_effort(
requested,
model=model,
custom_llm_provider=custom_llm_provider,
deployment_model_info=deployment_model_info,
model_group=model_group_from_metadata(kwargs.get("litellm_metadata"), kwargs.get("metadata")),
drop_params=drop_params,
).value
def translate(source: Mapping[str, object]) -> Mapping[str, object]:
return with_adaptive_thinking_as_effort(
source, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params
)
requested_output: Final = as_str_mapping(params.get("output_config"))
caller_output: Final = None if requested_output is None else resolve(requested_output)
caller_params: Final = params if requested_output is None else _with_output_config(params, caller_output)
if deployment_reasoning_effort_levels(deployment_model_info) is None:
return caller_params
if requested_output is not None and keeps_effort(requested_output, caller_output):
return _with_output_config(translate(params), caller_output)
translated: Final = translate(caller_params)
budget_output: Final = as_str_mapping(translated.get("output_config"))
return translated if budget_output is None else _with_output_config(translated, resolve(budget_output))
def _deployment_supports_cache_control_ttl(model_info: object) -> bool:
return isinstance(model_info, dict) and model_info.get("cache_control_ttl") is True
def _native_messages_config(
model: str, custom_llm_provider: str | None, model_info: object, api_base: str | None
) -> BaseAnthropicMessagesConfig | None:
provider_config: Final = (
ProviderConfigManager.get_provider_anthropic_messages_config(
model=model,
provider=litellm.LlmProviders(custom_llm_provider),
)
if custom_llm_provider is not None and custom_llm_provider in [provider.value for provider in LlmProviders]
else None
)
native_config: Final = (
provider_config if provider_config is not None and provider_config.serves_api_base(model, api_base) else None
)
if native_config is not None or not _deployment_passes_through_anthropic_messages(model_info):
return native_config
from litellm.llms.openai_like.messages.transformation import (
OpenAILikeAnthropicMessagesConfig,
)
return OpenAILikeAnthropicMessagesConfig(cache_control_ttl=_deployment_supports_cache_control_ttl(model_info))
async def _proxy_classifies(model: str, safeguards: object, native_config: BaseAnthropicMessagesConfig | None) -> bool:
if not proxy_serves_safeguards(safeguards):
return False
@ -244,7 +100,7 @@ async def proxy_classifies_tool_uses(model: str, custom_llm_provider: str | None
return await _proxy_classifies(
model,
kwargs.get("safeguards"),
_native_messages_config(
native_messages_config(
model, custom_llm_provider, kwargs.get("model_info"), api_base if isinstance(api_base, str) else None
),
)
@ -687,40 +543,44 @@ def anthropic_messages_handler(
**kwargs,
)
anthropic_messages_provider_config: Final = _native_messages_config(
model, custom_llm_provider, kwargs.get("model_info"), api_base
route: Final = plan_messages_route(
requested_model=original_model,
resolved_model=model,
custom_llm_provider=custom_llm_provider,
model_info=kwargs.get("model_info"),
api_base=api_base,
compaction_requested=kwargs.get("compaction") is not None,
force_chat_completions=bool(litellm.use_chat_completions_url_for_anthropic_messages),
stop_requested=bool(stop_sequences),
)
if anthropic_messages_provider_config is None:
# Route to Responses API for OpenAI / Azure, chat/completions for everything else.
if kwargs.get("compaction") is None and _should_route_to_responses_api(
custom_llm_provider, original_model, model, kwargs.get("model_info"), stop_sequences=stop_sequences
):
return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
max_tokens=max_tokens,
messages=messages,
model=original_model,
metadata=metadata,
stop_sequences=stop_sequences,
stream=stream,
system=cast( # cast-ok: anthropic accepts list system blocks; handler signature is narrower
"str | None", system
),
temperature=temperature,
thinking=thinking,
tool_choice=tool_choice,
tools=cast( # cast-ok: dict tools already fit the handler's union
"list[AllAnthropicToolsValues | dict[str, object]] | None", tools
),
top_k=top_k,
top_p=top_p,
_is_async=is_async,
api_key=api_key,
api_base=api_base,
client=client,
custom_llm_provider=custom_llm_provider,
**kwargs,
)
if isinstance(route, NativeResponses):
return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
max_tokens=max_tokens,
messages=messages,
model=original_model,
metadata=metadata,
stop_sequences=stop_sequences,
stream=stream,
system=cast( # cast-ok: anthropic accepts list system blocks; handler signature is narrower
"str | None", system
),
temperature=temperature,
thinking=thinking,
tool_choice=tool_choice,
tools=cast( # cast-ok: dict tools already fit the handler's union
"list[AllAnthropicToolsValues | dict[str, object]] | None", tools
),
top_k=top_k,
top_p=top_p,
_is_async=is_async,
api_key=api_key,
api_base=api_base,
client=client,
custom_llm_provider=custom_llm_provider,
**kwargs,
)
if isinstance(route, NativeChat):
# The in-gateway context_management polyfill runs inside
# ``async_anthropic_messages_handler`` so it can ``await`` the
# summarization model for ``compact_20260112``. ``context_management``
@ -761,23 +621,11 @@ def anthropic_messages_handler(
if marked_tools is not None:
AnthropicCacheControlHook.record_gateway_injection(request_kwargs, 1)
local_vars.update({**request_kwargs, **({"tools": list(marked_tools)} if marked_tools is not None else {})})
anthropic_messages_optional_request_params: Final = (
AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param(
params=local_vars,
model=model,
drop_params=litellm_params.get("drop_params") is True,
custom_llm_provider=custom_llm_provider,
)
)
resolved_api_base: Final = (
dynamic_api_base
if dynamic_api_base is not None and anthropic_messages_provider_config.uses_get_llm_provider_api_base()
else api_base
)
safeguard_kwargs: Final = (
_SAFEGUARD_KWARGS.validate_python(kwargs) if is_async is True and "safeguards" in kwargs else None
)
native_response: Final = base_llm_http_handler.anthropic_messages_handler(
native_response: Final = execute_messages(
http_handler=base_llm_http_handler,
model=model,
messages=cast( # cast-ok: anthropic messages are dicts at runtime
"list[dict[str, object]]",
@ -785,30 +633,23 @@ def anthropic_messages_handler(
cast(Sequence[object], messages) # cast-ok: anthropic payloads arrive as untyped dicts
),
),
anthropic_messages_provider_config=anthropic_messages_provider_config,
anthropic_messages_optional_request_params=dict(
_with_deployment_output_effort(
anthropic_messages_optional_request_params,
model=model,
custom_llm_provider=custom_llm_provider,
kwargs=cast(Mapping[str, object], kwargs), # cast-ok: router kwargs arrive untyped
drop_params=litellm.drop_params is True or litellm_params.get("drop_params") is True,
)
),
_is_async=is_async,
client=client,
provider_config=route.config,
request_params=local_vars,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
logging_obj=litellm_logging_obj,
api_key=api_key,
api_base=resolved_api_base,
api_base=api_base,
dynamic_api_base=dynamic_api_base,
client=client,
is_async=is_async,
stream=stream,
kwargs=kwargs,
request_kwargs=kwargs,
)
if safeguard_kwargs is None or not isinstance(native_response, Coroutine):
return native_response
return _with_native_safeguard_results(
native_response,
partial(_proxy_classifies, model, safeguard_kwargs.get("safeguards"), anthropic_messages_provider_config),
partial(_proxy_classifies, model, safeguard_kwargs.get("safeguards"), route.config),
partial(_native_safeguards_evaluator, safeguard_kwargs, messages),
)

View file

@ -19,11 +19,14 @@ from litellm.litellm_core_utils.asyncify import (
run_async_function, # pyright: ignore[reportUnknownVariableType] # untyped helper
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.prompt_templates.common_utils import (
cache_control_as_prompt_cache_breakpoint,
prompt_cache_breakpoint_for_wire,
with_prompt_cache_breakpoint,
)
from litellm.llms.anthropic.messages.held_stream_completion import hold_completion_of
from litellm.responses.bridge_executor import anative_responses, native_logging_obj, native_responses
from litellm.responses.utils import auto_reasoning_summary
from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS
from litellm.types.llms.anthropic import (
@ -38,7 +41,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import CountContextManagement
from litellm.types.utils import CallTypes, CountContextManagement
from litellm.utils import ProviderConfigManager
from ..adapters.handler import context_management_explicitly_dropped, request_safeguards_evaluator
@ -159,6 +162,9 @@ def _with_applied_edits(
return {**response, "context_management": ContextManagementResponse(applied_edits=list(applied_edits))}
_NEVER_FORWARDED: Final = CONVERTED_STREAM_KEYS | {"anthropic_messages"}
def _record_bridged_response(responses_kwargs: Mapping[str, object], result: ResponsesAPIResponse) -> None:
logging_obj: Final = litellm_logging_obj_from_kwargs(responses_kwargs)
if logging_obj is not None:
@ -166,8 +172,10 @@ def _record_bridged_response(responses_kwargs: Mapping[str, object], result: Res
def _forwarded_kwargs(extra_kwargs: Mapping[str, object] | None) -> Mapping[str, object]:
"""The litellm-specific kwargs forwarded verbatim onto the Responses API request."""
return extra_kwargs or {}
"""The litellm-specific kwargs (credentials, routing, metadata) that carry over to the Responses API call."""
return {
key: value for key, value in (extra_kwargs or {}).items() if key not in _NEVER_FORWARDED and value is not None
}
def _provider_returns_encrypted_reasoning(model: str, custom_llm_provider: object) -> bool:
@ -247,7 +255,7 @@ def _rewritten_blocks(content: object, rewrite: BlockRewrite) -> Sequence[Mappin
return ()
def _build_responses_kwargs(
def _build_responses_request(
*,
max_tokens: int,
messages: AnthropicRequestMessages,
@ -267,9 +275,7 @@ def _build_responses_kwargs(
output_format: AnthropicOutputSchema | None = None,
extra_kwargs: Mapping[str, object] | None = None,
) -> dict[str, object]:
"""
Build the kwargs dict to pass directly to litellm.responses() / litellm.aresponses().
"""
"""The Responses API request the Messages request translates to, without the forwarded litellm kwargs."""
forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs)
reject_malformed_tool_result_text(messages, model, forwarded_kwargs)
block_rewrite: Final = _prompt_cache_block_rewrite(model, forwarded_kwargs)
@ -314,7 +320,7 @@ def _build_responses_kwargs(
request_data["output_format"] = output_format
anthropic_request: Final = AnthropicMessagesRequest(**request_data)
responses_kwargs: Final = _ADAPTER.translate_request(
responses_request: Final = _ADAPTER.translate_request(
anthropic_request,
include_encrypted_reasoning=_provider_returns_encrypted_reasoning(
model, forwarded_kwargs.get("custom_llm_provider")
@ -325,7 +331,7 @@ def _build_responses_kwargs(
# Normalize reasoning effort based on model capabilities
# (e.g. "max" → "xhigh"/"high", "minimal" → "low" if unsupported)
reasoning: Final = responses_kwargs.get("reasoning")
reasoning: Final = responses_request.get("reasoning")
if isinstance(reasoning, dict):
effort: Final[object] = reasoning.get("effort")
if isinstance(effort, str):
@ -341,49 +347,39 @@ def _build_responses_kwargs(
deployment_model_info=forwarded_kwargs.get("model_info"),
)
if normalized != effort:
responses_kwargs["reasoning"] = {**reasoning, "effort": normalized}
responses_request["reasoning"] = {**reasoning, "effort": normalized}
if stream:
responses_kwargs["stream"] = True
# Forward litellm-specific kwargs (api_key, api_base, logging obj, etc.)
excluded: Final = CONVERTED_STREAM_KEYS | {"anthropic_messages"}
for key, value in forwarded_kwargs.items():
if key == "litellm_logging_obj" and value is not None:
from litellm.litellm_core_utils.litellm_logging import (
Logging as LiteLLMLoggingObject,
)
from litellm.types.utils import CallTypes
if isinstance(value, LiteLLMLoggingObject):
# Keep call_type as anthropic_messages so spend_logs are billed
# against /v1/messages; the success handler translates the
# Responses API result back to a ModelResponse for the row.
setattr(value, "call_type", CallTypes.anthropic_messages.value)
responses_kwargs[key] = value
elif key not in excluded and key not in responses_kwargs and value is not None:
responses_kwargs[key] = value
responses_request["stream"] = True
explicit_prompt_cache_key: Final = forwarded_kwargs.get("prompt_cache_key")
if explicit_prompt_cache_key is not None:
responses_kwargs["prompt_cache_key"] = explicit_prompt_cache_key
responses_request["prompt_cache_key"] = explicit_prompt_cache_key
deployment_include: Final = forwarded_kwargs.get("include")
bridge_include: Final = responses_kwargs.get("include")
bridge_include: Final = responses_request.get("include")
if isinstance(deployment_include, list) and isinstance(bridge_include, list):
responses_kwargs["include"] = [
responses_request["include"] = [
*bridge_include,
*(item for item in deployment_include if item not in bridge_include),
]
return responses_kwargs
return responses_request
def _reentrant_kwargs(
forwarded_kwargs: Mapping[str, object], request: Mapping[str, object], logging_obj: LiteLLMLoggingObj | None
) -> Mapping[str, object]:
if logging_obj is not None:
setattr(logging_obj, "call_type", CallTypes.anthropic_messages.value)
return {**forwarded_kwargs, **request}
class LiteLLMMessagesToResponsesAPIHandler:
"""
Handles Anthropic /v1/messages requests for OpenAI / Azure models by
calling litellm.responses() / litellm.aresponses() directly and translating
the response back to Anthropic format.
calling the native Responses executor and translating the response back
to Anthropic format.
"""
@staticmethod
@ -410,7 +406,7 @@ class LiteLLMMessagesToResponsesAPIHandler:
polyfill_result: Final = await _polyfilled(
model=model, messages=messages, tools=tools, system=system, edits=polyfilled_edits, kwargs=kwargs
)
responses_kwargs: Final = _build_responses_kwargs(
request: Final = _build_responses_request(
max_tokens=max_tokens,
messages=polyfill_result.messages if polyfill_result is not None else messages,
model=model,
@ -429,17 +425,25 @@ class LiteLLMMessagesToResponsesAPIHandler:
output_format=output_format,
extra_kwargs=kwargs,
)
forwarded_kwargs: Final = _forwarded_kwargs(kwargs)
logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs)
safeguards_evaluator: Final = request_safeguards_evaluator(kwargs, messages)
result: Final = await litellm.aresponses(**responses_kwargs)
native_logging: Final = native_logging_obj(logging_obj, forwarded_kwargs, request)
result: Final = (
await litellm.aresponses(**_reentrant_kwargs(forwarded_kwargs, request, logging_obj))
if native_logging is None
else await anative_responses(request, model, forwarded_kwargs, native_logging)
)
if stream:
wrapper: Final = AnthropicResponsesStreamWrapper(
responses_stream=result,
model=local_model_name(model, kwargs.get("custom_llm_provider")),
litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
litellm_logging_obj=logging_obj,
safeguards_evaluator=safeguards_evaluator,
applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None,
held_completion=None if native_logging is None else hold_completion_of(result, native_logging),
)
return wrapper.async_anthropic_sse_wrapper()
@ -449,7 +453,7 @@ class LiteLLMMessagesToResponsesAPIHandler:
served: Final = await with_safeguard_results(
_with_applied_edits(_ADAPTER.translate_response(result), polyfill_result), safeguards_evaluator
)
_record_bridged_response(responses_kwargs, result)
_record_bridged_response(kwargs, result)
return served
@staticmethod
@ -514,7 +518,7 @@ class LiteLLMMessagesToResponsesAPIHandler:
else None
)
polyfill_result: Final = ran if isinstance(ran, PolyfillResult) else None
responses_kwargs: Final = _build_responses_kwargs(
request: Final = _build_responses_request(
max_tokens=max_tokens,
messages=polyfill_result.messages if polyfill_result is not None else messages,
model=model,
@ -533,15 +537,22 @@ class LiteLLMMessagesToResponsesAPIHandler:
output_format=output_format,
extra_kwargs=kwargs,
)
result: Final = litellm.responses(**responses_kwargs)
forwarded_kwargs: Final = _forwarded_kwargs(kwargs)
logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs)
native_logging: Final = native_logging_obj(logging_obj, forwarded_kwargs, request)
result: Final = (
litellm.responses(**_reentrant_kwargs(forwarded_kwargs, request, logging_obj))
if native_logging is None
else native_responses(request, model, forwarded_kwargs, native_logging)
)
if stream:
wrapper: Final = AnthropicResponsesStreamWrapper(
responses_stream=result,
model=local_model_name(model, kwargs.get("custom_llm_provider")),
litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
litellm_logging_obj=logging_obj,
applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None,
held_completion=None if native_logging is None else hold_completion_of(result, native_logging),
)
return wrapper.async_anthropic_sse_wrapper()

View file

@ -38,6 +38,7 @@ from .transformation import (
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
from litellm.llms.anthropic.messages.held_stream_completion import HeldStreamCompletion
from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator
from litellm.types.llms.openai import ResponseIncompleteEvent
@ -131,12 +132,14 @@ class AnthropicResponsesStreamWrapper:
litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
safeguards_evaluator: "SafeguardsEvaluator | None" = None,
applied_edits: Sequence[AppliedEdit] | None = None,
held_completion: "HeldStreamCompletion | None" = None,
) -> None:
self.responses_stream = responses_stream
self._applied_edits: Final = tuple(applied_edits or ())
self.model = model
self.litellm_logging_obj = litellm_logging_obj
self.safeguards_evaluator: Final = safeguards_evaluator
self._held_completion: Final = held_completion
self._message_id: str = f"msg_{uuid.uuid4()}"
if litellm_logging_obj is not None:
litellm_logging_obj.record_streamed_anthropic_message_id(self._message_id)
@ -545,10 +548,22 @@ class AnthropicResponsesStreamWrapper:
async def _sse_frames(self) -> AsyncGenerator[bytes, None]:
safeguard_results: Final = StreamedSafeguardResults(self.safeguards_evaluator)
held: Final = self._held_completion
try:
async for chunk in self:
yield anthropic_sse_frame(await safeguard_results.observe(chunk))
except Exception:
if held is not None:
held.discard()
raise
else:
if held is not None and self._stream_failed:
held.discard()
if held is not None:
held.log(is_async=True)
finally:
if held is not None:
held.log(is_async=True, client_detached=True)
from litellm.llms.anthropic.messages.native.streaming_iterator import closing_superseded_attempt
if not closing_superseded_attempt():

View file

@ -1,7 +1,7 @@
import asyncio
import json
import time
from collections.abc import Callable, Coroutine
from collections.abc import Callable, Coroutine, Mapping
from typing import Final
import httpx
@ -114,7 +114,7 @@ class AzureOpenAIAssistantsAPIConfig:
def check_dynamic_azure_params(
azure_client_params: dict,
azure_client_params: Mapping[str, object],
azure_client: AzureOpenAI | AsyncAzureOpenAI | None,
) -> bool:
"""

File diff suppressed because it is too large Load diff

View file

@ -3,9 +3,11 @@ from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Itera
from types import MappingProxyType
from typing import Final, TypeAlias, cast # noqa: TID251 # native binding selects a sync result or an async awaitable
import litellm
from litellm.exceptions import BadRequestError
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.llms.anthropic.messages.native import handler as main
from litellm.native.route_plan import NativeMessages, NativeResponses, plan_messages_route
from litellm.rust_bridge import detectors
from litellm.rust_bridge.catalog import Feature, Route, RouteContext, Target
from litellm.rust_bridge.dispatch import PublicDispatch
@ -86,16 +88,20 @@ def _resolved(request: NativeCall) -> tuple[str, str | None]:
def _target(request: NativeCall, model: str, provider: str | None) -> Target:
requested_model: Final = str(request.resolved["model"])
model_info: Final = request.resolved.get("model_info")
api_base: Final = request.resolved.get("api_base")
native_config: Final = main._native_messages_config( # pyright: ignore[reportPrivateUsage] # the handler's own passthrough lookup
model, provider, model_info, api_base if isinstance(api_base, str) else None
route: Final = plan_messages_route(
requested_model=str(request.resolved["model"]),
resolved_model=model,
custom_llm_provider=provider,
model_info=request.resolved.get("model_info"),
api_base=api_base if isinstance(api_base, str) else None,
compaction_requested=request.resolved.get("compaction") is not None,
force_chat_completions=bool(litellm.use_chat_completions_url_for_anthropic_messages),
stop_requested=bool(request.resolved.get("stop_sequences")),
)
if native_config is not None:
if isinstance(route, NativeMessages):
return Target.MESSAGES
compacting: Final = request.resolved.get("compaction") is not None
if not compacting and main._should_route_to_responses_api(provider, requested_model, model, model_info): # pyright: ignore[reportPrivateUsage] # the handler's own translation choice
if isinstance(route, NativeResponses):
return Target.RESPONSES
return Target.CHAT_COMPLETIONS

1342
litellm/native/chat.py Normal file

File diff suppressed because it is too large Load diff

281
litellm/native/chat_mock.py Normal file
View file

@ -0,0 +1,281 @@
import asyncio
import time
from typing import (
Final,
cast, # noqa: TID251 # casts moved verbatim from litellm/main.py
)
import httpx
import openai
import litellm
from litellm.constants import DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT
from litellm.llms.base_llm.base_model_iterator import convert_model_response_to_streaming
from litellm.native.request_support import admitted_prompt_token_counter
from litellm.types.utils import ChatCompletionMessageToolCall, ModelResponseStream
from litellm.utils import (
CustomStreamWrapper,
ModelResponse,
Usage,
async_mock_completion_streaming_obj,
mock_completion_streaming_obj,
)
MOCK_RESPONSE_TYPE = str | Exception | dict | ModelResponse | ModelResponseStream
def handle_mock_potential_exceptions(
mock_response: str | Exception,
model: str,
custom_llm_provider: str | None = None,
) -> None:
if isinstance(mock_response, Exception):
if isinstance(mock_response, openai.APIError):
raise mock_response
raise litellm.MockException(
status_code=getattr(mock_response, "status_code", 500),
message=getattr(mock_response, "text", str(mock_response)),
llm_provider=getattr(mock_response, "llm_provider", custom_llm_provider or "openai"),
model=model,
request=httpx.Request(method="POST", url="https://api.openai.com/v1/"),
)
elif isinstance(mock_response, str) and mock_response == "litellm.RateLimitError":
raise litellm.RateLimitError(
message="this is a mock rate limit error",
llm_provider=getattr(mock_response, "llm_provider", custom_llm_provider or "openai"),
model=model,
)
elif isinstance(mock_response, str) and mock_response == "litellm.ContextWindowExceededError":
raise litellm.ContextWindowExceededError(
message="this is a mock context window exceeded error",
llm_provider=getattr(mock_response, "llm_provider", custom_llm_provider or "openai"),
model=model,
)
elif isinstance(mock_response, str) and mock_response == "litellm.InternalServerError":
raise litellm.InternalServerError(
message="this is a mock internal server error",
llm_provider=getattr(mock_response, "llm_provider", custom_llm_provider or "openai"),
model=model,
)
elif isinstance(mock_response, str) and mock_response.startswith("Exception: content_filter_policy"):
raise litellm.MockException(
status_code=400,
message=mock_response,
llm_provider="azure",
model=model,
request=httpx.Request(method="POST", url="https://api.openai.com/v1/"),
)
def handle_mock_timeout(
mock_timeout: bool | None,
timeout: float | str | httpx.Timeout | None,
model: str,
) -> None:
if mock_timeout is True and timeout is not None:
_sleep_for_timeout(timeout)
raise litellm.Timeout(
message="This is a mock timeout error",
llm_provider="openai",
model=model,
)
async def handle_mock_timeout_async(
mock_timeout: bool | None,
timeout: float | str | httpx.Timeout | None,
model: str,
) -> None:
if mock_timeout is True and timeout is not None:
await _sleep_for_timeout_async(timeout)
raise litellm.Timeout(
message="This is a mock timeout error",
llm_provider="openai",
model=model,
)
def _sleep_for_timeout(timeout: float | str | httpx.Timeout):
if isinstance(timeout, float):
time.sleep(timeout)
elif isinstance(timeout, str):
time.sleep(float(timeout))
elif isinstance(timeout, httpx.Timeout) and timeout.connect is not None:
time.sleep(timeout.connect)
async def _sleep_for_timeout_async(timeout: float | str | httpx.Timeout):
if isinstance(timeout, float):
await asyncio.sleep(timeout)
elif isinstance(timeout, str):
await asyncio.sleep(float(timeout))
elif isinstance(timeout, httpx.Timeout) and timeout.connect is not None:
await asyncio.sleep(timeout.connect)
def mock_completion(
model: str,
messages: list,
stream: bool | None = False,
n: int | None = None,
mock_response: MOCK_RESPONSE_TYPE | None = "This is a mock request",
mock_tool_calls: list | None = None,
mock_timeout: bool | None = False,
logging=None,
custom_llm_provider=None,
timeout: float | str | httpx.Timeout | None = None,
prompt_tokens: int | None = None,
**kwargs,
):
"""
Generate a mock completion response for testing or debugging purposes.
This is a helper function that simulates the response structure of the OpenAI completion API.
Parameters:
model (str): The name of the language model for which the mock response is generated.
messages (List): A list of message objects representing the conversation context.
stream (bool, optional): If True, returns a mock streaming response (default is False).
mock_response (str, optional): The content of the mock response (default is "This is a mock request").
mock_timeout (bool, optional): If True, the mock response will be a timeout error (default is False).
timeout (float, optional): The timeout value to use for the mock response (default is None).
**kwargs: Additional keyword arguments that can be used but are not required.
Returns:
litellm.ModelResponse: A ModelResponse simulating a completion response with the specified model, messages, and mock response.
Raises:
Exception: If an error occurs during the generation of the mock completion response.
Note:
- This function is intended for testing or debugging purposes to generate mock completion responses.
- If 'stream' is True, it returns a response that mimics the behavior of a streaming completion.
"""
try:
is_acompletion: Final = kwargs.get("acompletion") or False
if mock_response is None:
mock_response = "This is a mock request"
handle_mock_timeout(mock_timeout=mock_timeout, timeout=timeout, model=model)
## LOGGING
if logging is not None:
logging.pre_call(
input=messages,
api_key="mock-key",
)
if isinstance(mock_response, str) or isinstance(mock_response, Exception):
handle_mock_potential_exceptions(
mock_response=mock_response,
model=model,
custom_llm_provider=custom_llm_provider,
)
mock_response = cast(
str | dict | ModelResponse | ModelResponseStream, mock_response
) # after this point, mock_response is a string, dict, ModelResponse, or ModelResponseStream
if isinstance(mock_response, str) and mock_response.startswith("Exception: mock_streaming_error"):
mock_response = litellm.MockException(
message="This is a mock error raised mid-stream",
llm_provider="anthropic",
model=model,
status_code=529,
)
time_delay: Final = kwargs.get("mock_delay", None)
if time_delay is not None and not is_acompletion:
time.sleep(time_delay)
if isinstance(mock_response, dict):
return ModelResponse(**mock_response)
if isinstance(mock_response, ModelResponse):
if not stream:
return mock_response
# convert to ModelResponseStream
mock_response = convert_model_response_to_streaming(mock_response)
model_response: ModelResponse | ModelResponseStream = ModelResponse()
if stream is True:
model_response = ModelResponseStream()
count_prompt_tokens: Final = admitted_prompt_token_counter(prompt_tokens)
# don't try to access stream object,
if kwargs.get("acompletion", False) is True:
return CustomStreamWrapper(
completion_stream=async_mock_completion_streaming_obj(
model_response, mock_response=mock_response, model=model, n=n, prompt_tokens=prompt_tokens
),
model=model,
custom_llm_provider="openai",
logging_obj=logging,
count_prompt_tokens=count_prompt_tokens,
)
return CustomStreamWrapper(
completion_stream=mock_completion_streaming_obj(
model_response, mock_response=mock_response, model=model, n=n, prompt_tokens=prompt_tokens
),
model=model,
custom_llm_provider="openai",
logging_obj=logging,
count_prompt_tokens=count_prompt_tokens,
)
if isinstance(mock_response, litellm.MockException):
raise mock_response
# At this point, mock_response must be a string (all other types have been handled or returned early)
mock_response = cast(str, mock_response)
if n is None:
model_response.choices[0].message.content = mock_response
else:
_all_choices: Final = []
for i in range(n):
_choice = litellm.utils.Choices(
index=i,
message=litellm.utils.Message(content=mock_response, role="assistant"),
)
_all_choices.append(_choice)
model_response.choices = _all_choices
model_response.created = int(time.time())
model_response.model = model
if mock_tool_calls:
model_response.choices[0].message.tool_calls = [
ChatCompletionMessageToolCall(**tool_call) for tool_call in mock_tool_calls
]
usage_prompt_tokens: Final = (
prompt_tokens if prompt_tokens is not None else DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT
)
setattr(
model_response,
"usage",
Usage(
prompt_tokens=usage_prompt_tokens,
completion_tokens=DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT,
total_tokens=usage_prompt_tokens + DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT,
),
)
if custom_llm_provider is not None:
model_response.hidden_params["custom_llm_provider"] = custom_llm_provider
else:
try:
_, inferred_provider, _, _ = litellm.utils.get_llm_provider(model=model)
model_response.hidden_params["custom_llm_provider"] = inferred_provider
except Exception:
# dont let setting a hidden param block a mock_respose
pass
if logging is not None:
logging.post_call(
input=messages,
api_key="my-secret-key",
original_response="my-original-response",
)
return model_response
except Exception as e:
if isinstance(e, openai.APIError):
raise e
raise Exception(f"Mock completion response failed - {e}")

283
litellm/native/chat_plan.py Normal file
View file

@ -0,0 +1,283 @@
import asyncio
import contextvars
from collections.abc import Iterable, Mapping
from dataclasses import dataclass
from functools import partial
from types import MappingProxyType
from typing import (
Final,
cast, # noqa: TID251 # casts moved verbatim from litellm/main.py
)
from pydantic import TypeAdapter
import litellm
from litellm.litellm_core_utils.chat_completion_agentic_loop import maybe_run_chat_completion_agentic_loop
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.prompt_templates.common_utils import get_completion_messages
from litellm.litellm_core_utils.rules import Rules
from litellm.native.chat import OPTIONAL_PARAMS_ADAPTER, execute_chat, resolve_dispatched_chat_response
from litellm.native.request_support import resolve_control_options
from litellm.types.litellm_params import ControlOptions
from litellm.types.llms.anthropic import AnthropicThinkingParam
from litellm.utils import (
CustomStreamWrapper,
ModelResponse,
exception_type,
get_llm_provider,
get_non_default_completion_params,
post_call_processing,
validate_and_fix_openai_messages,
validate_and_fix_openai_tools,
validate_and_fix_thinking_param,
validate_chat_completion_tool_choice,
validate_openai_optional_params,
)
_PLANNED_CHAT_ARGS: Final = frozenset(
("kwargs", "non_default_params", "control_options", "prompt_id", "prompt_variables")
)
_EXECUTE_CHAT_CODE: Final = execute_chat.__code__
_EXECUTE_CHAT_PARAMS: Final = (
frozenset(_EXECUTE_CHAT_CODE.co_varnames[: _EXECUTE_CHAT_CODE.co_kwonlyargcount]) - _PLANNED_CHAT_ARGS
)
_SKIP_RESPONSES_API_BRIDGE_KWARG: Final = "_skip_responses_api_bridge"
_CHAT_SURFACE_KWARGS: Final = (
"fallbacks",
"model_list",
"prompt_id",
"litellm_system_prompt",
"ensure_alternating_roles",
"user_continue_message",
"assistant_continue_message",
"mock_delay",
"mock_timeout",
)
_TEXT_COMPLETION_PROVIDERS: Final = frozenset(
("text-completion-openai", "text-completion-codestral", "text-completion-inception")
)
@dataclass(frozen=True, slots=True)
class PlannedChat:
model: str
custom_llm_provider: str
named: Mapping[str, object]
kwargs: Mapping[str, object]
non_default_params: Mapping[str, object]
control_options: ControlOptions
def _optional_str(value: object) -> str | None:
return value if isinstance(value, str) else None
_OBJECT_LIST: Final = TypeAdapter(list[object])
def _object_list(value: object) -> list[object] | None: # mutable-ok: feeds validate_and_fix_openai_* which take a list
return _OBJECT_LIST.validate_python(value) if isinstance(value, list) else None
def _tool_choice(value: object) -> str | dict[str, object] | None: # mutable-ok: its validator takes a dict
if isinstance(value, str):
return value
return OPTIONAL_PARAMS_ADAPTER.validate_python(value) if isinstance(value, dict) else None
def _stop(value: object) -> str | list[str] | None: # mutable-ok: validate_openai_optional_params takes a list
if isinstance(value, list):
return [item for item in _OBJECT_LIST.validate_python(value) if isinstance(item, str)]
return value if isinstance(value, str) else None
def _thinking(value: object) -> AnthropicThinkingParam | bool | None:
if isinstance(value, dict):
return cast(AnthropicThinkingParam, value) # cast-ok: the caller's thinking dict, normalized by the validator
return value if isinstance(value, bool) else None
def _chat_entry_points_wrapped() -> bool:
"""Tracers such as ddtrace wrap `litellm.completion`/`acompletion`, so bridged calls must go through them."""
from litellm.chat_completions import dispatch
return litellm.completion is not dispatch.completion or litellm.acompletion is not dispatch.acompletion
def _reads_chat_cache(request: Mapping[str, object]) -> bool:
"""The chat response cache is read and written by the `acompletion` surface, keyed on the chat request."""
return litellm.cache is not None or request.get("caching") is True
def _uses_chat_surface(request: Mapping[str, object]) -> bool:
from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler
from litellm.types.llms.openai import ToolParam
tools: Final = request.get("tools")
return (
litellm.model_fallbacks is not None
or _reads_chat_cache(request)
or _chat_entry_points_wrapped()
or any(request.get(key) is not None for key in _CHAT_SURFACE_KWARGS)
or (
isinstance(tools, list)
and LiteLLM_Proxy_MCP_Handler.should_use_litellm_mcp_gateway(
tools=cast(Iterable[ToolParam], tools) # cast-ok: OpenAI-format tools from the caller's request
)
)
)
def plan_chat(request: Mapping[str, object], logging_obj: LiteLLMLoggingObj) -> PlannedChat | None:
model: Final = request.get("model")
if not isinstance(model, str) or _uses_chat_surface(request):
return None
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
custom_llm_provider: Final = _optional_str(request.get("custom_llm_provider"))
api_base: Final = _optional_str(request.get("api_base"))
_, resolved_provider, _, _ = get_llm_provider(
model=model, custom_llm_provider=custom_llm_provider, api_base=api_base
)
if resolved_provider in _TEXT_COMPLETION_PROVIDERS:
return None
kwargs: Final = {
key: value
for key, value in request.items()
if key not in _EXECUTE_CHAT_PARAMS and key != _SKIP_RESPONSES_API_BRIDGE_KWARG
}
tools: Final = validate_and_fix_openai_tools(tools=_object_list(request.get("tools")))
messages: Final = get_completion_messages(
messages=validate_and_fix_openai_messages(messages=_object_list(request.get("messages")) or []),
ensure_alternating_roles=False,
user_continue_message=None,
assistant_continue_message=None,
)
non_default_params: Final = get_non_default_completion_params(kwargs=kwargs)
AnthropicCacheControlHook.maybe_seed_default_injection_points(
non_default_params=non_default_params,
messages=messages,
model=model,
custom_llm_provider=custom_llm_provider,
tools=tools,
enable_prompt_caching=request.get("enable_prompt_caching") is True,
api_base=api_base,
)
if logging_obj.should_run_prompt_management_hooks(non_default_params=non_default_params, tools=tools):
return None
named: Final = {key: value for key, value in request.items() if key in _EXECUTE_CHAT_PARAMS}
tool_choice: Final = validate_chat_completion_tool_choice(
tool_choice=_tool_choice(request.get("tool_choice")), model=model
)
return PlannedChat(
model=model,
custom_llm_provider=resolved_provider,
named=MappingProxyType(
{
**named,
"messages": messages,
"tools": tools,
"tool_choice": tool_choice,
"stop": validate_openai_optional_params(stop=_stop(request.get("stop"))),
"thinking": validate_and_fix_thinking_param(thinking=_thinking(request.get("thinking"))),
"skip_responses_api_bridge": request.get(_SKIP_RESPONSES_API_BRIDGE_KWARG) is True,
}
),
kwargs=MappingProxyType(kwargs),
non_default_params=MappingProxyType(non_default_params),
control_options=resolve_control_options(kwargs, model),
)
def _run_planned_chat(plan: PlannedChat, is_async: bool) -> ModelResponse | CustomStreamWrapper:
return execute_chat(
**plan.named, # pyright: ignore[reportArgumentType] # named holds the caller's validated OpenAI-format params
kwargs={**plan.kwargs, "acompletion": is_async},
non_default_params=dict(plan.non_default_params),
control_options=plan.control_options,
prompt_id=None,
prompt_variables=None,
)
def _post_call_checked(
plan: PlannedChat, response: ModelResponse | CustomStreamWrapper, is_async: bool
) -> ModelResponse | CustomStreamWrapper:
"""Runs the post-call rules and JSON-schema validation the ``completion`` client wrapper runs on its result."""
post_call_processing(
original_response=response,
model=plan.model,
optional_params={**plan.kwargs, **plan.named},
original_function=litellm.acompletion if is_async else litellm.completion,
rules_obj=Rules(),
)
return response
def execute_planned_chat(plan: PlannedChat) -> ModelResponse | CustomStreamWrapper:
return _post_call_checked(plan, _run_planned_chat(plan, is_async=False), is_async=False)
async def _settled_chat_response(
plan: PlannedChat, loop: asyncio.AbstractEventLoop
) -> ModelResponse | CustomStreamWrapper:
pending: Final[object] = await loop.run_in_executor(
None, partial(contextvars.copy_context().run, partial(_run_planned_chat, plan, True))
)
if asyncio.iscoroutine(pending):
return await resolve_dispatched_chat_response(pending)
if isinstance(pending, (ModelResponse, CustomStreamWrapper)):
return pending
raise TypeError(f"chat provider returned {type(pending).__name__}, expected a ModelResponse or a stream")
async def _with_agentic_loop(plan: PlannedChat, response: ModelResponse) -> ModelResponse | CustomStreamWrapper:
looped: Final = await maybe_run_chat_completion_agentic_loop(
response=response,
model=plan.model,
messages=_object_list(plan.named["messages"]) or [],
optional_params={
key: value
for key, value in plan.named.items()
if value is not None and key not in ("model", "messages", "stream", "deployment_id")
},
kwargs=dict(plan.kwargs),
logging_obj=plan.kwargs.get("litellm_logging_obj"),
custom_llm_provider=plan.custom_llm_provider,
stream=plan.named.get("stream") is True,
)
return response if looped is None else looped
async def _aexecute_planned_chat(plan: PlannedChat) -> ModelResponse | CustomStreamWrapper:
loop: Final = asyncio.get_event_loop()
response: Final = await _settled_chat_response(plan, loop)
if isinstance(response, CustomStreamWrapper):
response.set_logging_event_loop(loop=loop)
return response
return await _with_agentic_loop(plan, response)
async def aexecute_planned_chat(plan: PlannedChat) -> ModelResponse | CustomStreamWrapper:
"""Maps a provider failure raised while awaiting the call, as ``acompletion`` does."""
try:
response: Final = await _aexecute_planned_chat(plan)
except Exception as e: # noqa: BLE001 # every provider failure maps to the litellm exception acompletion raises
raise exception_type(
model=plan.model,
custom_llm_provider=plan.custom_llm_provider or "openai",
original_exception=e,
completion_kwargs=dict(plan.named),
extra_kwargs=dict(plan.kwargs),
) from e
return _post_call_checked(plan, response, is_async=True)

View file

@ -0,0 +1,273 @@
from collections.abc import Mapping, Sequence
from typing import Final, cast # noqa: TID251 # casts moved verbatim from litellm/main.py
from pydantic import TypeAdapter
import litellm
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.native.chat_providers.handlers import base_llm_http_handler
from litellm.native.chat_providers.support import dispatch_client_http, get_encoding
from litellm.secret_managers.main import get_secret_str
from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult
from litellm.types.llms.openai import AllMessageValues
def complete_ragflow(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
provider_config: Final = ctx.provider_config
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
try:
response: Final = base_llm_http_handler.completion(
model=model,
messages=messages,
headers=headers,
model_response=model_response,
api_key=api_key,
api_base=api_base,
acompletion=acompletion,
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
timeout=timeout,
client=client,
custom_llm_provider=custom_llm_provider,
encoding=get_encoding(),
stream=stream,
provider_config=provider_config,
)
except Exception as e:
logging.post_call(
input=messages,
api_key=api_key,
original_response=str(e),
additional_args={"headers": headers},
)
raise e
return response
def complete_a2a(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
provider_config: Final = ctx.provider_config
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
(
api_base,
api_key,
headers,
) = litellm.A2AConfig.resolve_agent_config_from_registry(
agent_name=model,
api_base=api_base,
api_key=api_key,
headers=headers,
optional_params=optional_params,
)
# Fall back to environment variables and defaults
api_base = api_base or litellm.api_base or get_secret_str("A2A_API_BASE")
if api_base is None:
raise Exception(
"api_base is required for A2A provider. "
"Either provide api_base parameter, set A2A_API_BASE environment variable, "
"or register the agent in the proxy with model='a2a/<agent-name>'."
)
headers = headers or litellm.headers
return base_llm_http_handler.completion(
model=model,
stream=stream,
messages=messages,
acompletion=acompletion,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider=custom_llm_provider,
timeout=timeout,
headers=headers,
encoding=get_encoding(),
api_key=api_key,
logging_obj=logging,
client=client,
provider_config=provider_config,
)
def complete_langgraph(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
from litellm.llms.langgraph.chat.transformation import LangGraphConfig
(
api_base,
api_key,
) = LangGraphConfig().get_openai_compatible_provider_info(
api_base=api_base or litellm.api_base,
api_key=api_key or litellm.api_key,
)
headers = headers or litellm.headers
return base_llm_http_handler.completion(
model=model,
stream=stream,
messages=messages,
acompletion=acompletion,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider=custom_llm_provider,
timeout=timeout,
headers=headers,
encoding=get_encoding(),
api_key=api_key,
logging_obj=logging,
client=client,
)
def complete_microsoft_365_copilot(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
from litellm.llms.microsoft_365_copilot.chat.handler import acompletion, completion
from litellm.llms.microsoft_365_copilot.common_utils import as_secret_fields
messages: Final[Sequence[AllMessageValues]] = cast( # cast-ok: dispatch messages are prevalidated
Sequence[AllMessageValues],
ctx.messages,
)
optional_params_value: Final[object] = cast( # cast-ok: dispatch params are validated below
object,
ctx.optional_params,
)
optional_params: Final = TypeAdapter(Mapping[str, object]).validate_python(optional_params_value)
kwargs_value: Final[object] = cast( # cast-ok: dispatch kwargs are validated below
object,
ctx.kwargs,
)
kwargs: Final = TypeAdapter(Mapping[str, object]).validate_python(kwargs_value)
secret_fields: Final = as_secret_fields(kwargs.get("secret_fields"))
client: Final = dispatch_client_http(ctx)
if ctx.acompletion:
async_client: Final = client if isinstance(client, AsyncHTTPHandler) else None
return acompletion(
model=ctx.model,
messages=messages,
api_key=ctx.api_key,
litellm_params=ctx.litellm_params,
optional_params=optional_params,
secret_fields=secret_fields,
stream=ctx.stream is True,
timeout=ctx.timeout,
logging_obj=ctx.logging,
client=async_client,
shared_session=ctx.shared_session,
)
sync_client: Final = client if isinstance(client, HTTPHandler) else None
return completion(
model=ctx.model,
messages=messages,
api_key=ctx.api_key,
litellm_params=ctx.litellm_params,
optional_params=optional_params,
secret_fields=secret_fields,
stream=ctx.stream is True,
timeout=ctx.timeout,
logging_obj=ctx.logging,
client=sync_client,
)
def complete_langflow(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
from litellm.llms.langflow.chat.transformation import LangFlowConfig
(
api_base,
api_key,
) = LangFlowConfig().get_openai_compatible_provider_info(
api_base=api_base or litellm.api_base,
api_key=api_key or litellm.api_key,
)
headers = headers or litellm.headers
return base_llm_http_handler.completion(
model=model,
stream=stream,
messages=messages,
acompletion=acompletion,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider=custom_llm_provider,
timeout=timeout,
headers=headers,
encoding=get_encoding(),
api_key=api_key,
logging_obj=logging,
client=client,
)

View file

@ -0,0 +1,132 @@
import os
from typing import Final, cast # noqa: TID251 # casts moved verbatim from litellm/main.py
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.native.chat_providers.handlers import anthropic_chat_completions, base_llm_http_handler
from litellm.native.chat_providers.support import dispatch_client_http, get_encoding, print_verbose
from litellm.secret_managers.main import get_secret_bool
from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult
from litellm.utils import get_secret
def complete_anthropic_text(
ctx: CompletionDispatchContext,
) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
custom_prompt_dict = ctx.custom_prompt_dict
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
api_key = api_key or litellm.anthropic_key or litellm.api_key or os.environ.get("ANTHROPIC_API_KEY")
custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict
api_base = cast(
str | None,
api_base
or litellm.api_base
or get_secret("ANTHROPIC_API_BASE")
or get_secret("ANTHROPIC_BASE_URL")
or "https://api.anthropic.com/v1/complete",
)
# Check if we should disable automatic URL suffix appending
disable_url_suffix: Final = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX")
if api_base is not None and not disable_url_suffix:
api_base = ensure_api_route(api_base, "v1/complete")
elif disable_url_suffix:
verbose_logger.debug("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/complete suffix")
return base_llm_http_handler.completion(
model=model,
stream=stream,
messages=messages,
acompletion=acompletion,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider="anthropic_text",
timeout=timeout,
headers=headers,
encoding=get_encoding(),
api_key=api_key,
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
)
def complete_anthropic(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
custom_prompt_dict = ctx.custom_prompt_dict
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logger_fn: Final = ctx.logger_fn
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
timeout: Final = ctx.timeout
api_key = api_key or litellm.anthropic_key or litellm.api_key or os.environ.get("ANTHROPIC_API_KEY")
custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict
# call /messages
# default route for all anthropic models
api_base = cast(
str | None,
api_base
or litellm.api_base
or get_secret("ANTHROPIC_API_BASE")
or get_secret("ANTHROPIC_BASE_URL")
or "https://api.anthropic.com/v1/messages",
)
# Check if we should disable automatic URL suffix appending
disable_url_suffix: Final = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX")
if api_base is not None and not disable_url_suffix:
api_base = ensure_api_route(api_base, "v1/messages")
elif disable_url_suffix:
verbose_logger.debug("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/messages suffix")
response: Final = anthropic_chat_completions.completion(
model=model,
messages=messages,
api_base=api_base,
acompletion=acompletion,
custom_prompt_dict=litellm.custom_prompt_dict,
model_response=model_response,
print_verbose=print_verbose,
optional_params=optional_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
encoding=get_encoding(), # for calculating input/output tokens
api_key=api_key,
logging_obj=logging,
headers=headers,
timeout=timeout,
client=client,
custom_llm_provider=custom_llm_provider,
)
if optional_params.get("stream", False) or acompletion is True:
## LOGGING
logging.post_call(
input=messages,
api_key=api_key,
original_response=response,
)
return response

View file

@ -0,0 +1,400 @@
from collections.abc import Mapping
from functools import partial
from typing import Final
import httpx
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import (
AmazonAnthropicClaudeConfig,
)
from litellm.llms.bedrock.chat.native_output_fallback import NativeOutputAttemptResult, call_with_native_output_fallback
from litellm.llms.bedrock.common_utils import (
BedrockModelInfo,
NativeOutputFallback,
bedrock_route_for_request,
bedrock_strict_tools_fallback,
deployment_model_info,
without_bedrock_native_output_opt_in,
without_bedrock_route_prefix,
)
from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantle_chat_config
from litellm.native.chat_providers.handlers import (
base_llm_http_handler,
bedrock_converse_chat_completion,
openai_like_chat_completion,
sagemaker_llm,
)
from litellm.native.chat_providers.support import (
dispatch_client_http,
get_encoding,
print_verbose,
provider_config_items,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager, get_secret
_ADDITIONAL_DROP_PARAMS_ADAPTER: Final = TypeAdapter(list[str])
_OPTIONAL_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object])
def complete_bedrock_mantle(
ctx: CompletionDispatchContext,
) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
api_base = api_base or litellm.api_base or get_secret("BEDROCK_MANTLE_API_BASE")
api_key = api_key or litellm.api_key or get_secret("BEDROCK_MANTLE_API_KEY")
headers = headers or litellm.headers
config: Final = bedrock_mantle_chat_config(model).get_config()
for k, v in provider_config_items(config):
if k not in optional_params:
optional_params[k] = v
return base_llm_http_handler.completion(
model=model,
stream=stream,
messages=messages,
acompletion=acompletion,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider=custom_llm_provider,
timeout=timeout,
headers=headers,
encoding=get_encoding(),
api_key=api_key,
logging_obj=logging,
client=client,
)
def complete_amazon_nova(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
api_base = ctx.api_base
api_key = ctx.api_key
custom_llm_provider: Final = ctx.custom_llm_provider
custom_prompt_dict: Final = ctx.custom_prompt_dict
litellm_params: Final = ctx.litellm_params
logger_fn: Final = ctx.logger_fn
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
timeout: Final = ctx.timeout
api_key = api_key or litellm.amazon_nova_api_key or get_secret_str("AMAZON_NOVA_API_KEY") or litellm.api_key
api_base = (
api_base or litellm.api_base or get_secret_str("AMAZON_NOVA_API_BASE") or "https://api.nova.amazon.com/v1"
)
return openai_like_chat_completion.completion(
model=model,
messages=messages,
api_base=api_base,
model_response=model_response,
print_verbose=print_verbose,
optional_params=optional_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
encoding=get_encoding(),
api_key=api_key,
logging_obj=logging,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
custom_prompt_dict=custom_prompt_dict,
)
def complete_sagemaker_chat(
ctx: CompletionDispatchContext,
) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
stream: Final = ctx.stream
timeout: Final = ctx.timeout
return base_llm_http_handler.completion(
model=model,
stream=stream,
messages=messages,
acompletion=acompletion,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
custom_llm_provider=custom_llm_provider,
timeout=timeout,
headers=headers,
encoding=get_encoding(),
api_key=api_key,
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
client=client,
)
def complete_sagemaker(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
custom_prompt_dict: Final = ctx.custom_prompt_dict
hf_model_name: Final = ctx.hf_model_name
litellm_params: Final = ctx.litellm_params
logger_fn: Final = ctx.logger_fn
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
return sagemaker_llm.completion(
model=model,
messages=messages,
model_response=model_response,
print_verbose=print_verbose,
optional_params=optional_params,
litellm_params=litellm_params,
custom_prompt_dict=custom_prompt_dict,
hf_model_name=hf_model_name,
logger_fn=logger_fn,
encoding=get_encoding(),
logging_obj=logging,
acompletion=acompletion,
)
_AWS_BEDROCK_CLIENT_CREDENTIAL_KEYS: Final = (
"aws_access_key_id",
"aws_secret_access_key",
"aws_session_token",
"aws_region_name",
)
def _native_output_fallback(
ctx: CompletionDispatchContext,
first_attempt_params: Mapping[str, object],
first_attempt_litellm_params: Mapping[str, object],
) -> NativeOutputFallback:
litellm_params: Final = without_bedrock_native_output_opt_in(first_attempt_litellm_params)
remapped_params: Final = ctx.optional_params_for_model_info(deployment_model_info(litellm_params))
return NativeOutputFallback(
optional_params={
**{key: value for key, value in remapped_params.items() if key != "aws_bedrock_client"},
**{
key: first_attempt_params[key]
for key in _AWS_BEDROCK_CLIENT_CREDENTIAL_KEYS
if key in first_attempt_params
},
},
litellm_params=litellm_params,
)
def _converse_native_output_fallback(
ctx: CompletionDispatchContext,
first_attempt_params: Mapping[str, object],
first_attempt_litellm_params: Mapping[str, object],
) -> NativeOutputFallback | None:
fallback: Final = _native_output_fallback(ctx, first_attempt_params, first_attempt_litellm_params)
return None if AmazonConverseConfig.sends_native_text_format(fallback.optional_params) else fallback
def _native_output_attempt_timeout(
ctx: CompletionDispatchContext, fallback: NativeOutputFallback | None
) -> float | str | httpx.Timeout | None:
if fallback is None:
return ctx.timeout
retry_timeout: Final = ctx.timeout_for_optional_params(fallback.optional_params)
ctx.logging.record_applied_timeout(retry_timeout)
return retry_timeout
def complete_bedrock(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_prompt_dict = ctx.custom_prompt_dict
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logger_fn: Final = ctx.logger_fn
logging: Final = ctx.logging
messages: Final = ctx.messages
model = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
provider_config = ctx.provider_config
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict
if "aws_bedrock_client" in optional_params:
verbose_logger.warning(
"'aws_bedrock_client' is a deprecated param. Please move to another auth method - https://docs.litellm.ai/docs/providers/bedrock#boto3---authentication."
)
# Extract credentials for legacy boto3 client and pass thru to httpx
aws_bedrock_client: Final = optional_params.pop("aws_bedrock_client")
creds: Final = aws_bedrock_client._get_credentials().get_frozen_credentials()
if creds.access_key:
optional_params["aws_access_key_id"] = creds.access_key
if creds.secret_key:
optional_params["aws_secret_access_key"] = creds.secret_key
if creds.token:
optional_params["aws_session_token"] = creds.token
if "aws_region_name" not in optional_params or optional_params["aws_region_name"] is None:
optional_params["aws_region_name"] = aws_bedrock_client.meta.region_name
additional_drop_params: Final = (
_ADDITIONAL_DROP_PARAMS_ADAPTER.validate_python(ctx.kwargs["additional_drop_params"])
if ctx.kwargs.get("additional_drop_params") is not None
else None
)
bedrock_route: Final = bedrock_route_for_request(model, ctx.request_params, additional_drop_params)
if bedrock_route == "claude_platform":
provider_config = ProviderConfigManager.get_provider_chat_config(
model=model,
provider=LlmProviders.BEDROCK,
)
model = BedrockModelInfo.get_claude_platform_model(model)
return base_llm_http_handler.completion(
model=model,
stream=stream,
messages=messages,
acompletion=acompletion,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider="bedrock",
timeout=timeout,
headers=headers,
encoding=get_encoding(),
api_key=api_key,
logging_obj=logging,
client=client,
provider_config=provider_config,
)
elif bedrock_route == "converse":
model = without_bedrock_route_prefix(model)
converse_model: Final = model
def converse_attempt(fallback: NativeOutputFallback | None) -> NativeOutputAttemptResult:
return bedrock_converse_chat_completion.completion(
model=converse_model,
messages=messages,
custom_prompt_dict=custom_prompt_dict,
model_response=model_response,
optional_params=optional_params if fallback is None else dict(fallback.optional_params),
litellm_params=litellm_params if fallback is None else dict(fallback.litellm_params),
logger_fn=logger_fn,
encoding=get_encoding(),
logging_obj=logging,
extra_headers=headers, # Use merged headers instead of original extra_headers
timeout=_native_output_attempt_timeout(ctx, fallback),
acompletion=acompletion,
client=client,
api_base=api_base,
api_key=api_key,
)
converse_params: Final = _OPTIONAL_PARAMS_ADAPTER.validate_python(optional_params)
response = call_with_native_output_fallback(
converse_attempt,
(
partial(_converse_native_output_fallback, ctx, converse_params, dict(litellm_params))
if AmazonConverseConfig().uses_opt_in_native_output(converse_model, converse_params, litellm_params)
else None
),
converse_model,
strict_tools_fallback=(
partial(bedrock_strict_tools_fallback, converse_model, converse_params, dict(litellm_params))
if drops_unsupported_params(litellm_params.get("drop_params"))
else None
),
)
elif bedrock_route == "converse_like":
model = model.replace("converse_like/", "")
response = base_llm_http_handler.completion(
model=model,
stream=stream,
messages=messages,
acompletion=acompletion,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
custom_llm_provider="bedrock",
timeout=timeout,
headers=headers,
encoding=get_encoding(),
api_key=api_key,
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
client=client,
)
else:
def invoke_attempt(fallback: NativeOutputFallback | None) -> NativeOutputAttemptResult:
return base_llm_http_handler.completion(
model=model,
stream=stream,
messages=messages,
acompletion=acompletion,
api_base=api_base,
model_response=model_response,
optional_params=optional_params if fallback is None else dict(fallback.optional_params),
litellm_params=litellm_params if fallback is None else dict(fallback.litellm_params),
custom_llm_provider="bedrock",
timeout=_native_output_attempt_timeout(ctx, fallback),
headers=headers,
encoding=get_encoding(),
api_key=api_key,
logging_obj=logging,
client=client,
)
invoke_params: Final = _OPTIONAL_PARAMS_ADAPTER.validate_python(optional_params)
response = call_with_native_output_fallback(
invoke_attempt,
(
partial(_native_output_fallback, ctx, invoke_params, dict(litellm_params))
if isinstance(provider_config, AmazonAnthropicClaudeConfig)
and provider_config.uses_opt_in_native_output(model, invoke_params, litellm_params)
else None
),
model,
)
return response

View file

@ -0,0 +1,399 @@
from typing import Final, cast # noqa: TID251 # casts moved verbatim from litellm/main.py
import openai
import litellm
from litellm.litellm_core_utils.prompt_templates.factory import stringify_json_tool_call_content
from litellm.llms.azure.azure import check_dynamic_azure_params
from litellm.llms.azure_ai.anthropic.transformation import azure_anthropic_messages_url
from litellm.native.chat_providers.handlers import (
azure_anthropic_chat_completions,
azure_chat_completions,
azure_o1_chat_completions,
azure_text_completions,
base_llm_http_handler,
)
from litellm.native.chat_providers.support import (
dispatch_client_azure,
dispatch_client_http,
get_encoding,
print_verbose,
provider_config_items,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult
from litellm.utils import get_secret
def complete_azure(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
_azure_detection_model: Final = ctx._azure_detection_model
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
api_version = ctx.api_version
client: Final = dispatch_client_azure(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
extra_headers: Final = ctx.extra_headers
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
logger_fn: Final = ctx.logger_fn
logging: Final = ctx.logging
max_retries: Final = ctx.max_retries
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
timeout: Final = ctx.timeout
dynamic_params = False
if client is not None and (isinstance(client, openai.AzureOpenAI) or isinstance(client, openai.AsyncAzureOpenAI)):
dynamic_params = check_dynamic_azure_params(
azure_client_params={"api_version": api_version},
azure_client=client,
)
api_type: Final = get_secret("AZURE_API_TYPE") or "azure"
api_base = api_base or litellm.api_base or get_secret("AZURE_API_BASE")
api_version = (
api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") or litellm.AZURE_DEFAULT_API_VERSION
)
api_key = (
api_key
or litellm.api_key
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
)
azure_ad_token: Final = optional_params.get("extra_body", {}).pop("azure_ad_token", None) or get_secret_str(
"AZURE_AD_TOKEN"
)
azure_ad_token_provider_value: Final = litellm_params.get("azure_ad_token_provider", None)
azure_ad_token_provider: Final = azure_ad_token_provider_value if callable(azure_ad_token_provider_value) else None
headers = headers or litellm.headers
if extra_headers is not None:
optional_params["extra_headers"] = extra_headers
if max_retries is not None:
optional_params["max_retries"] = max_retries
if litellm.AzureOpenAIO1Config().is_o_series_model(model=_azure_detection_model):
## LOAD CONFIG - if set
config = litellm.AzureOpenAIO1Config.get_config()
for k, v in provider_config_items(config):
if (
k not in optional_params
): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in
optional_params[k] = v
response = azure_o1_chat_completions.completion(
model=model,
messages=messages,
headers=headers,
api_key=api_key,
api_base=api_base,
api_version=api_version,
dynamic_params=dynamic_params,
azure_ad_token=azure_ad_token,
model_response=model_response,
print_verbose=print_verbose,
optional_params=optional_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
logging_obj=logging,
acompletion=acompletion,
timeout=timeout,
client=client, # pass AsyncAzureOpenAI, AzureOpenAI client
custom_llm_provider=custom_llm_provider,
)
else:
## LOAD CONFIG - if set
config = litellm.AzureOpenAIConfig.get_config()
for k, v in provider_config_items(config):
if (
k not in optional_params
): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in
optional_params[k] = v
## COMPLETION CALL
response = azure_chat_completions.completion(
model=model,
messages=messages,
headers=headers,
api_key=api_key,
api_base=api_base,
api_version=api_version,
api_type=api_type,
dynamic_params=dynamic_params,
azure_ad_token=azure_ad_token,
azure_ad_token_provider=azure_ad_token_provider,
model_response=model_response,
print_verbose=print_verbose,
optional_params=optional_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
logging_obj=logging,
acompletion=acompletion,
timeout=timeout,
client=client, # pass AsyncAzureOpenAI, AzureOpenAI client
)
if optional_params.get("stream", False):
## LOGGING
logging.post_call(
input=messages,
api_key=api_key,
original_response=response,
additional_args={
"headers": headers,
"api_version": api_version,
"api_base": api_base,
},
)
return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
def complete_azure_text(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
api_version = ctx.api_version
client: Final = dispatch_client_azure(ctx)
extra_headers: Final = ctx.extra_headers
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
logger_fn: Final = ctx.logger_fn
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
timeout: Final = ctx.timeout
api_type: Final = get_secret_str("AZURE_API_TYPE") or "azure"
api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
if api_base is None:
raise ValueError(
"api_base is required for Azure OpenAI LLM provider. Either set it dynamically or set the AZURE_API_BASE environment variable."
)
api_version = api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION")
api_key = (
api_key
or litellm.api_key
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
)
azure_ad_token: Final = optional_params.get("extra_body", {}).pop("azure_ad_token", None) or get_secret_str(
"AZURE_AD_TOKEN"
)
azure_ad_token_provider_value: Final = litellm_params.get("azure_ad_token_provider", None)
azure_ad_token_provider: Final = azure_ad_token_provider_value if callable(azure_ad_token_provider_value) else None
headers = headers or litellm.headers
if extra_headers is not None:
optional_params["extra_headers"] = extra_headers
## LOAD CONFIG - if set
config: Final = litellm.AzureOpenAIConfig.get_config()
for k, v in provider_config_items(config):
if (
k not in optional_params
): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in
optional_params[k] = v
## COMPLETION CALL
response: Final = azure_text_completions.completion(
model=model,
messages=messages,
headers=headers,
api_key=api_key,
api_base=api_base,
api_version=cast(str, api_version),
api_type=api_type,
azure_ad_token=azure_ad_token,
azure_ad_token_provider=azure_ad_token_provider,
model_response=model_response,
print_verbose=print_verbose,
optional_params=optional_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
logging_obj=logging,
acompletion=acompletion,
timeout=timeout,
client=client, # pass AsyncAzureOpenAI, AzureOpenAI client
)
if optional_params.get("stream", False) or acompletion is True:
## LOGGING
logging.post_call(
input=messages,
api_key=api_key,
original_response=response,
additional_args={
"headers": headers,
"api_version": api_version,
"api_base": api_base,
},
)
return response
def complete_azure_ai(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
extra_headers: Final = ctx.extra_headers
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
logger_fn: Final = ctx.logger_fn
logging: Final = ctx.logging
messages = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
azure_ai_route: Final = AzureFoundryModelInfo.get_azure_ai_route(model)
# Check if this is an agents route - model format: azure_ai/agents/<agent_id>
if azure_ai_route == "agents":
from litellm.llms.azure_ai.agents import AzureAIAgentsConfig
api_base = AzureFoundryModelInfo.get_api_base(api_base)
if api_base is None:
raise ValueError(
"Azure AI Agents requests require an api_base. Set `api_base` or the AZURE_AI_API_BASE env var."
)
api_key = AzureFoundryModelInfo.get_api_key(api_key)
response = AzureAIAgentsConfig.completion(
model=model,
messages=messages,
api_base=api_base,
api_key=api_key,
model_response=model_response,
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
timeout=timeout,
acompletion=acompletion,
stream=stream,
headers=headers or litellm.headers,
)
# Check if this is a Claude model - route to Azure Anthropic handler
elif "claude" in model.lower():
# Use Azure Anthropic handler for Claude models
api_base = AzureFoundryModelInfo.get_api_base(api_base)
if api_base is None:
raise ValueError(
"Azure Anthropic requests require an api_base. Set `api_base` or the AZURE_AI_API_BASE env var."
)
api_key = AzureFoundryModelInfo.get_api_key(api_key)
if api_base:
api_base = azure_anthropic_messages_url(api_base)
response = azure_anthropic_chat_completions.completion(
model=model,
messages=messages,
api_base=api_base,
acompletion=acompletion,
custom_prompt_dict=litellm.custom_prompt_dict,
model_response=model_response,
print_verbose=print_verbose,
optional_params=optional_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
encoding=get_encoding(),
api_key=api_key,
logging_obj=logging,
headers=headers,
timeout=timeout,
client=client,
custom_llm_provider=custom_llm_provider,
)
if optional_params.get("stream", False) or acompletion is True:
## LOGGING
logging.post_call(
input=messages,
api_key=api_key,
original_response=response,
)
response = response
else:
# Non-Claude models use standard Azure AI flow
api_base = AzureFoundryModelInfo.get_api_base(api_base)
# set API KEY
api_key = AzureFoundryModelInfo.get_api_key(api_key)
headers = headers or litellm.headers
if extra_headers is not None:
optional_params["extra_headers"] = extra_headers
## FOR COHERE
if "command-r" in model: # make sure tool call in messages are str
messages = stringify_json_tool_call_content(messages=messages)
## COMPLETION CALL
try:
response = base_llm_http_handler.completion(
model=model,
messages=messages,
headers=headers,
model_response=model_response,
api_key=api_key,
api_base=api_base,
acompletion=acompletion,
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
timeout=timeout,
client=client, # pass AsyncOpenAI, OpenAI client
custom_llm_provider=custom_llm_provider,
encoding=get_encoding(),
stream=stream,
)
except Exception as e:
## LOGGING - log the original exception returned
logging.post_call(
input=messages,
api_key=api_key,
original_response=str(e),
additional_args={"headers": headers},
)
raise e
if optional_params.get("stream", False):
## LOGGING
logging.post_call(
input=messages,
api_key=api_key,
original_response=response,
additional_args={"headers": headers},
)
return response

View file

@ -0,0 +1,147 @@
import time
from typing import Final
import httpx
import litellm
from litellm.exceptions import LiteLLMUnknownProvider
from litellm.llms.custom_llm import CustomLLM, custom_chat_llm_router
from litellm.native.chat_providers.support import get_encoding, print_verbose
from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult
from litellm.utils import CustomStreamWrapper
def _custom_api_first_output(resp: httpx.Response | None) -> str:
return resp.json()["data"][0]["output"][0]
def complete_custom(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
api_base: Final = ctx.api_base
headers: Final = ctx.headers
kwargs: Final = ctx.kwargs
max_tokens: Final = ctx.max_tokens
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
temperature: Final = ctx.temperature
top_p: Final = ctx.top_p
url: Final = litellm.api_base or api_base or ""
if url is None or url == "":
raise ValueError("api_base not set. Set api_base or litellm.api_base for custom endpoints")
"""
assume input to custom LLM api bases follow this format:
resp = litellm.module_level_client.post(
api_base,
json={
'model': 'meta-llama/Llama-2-13b-hf', # model name
'params': {
'prompt': ["The capital of France is P"],
'max_tokens': 32,
'temperature': 0.7,
'top_p': 1.0,
'top_k': 40,
}
}
)
"""
prompt: Final = " ".join([message["content"] for message in messages])
resp: Final = litellm.module_level_client.post(
url,
headers=headers,
json={
"model": model,
"params": {
"prompt": [prompt],
"max_tokens": max_tokens,
"temperature": temperature,
"top_p": top_p,
"top_k": kwargs.get("top_k"),
},
**kwargs.get("extra_body", {}),
},
)
"""
assume all responses from custom api_bases of this format:
{
'data': [
{
'prompt': 'The capital of France is P',
'output': ['The capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France'],
'params': {'temperature': 0.7, 'top_k': 40, 'top_p': 1}}],
'message': 'ok'
}
]
}
"""
string_response: Final = _custom_api_first_output(resp)
## RESPONSE OBJECT
model_response.choices[0].message.content = string_response
model_response.created = int(time.time())
model_response.model = model
return model_response
def complete_custom_providers(
ctx: CompletionDispatchContext,
) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
client: Final = ctx.client
custom_llm_provider: Final = ctx.custom_llm_provider
custom_prompt_dict: Final = ctx.custom_prompt_dict
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
logger_fn: Final = ctx.logger_fn
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
stream: Final = ctx.stream
timeout: Final = ctx.timeout
custom_handler: CustomLLM | None = None
for item in litellm.custom_provider_map:
if item["provider"] == custom_llm_provider:
custom_handler = item["custom_handler"]
if custom_handler is None:
raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider)
## ROUTE LLM CALL ##
handler_fn: Final = custom_chat_llm_router(async_fn=acompletion, stream=stream, custom_llm=custom_handler)
headers = headers or litellm.headers or {}
## CALL FUNCTION
response: Final = handler_fn(
model=model,
messages=messages,
headers=headers,
model_response=model_response,
print_verbose=print_verbose,
api_key=api_key,
api_base=api_base,
acompletion=acompletion,
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
timeout=timeout,
custom_prompt_dict=custom_prompt_dict,
client=client, # pass AsyncOpenAI, OpenAI client
encoding=get_encoding(),
)
if stream is True:
return CustomStreamWrapper(
completion_stream=response,
model=model,
custom_llm_provider=custom_llm_provider,
logging_obj=logging,
)
return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract

View file

@ -0,0 +1,299 @@
from typing import Final
import openai
import litellm
from litellm.constants import NADIR_DEFAULT_API_BASE
from litellm.native.chat_providers.handlers import base_llm_http_handler
from litellm.native.chat_providers.support import dispatch_client_http, get_encoding, provider_config_items
from litellm.secret_managers.main import get_secret_str
from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult
from litellm.utils import get_secret
def complete_cometapi(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
provider_config: Final = ctx.provider_config
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
api_key = api_key or litellm.cometapi_key or get_secret_str("COMETAPI_KEY") or litellm.api_key
api_base = api_base or litellm.api_base or get_secret_str("COMETAPI_API_BASE") or "https://api.cometapi.com/v1"
## COMPLETION CALL
response: Final = base_llm_http_handler.completion(
model=model,
messages=messages,
headers=headers,
model_response=model_response,
api_key=api_key,
api_base=api_base,
acompletion=acompletion,
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
timeout=timeout,
client=client,
custom_llm_provider=custom_llm_provider,
encoding=get_encoding(),
stream=stream,
provider_config=provider_config,
)
## LOGGING
logging.post_call(input=messages, api_key=api_key, original_response=response)
return response
def complete_openrouter(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = dispatch_client_http(ctx)
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
api_base = api_base or litellm.api_base or get_secret_str("OPENROUTER_API_BASE") or "https://openrouter.ai/api/v1"
api_key = (
api_key
or litellm.api_key
or litellm.openrouter_key
or get_secret_str("OPENROUTER_API_KEY")
or get_secret_str("OR_API_KEY")
)
openrouter_site_url: Final = get_secret("OR_SITE_URL") or "https://litellm.ai"
openrouter_app_name: Final = get_secret("OR_APP_NAME") or "liteLLM"
openrouter_headers: Final = {
"HTTP-Referer": openrouter_site_url,
"X-Title": openrouter_app_name,
}
_headers: Final = headers or litellm.headers
if _headers:
openrouter_headers.update(_headers)
headers = openrouter_headers
## Load Config
config: Final = litellm.OpenrouterConfig.get_config()
for k, v in provider_config_items(config):
if k == "extra_body":
# we use openai 'extra_body' to pass openrouter specific params - transforms, route, models
if "extra_body" in optional_params:
optional_params[k].update(v)
else:
optional_params[k] = v
elif k not in optional_params:
optional_params[k] = v
## COMPLETION CALL
response: Final = base_llm_http_handler.completion(
model=model,
stream=stream,
messages=messages,
acompletion=acompletion,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider="openrouter",
timeout=timeout,
headers=headers,
encoding=get_encoding(),
api_key=api_key,
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
client=client,
)
## LOGGING
logging.post_call(input=messages, api_key=openai.api_key, original_response=response)
return response
def complete_nadir(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
api_base: Final = ctx.api_base or litellm.api_base or get_secret_str("NADIR_API_BASE") or NADIR_DEFAULT_API_BASE
api_key: Final = ctx.api_key
response: Final = base_llm_http_handler.completion(
model=ctx.model,
stream=ctx.stream,
messages=ctx.messages,
acompletion=ctx.acompletion,
api_base=api_base,
model_response=ctx.model_response,
optional_params=ctx.optional_params,
litellm_params=ctx.litellm_params,
shared_session=ctx.shared_session,
custom_llm_provider="nadir",
timeout=ctx.timeout,
headers=ctx.headers or litellm.headers,
encoding=get_encoding(),
api_key=api_key,
logging_obj=ctx.logging,
client=dispatch_client_http(ctx),
)
ctx.logging.post_call(input=ctx.messages, api_key=api_key, original_response=response)
return response
def complete_vercel_ai_gateway(
ctx: CompletionDispatchContext,
) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = dispatch_client_http(ctx)
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
api_base = (
api_base
or litellm.api_base
or get_secret_str("VERCEL_AI_GATEWAY_API_BASE")
or "https://ai-gateway.vercel.sh/v1"
)
api_key = api_key or litellm.api_key or get_secret("VERCEL_AI_GATEWAY_API_KEY")
vercel_site_url: Final = get_secret("VERCEL_SITE_URL") or "https://litellm.ai"
vercel_app_name: Final = get_secret("VERCEL_APP_NAME") or "liteLLM"
vercel_headers: Final = {
"http-referer": vercel_site_url,
"x-title": vercel_app_name,
}
_headers: Final = headers or litellm.headers
if _headers:
vercel_headers.update(_headers)
headers = vercel_headers
## Load Config
config: Final = litellm.VercelAIGatewayConfig.get_config()
for k, v in provider_config_items(config):
if k == "extra_body":
# we use openai 'extra_body' to pass vercel specific params - providerOptions
if "extra_body" in optional_params:
optional_params[k].update(v)
else:
optional_params[k] = v
elif k not in optional_params:
optional_params[k] = v
## COMPLETION CALL
response: Final = base_llm_http_handler.completion(
model=model,
stream=stream,
messages=messages,
acompletion=acompletion,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider="vercel_ai_gateway",
timeout=timeout,
headers=headers,
encoding=get_encoding(),
api_key=api_key,
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
client=client,
)
## LOGGING
logging.post_call(input=messages, api_key=openai.api_key, original_response=response)
return response
def complete_edenai(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
api_base: Final = litellm.EdenAIChatConfig.get_api_base(ctx.api_base)
api_key: Final = litellm.EdenAIChatConfig.get_api_key(ctx.api_key or litellm.api_key)
response: Final = base_llm_http_handler.completion(
model=ctx.model,
messages=ctx.messages,
api_base=api_base,
custom_llm_provider="edenai",
model_response=ctx.model_response,
encoding=get_encoding(),
logging_obj=ctx.logging,
optional_params=ctx.optional_params,
timeout=ctx.timeout,
litellm_params=ctx.litellm_params,
shared_session=ctx.shared_session,
acompletion=ctx.acompletion,
stream=ctx.stream,
api_key=api_key,
headers=ctx.headers or litellm.headers,
client=dispatch_client_http(ctx),
provider_config=ctx.provider_config,
)
ctx.logging.post_call(input=ctx.messages, api_key=api_key, original_response=response)
return response
def complete_fal_ai(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
if ctx.stream:
raise litellm.FalAIError(
status_code=400,
message="fal_ai chat completions do not support streaming",
)
api_base: Final = litellm.FalAIChatConfig.get_api_base(ctx.api_base)
api_key: Final = litellm.FalAIChatConfig.get_api_key(ctx.api_key or litellm.api_key)
response: Final = base_llm_http_handler.completion(
model=ctx.model,
messages=ctx.messages,
api_base=api_base,
custom_llm_provider="fal_ai",
model_response=ctx.model_response,
encoding=get_encoding(),
logging_obj=ctx.logging,
optional_params=ctx.optional_params,
timeout=ctx.timeout,
litellm_params=ctx.litellm_params,
shared_session=ctx.shared_session,
acompletion=ctx.acompletion,
stream=ctx.stream,
api_key=api_key,
headers=ctx.headers or litellm.headers,
client=dispatch_client_http(ctx),
provider_config=ctx.provider_config,
)
ctx.logging.post_call(input=ctx.messages, api_key=api_key, original_response=response)
return response

View file

@ -0,0 +1,311 @@
from typing import Final
import litellm
from litellm.litellm_core_utils.core_helpers import safe_deep_copy
from litellm.llms.gemini.common_utils import get_api_key_from_env
from litellm.llms.vertex_ai import vertex_ai_non_gemini
from litellm.llms.vertex_ai.common_utils import VertexAIModelRoute, get_vertex_ai_model_route
from litellm.native.chat_providers.handlers import (
base_llm_http_handler,
gdc_transformation,
vertex_chat_completion,
vertex_gemma_chat_completion,
vertex_model_garden_chat_completion,
vertex_partner_models_chat_completion,
)
from litellm.native.chat_providers.support import dispatch_client_http, get_encoding, print_verbose
from litellm.secret_managers.main import get_secret_str
from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult
from litellm.utils import CustomStreamWrapper, get_secret
def complete_vertex_ai_beta(
ctx: CompletionDispatchContext,
) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key: Final = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logger_fn: Final = ctx.logger_fn
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
timeout: Final = ctx.timeout
vertex_ai_project: Final = (
optional_params.pop("vertex_project", None)
or optional_params.pop("vertex_ai_project", None)
or litellm.vertex_project
or get_secret("VERTEXAI_PROJECT")
)
vertex_ai_location: Final = (
optional_params.pop("vertex_location", None)
or optional_params.pop("vertex_ai_location", None)
or litellm.vertex_location
or get_secret("VERTEXAI_LOCATION")
)
vertex_credentials: Final = (
optional_params.pop("vertex_credentials", None)
or optional_params.pop("vertex_ai_credentials", None)
or get_secret("VERTEXAI_CREDENTIALS")
)
gemini_api_key: Final = (
api_key
or get_api_key_from_env()
or get_secret("PALM_API_KEY") # older palm api key should also work
or litellm.api_key
)
api_base = api_base or litellm.api_base or get_secret("GEMINI_API_BASE")
new_params: Final = safe_deep_copy(optional_params or {})
return vertex_chat_completion.completion(
model=model,
messages=messages,
model_response=model_response,
print_verbose=print_verbose,
optional_params=new_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
encoding=get_encoding(),
vertex_location=vertex_ai_location,
vertex_project=vertex_ai_project,
vertex_credentials=vertex_credentials,
gemini_api_key=gemini_api_key,
logging_obj=logging,
acompletion=acompletion,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
client=client,
api_base=api_base,
extra_headers=headers,
)
def complete_vertex_ai(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
custom_prompt_dict: Final = ctx.custom_prompt_dict
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logger_fn: Final = ctx.logger_fn
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response = ctx.model_response
optional_params: Final = ctx.optional_params
stream: Final = ctx.stream
timeout: Final = ctx.timeout
vertex_ai_project: Final = (
optional_params.pop("vertex_project", None)
or optional_params.pop("vertex_ai_project", None)
or litellm.vertex_project
or get_secret("VERTEXAI_PROJECT")
)
vertex_ai_location: Final = (
optional_params.pop("vertex_location", None)
or optional_params.pop("vertex_ai_location", None)
or litellm.vertex_location
or get_secret("VERTEXAI_LOCATION")
)
vertex_credentials: Final = (
optional_params.pop("vertex_credentials", None)
or optional_params.pop("vertex_ai_credentials", None)
or get_secret("VERTEXAI_CREDENTIALS")
)
api_base = api_base or litellm.api_base or get_secret("VERTEXAI_API_BASE")
new_params: Final = safe_deep_copy(optional_params or {})
model_route: Final = get_vertex_ai_model_route(model=model, litellm_params=litellm_params)
if model_route == VertexAIModelRoute.PARTNER_MODELS:
model_response = vertex_partner_models_chat_completion.completion(
model=model,
messages=messages,
model_response=model_response,
print_verbose=print_verbose,
optional_params=new_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
encoding=get_encoding(),
api_base=api_base,
vertex_location=vertex_ai_location,
vertex_project=vertex_ai_project,
vertex_credentials=vertex_credentials,
logging_obj=logging,
acompletion=acompletion,
headers=headers,
custom_prompt_dict=custom_prompt_dict,
timeout=timeout,
client=client,
)
elif model_route == VertexAIModelRoute.GEMINI:
model_response = vertex_chat_completion.completion(
model=model,
messages=messages,
model_response=model_response,
print_verbose=print_verbose,
optional_params=new_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
encoding=get_encoding(),
vertex_location=vertex_ai_location,
vertex_project=vertex_ai_project,
vertex_credentials=vertex_credentials,
gemini_api_key=None,
logging_obj=logging,
acompletion=acompletion,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
client=client,
api_base=api_base,
extra_headers=headers,
)
elif model_route == VertexAIModelRoute.GEMMA:
# Vertex Gemma Models with custom prediction endpoint
model_response = vertex_gemma_chat_completion.completion(
model=model,
messages=messages,
model_response=model_response,
print_verbose=print_verbose,
optional_params=new_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
encoding=get_encoding(),
api_base=api_base,
vertex_location=vertex_ai_location,
vertex_project=vertex_ai_project,
vertex_credentials=vertex_credentials,
logging_obj=logging,
acompletion=acompletion,
headers=headers,
custom_prompt_dict=custom_prompt_dict,
timeout=timeout,
client=client,
)
elif model_route == VertexAIModelRoute.MODEL_GARDEN:
# Vertex Model Garden - OpenAI compatible models
model_response = vertex_model_garden_chat_completion.completion(
model=model,
messages=messages,
model_response=model_response,
print_verbose=print_verbose,
optional_params=new_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
encoding=get_encoding(),
api_base=api_base,
vertex_location=vertex_ai_location,
vertex_project=vertex_ai_project,
vertex_credentials=vertex_credentials,
logging_obj=logging,
acompletion=acompletion,
headers=headers,
custom_prompt_dict=custom_prompt_dict,
timeout=timeout,
client=client,
)
elif model_route == VertexAIModelRoute.AGENT_ENGINE:
# Vertex AI Agent Engine (Reasoning Engines)
from litellm.llms.vertex_ai.agent_engine.transformation import (
VertexAgentEngineConfig,
)
vertex_agent_engine_config: Final = VertexAgentEngineConfig()
# Update litellm_params with vertex credentials
litellm_params["vertex_project"] = vertex_ai_project
litellm_params["vertex_location"] = vertex_ai_location
litellm_params["vertex_credentials"] = vertex_credentials
model_response = base_llm_http_handler.completion(
model=model,
stream=stream,
messages=messages,
model_response=model_response,
optional_params=new_params,
litellm_params=litellm_params,
encoding=get_encoding(),
api_key=None,
api_base=api_base,
logging_obj=logging,
acompletion=acompletion,
timeout=timeout,
client=client,
custom_llm_provider="vertex_ai",
provider_config=vertex_agent_engine_config,
headers=headers or {},
)
else: # VertexAIModelRoute.NON_GEMINI
model_response = vertex_ai_non_gemini.completion(
model=model,
messages=messages,
model_response=model_response,
print_verbose=print_verbose,
optional_params=new_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
encoding=get_encoding(),
vertex_location=vertex_ai_location,
vertex_project=vertex_ai_project,
vertex_credentials=vertex_credentials,
logging_obj=logging,
acompletion=acompletion,
)
if "stream" in optional_params and optional_params["stream"] is True and acompletion is False:
return CustomStreamWrapper(
model_response,
model,
custom_llm_provider="vertex_ai",
logging_obj=logging,
)
return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
def complete_gdc(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
stream: Final = ctx.stream
timeout: Final = ctx.timeout
api_key = api_key or litellm.gdc_key or get_secret_str("GDC_API_KEY") or litellm.api_key
api_base = api_base or litellm.gdc_api_base or get_secret_str("GDC_API_BASE") or litellm.api_base
return base_llm_http_handler.completion(
model=model,
messages=messages,
headers=headers,
model_response=model_response,
api_key=api_key,
api_base=api_base,
acompletion=acompletion,
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
timeout=timeout,
client=client,
custom_llm_provider=custom_llm_provider,
encoding=get_encoding(),
stream=stream,
provider_config=gdc_transformation,
)

View file

@ -0,0 +1,51 @@
from typing import Final
from litellm.llms.anthropic.chat import AnthropicChatCompletion
from litellm.llms.azure.azure import AzureChatCompletion
from litellm.llms.azure.chat.o_series_handler import AzureOpenAIO1ChatCompletion
from litellm.llms.azure.completion.handler import AzureTextCompletion
from litellm.llms.azure_ai.anthropic.handler import AzureAnthropicChatCompletion
from litellm.llms.bedrock.chat import BedrockConverseLLM
from litellm.llms.bytez.chat.transformation import BytezChatConfig
from litellm.llms.codestral.completion.handler import CodestralTextCompletion
from litellm.llms.custom_httpx.aiohttp_handler import BaseLLMAIOHTTPHandler
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.gdc.chat.transformation import GDCGeminiConfig
from litellm.llms.lemonade.chat.transformation import LemonadeChatConfig
from litellm.llms.openai.completion.handler import OpenAITextCompletion
from litellm.llms.openai.openai import OpenAIChatCompletion
from litellm.llms.openai_like.chat.handler import OpenAILikeChatHandler
from litellm.llms.ovhcloud.chat.transformation import OVHCloudChatConfig
from litellm.llms.predibase.chat.handler import PredibaseChatCompletion
from litellm.llms.sagemaker.completion.handler import SagemakerLLM
from litellm.llms.sap.chat.handler import GenAIHubOrchestration
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM
from litellm.llms.vertex_ai.vertex_ai_partner_models.main import VertexAIPartnerModels
from litellm.llms.vertex_ai.vertex_gemma_models.main import VertexAIGemmaModels
from litellm.llms.vertex_ai.vertex_model_garden.main import VertexAIModelGardenModels
from litellm.llms.watsonx.chat.handler import WatsonXChatHandler
openai_chat_completions = OpenAIChatCompletion()
openai_text_completions: Final = OpenAITextCompletion()
sap_gen_ai_hub_chat_completions: Final = GenAIHubOrchestration()
anthropic_chat_completions: Final = AnthropicChatCompletion()
azure_anthropic_chat_completions: Final = AzureAnthropicChatCompletion()
azure_chat_completions = AzureChatCompletion()
azure_o1_chat_completions: Final = AzureOpenAIO1ChatCompletion()
azure_text_completions: Final = AzureTextCompletion()
predibase_chat_completions: Final = PredibaseChatCompletion()
codestral_text_completions: Final = CodestralTextCompletion()
bedrock_converse_chat_completion: Final = BedrockConverseLLM()
vertex_chat_completion: Final = VertexLLM()
vertex_partner_models_chat_completion: Final = VertexAIPartnerModels()
vertex_gemma_chat_completion: Final = VertexAIGemmaModels()
vertex_model_garden_chat_completion: Final = VertexAIModelGardenModels()
gdc_transformation: Final = GDCGeminiConfig()
sagemaker_llm: Final = SagemakerLLM()
watsonx_chat_completion: Final = WatsonXChatHandler()
openai_like_chat_completion: Final = OpenAILikeChatHandler()
base_llm_http_handler = BaseLLMHTTPHandler()
base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler()
bytez_transformation: Final = BytezChatConfig()
ovhcloud_transformation: Final = OVHCloudChatConfig()
lemonade_transformation: Final = LemonadeChatConfig()

View file

@ -0,0 +1,926 @@
import os
from types import MappingProxyType
from typing import Final, cast # noqa: TID251 # casts moved verbatim from litellm/main.py
import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.llms.cohere.common_utils import CohereModelInfo
from litellm.llms.deprecated_providers import aleph_alpha
from litellm.llms.nlp_cloud.chat.handler import completion as nlp_cloud_chat_completion
from litellm.llms.replicate.chat.handler import completion as replicate_chat_completion
from litellm.native.chat_providers.handlers import (
base_llm_http_handler,
bytez_transformation,
codestral_text_completions,
openai_like_chat_completion,
predibase_chat_completions,
)
from litellm.native.chat_providers.support import (
dispatch_client_http,
get_encoding,
print_verbose,
provider_config_items,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult
from litellm.utils import CustomStreamWrapper, get_secret
def complete_deepseek(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
provider_config: Final = ctx.provider_config
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
try:
response: Final = base_llm_http_handler.completion(
model=model,
messages=messages,
headers=headers,
model_response=model_response,
api_key=api_key,
api_base=api_base,
acompletion=acompletion,
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
timeout=timeout,
client=client,
custom_llm_provider=custom_llm_provider,
encoding=get_encoding(),
stream=stream,
provider_config=provider_config,
)
except Exception as e:
## LOGGING - log the original exception returned
logging.post_call(
input=messages,
api_key=api_key,
original_response=str(e),
additional_args={"headers": headers},
)
raise e
return response
def complete_fireworks_ai(
ctx: CompletionDispatchContext,
) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
provider_config: Final = ctx.provider_config
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
optional_params: Final = (
provider_config.map_extra_body_params(optional_params=ctx.optional_params, model=model)
if isinstance(provider_config, litellm.FireworksAIConfig)
else ctx.optional_params
)
try:
response: Final = base_llm_http_handler.completion(
model=model,
messages=messages,
headers=headers,
model_response=model_response,
api_key=api_key,
api_base=api_base,
acompletion=acompletion,
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
timeout=timeout,
client=client,
custom_llm_provider=custom_llm_provider,
encoding=get_encoding(),
stream=stream,
provider_config=provider_config,
)
except Exception as e:
## LOGGING - log the original exception returned
logging.post_call(
input=messages,
api_key=api_key,
original_response=str(e),
additional_args={"headers": headers},
)
raise e
return response
def complete_together_ai(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
provider_config: Final = ctx.provider_config
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
try:
response: Final = base_llm_http_handler.completion(
model=model,
messages=messages,
headers=headers,
model_response=model_response,
api_key=api_key,
api_base=api_base,
acompletion=acompletion,
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
timeout=timeout,
client=client,
custom_llm_provider=custom_llm_provider,
encoding=get_encoding(),
stream=stream,
provider_config=provider_config,
)
except Exception as e:
logging.post_call(
input=messages,
api_key=api_key,
original_response=str(e),
additional_args=MappingProxyType({"headers": headers}),
)
raise
return response
def complete_scaledown(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
try:
return base_llm_http_handler.completion(
model=ctx.model,
messages=ctx.messages,
headers=ctx.headers,
model_response=ctx.model_response,
api_key=ctx.api_key,
api_base=ctx.api_base,
acompletion=ctx.acompletion,
logging_obj=ctx.logging,
optional_params=ctx.optional_params,
litellm_params=ctx.litellm_params,
shared_session=ctx.shared_session,
timeout=cast( # cast-ok: CompletionTimeout.resolve normalizes this value
float | httpx.Timeout, ctx.timeout
),
client=dispatch_client_http(ctx),
custom_llm_provider=ctx.custom_llm_provider,
encoding=get_encoding(),
stream=ctx.stream,
provider_config=ctx.provider_config,
)
except Exception as exc:
ctx.logging.post_call(
input=ctx.messages,
api_key=ctx.api_key,
original_response=str(exc),
additional_args={"headers": ctx.headers},
)
raise
def complete_xai(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
provider_config: Final = ctx.provider_config
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
try:
response: Final = base_llm_http_handler.completion(
model=model,
messages=messages,
headers=headers,
model_response=model_response,
api_key=api_key,
api_base=api_base,
acompletion=acompletion,
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
timeout=timeout,
client=client,
custom_llm_provider=custom_llm_provider,
encoding=get_encoding(),
stream=stream,
provider_config=provider_config,
)
except Exception as e:
## LOGGING - log the original exception returned
logging.post_call(
input=messages,
api_key=api_key,
original_response=str(e),
additional_args={"headers": headers},
)
raise e
return response
def complete_groq(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
api_base = (
api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there
or litellm.api_base
or get_secret("GROQ_API_BASE")
or "https://api.groq.com/openai/v1"
)
# set API KEY
api_key = (
api_key
or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there
or litellm.groq_key
or get_secret("GROQ_API_KEY")
)
headers = headers or litellm.headers
## LOAD CONFIG - if set
config: Final = litellm.GroqChatConfig.get_config()
for k, v in provider_config_items(config):
if (
k not in optional_params
): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in
optional_params[k] = v
return base_llm_http_handler.completion(
model=model,
stream=stream,
messages=messages,
acompletion=acompletion,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider=custom_llm_provider,
timeout=timeout,
headers=headers,
encoding=get_encoding(),
api_key=api_key,
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
client=client,
)
def complete_opencode_go(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
return base_llm_http_handler.completion( # pyright: ignore[reportUnknownMemberType] # shared legacy handler has untyped request mappings
model=ctx.model,
stream=ctx.stream,
messages=ctx.messages, # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # existing dispatch context carries validated messages
acompletion=ctx.acompletion,
api_base=ctx.api_base,
model_response=ctx.model_response,
optional_params=ctx.optional_params, # pyright: ignore[reportUnknownMemberType] # existing dispatch context carries the mapped provider parameters
litellm_params=ctx.litellm_params,
shared_session=ctx.shared_session,
custom_llm_provider=ctx.custom_llm_provider,
timeout=ctx.timeout, # pyright: ignore[reportArgumentType] # completion normalizes timeout before constructing this context
headers=ctx.headers, # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # legacy context retains caller headers
encoding=get_encoding(),
api_key=ctx.api_key,
logging_obj=ctx.logging,
client=dispatch_client_http(ctx),
)
def complete_minimax(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
provider_config: Final = ctx.provider_config
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
api_key = api_key or get_secret_str("MINIMAX_API_KEY") or litellm.api_key
api_base = api_base or litellm.api_base or get_secret_str("MINIMAX_API_BASE") or "https://api.minimax.io/v1"
response: Final = base_llm_http_handler.completion(
model=model,
messages=messages,
api_base=api_base,
custom_llm_provider=custom_llm_provider,
model_response=model_response,
encoding=get_encoding(),
logging_obj=logging,
optional_params=optional_params,
timeout=timeout,
litellm_params=litellm_params,
shared_session=shared_session,
acompletion=acompletion,
stream=stream,
api_key=api_key,
headers=headers,
client=client,
provider_config=provider_config,
)
logging.post_call(input=messages, api_key=api_key, original_response=response)
return response
def complete_mistral(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
provider_config: Final = ctx.provider_config
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
api_key = api_key or litellm.api_key or get_secret("MISTRAL_API_KEY")
api_base = api_base or litellm.api_base or get_secret("MISTRAL_API_BASE") or "https://api.mistral.ai/v1"
return base_llm_http_handler.completion(
model=model,
messages=messages,
api_base=api_base,
custom_llm_provider=custom_llm_provider,
model_response=model_response,
encoding=get_encoding(),
logging_obj=logging,
optional_params=optional_params,
timeout=timeout,
litellm_params=litellm_params,
shared_session=shared_session,
acompletion=acompletion,
stream=stream,
api_key=api_key,
headers=headers,
client=client,
provider_config=provider_config,
)
def complete_replicate(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key: Final = ctx.api_key
custom_prompt_dict = ctx.custom_prompt_dict
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logger_fn: Final = ctx.logger_fn
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response = ctx.model_response
optional_params: Final = ctx.optional_params
replicate_key: Final = (
api_key
or litellm.replicate_key
or litellm.api_key
or get_secret("REPLICATE_API_KEY")
or get_secret("REPLICATE_API_TOKEN")
)
api_base = api_base or litellm.api_base or get_secret("REPLICATE_API_BASE") or "https://api.replicate.com/v1"
custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict
model_response = replicate_chat_completion(
model=model,
messages=messages,
api_base=api_base,
model_response=model_response,
print_verbose=print_verbose,
optional_params=optional_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
encoding=get_encoding(), # for calculating input/output tokens
api_key=replicate_key,
logging_obj=logging,
custom_prompt_dict=custom_prompt_dict,
acompletion=acompletion,
headers=headers,
)
if optional_params.get("stream", False) is True:
## LOGGING
logging.post_call(
input=messages,
api_key=replicate_key,
original_response=model_response,
)
return model_response
def complete_nlp_cloud(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key: Final = ctx.api_key
litellm_params: Final = ctx.litellm_params
logger_fn: Final = ctx.logger_fn
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
nlp_cloud_key: Final = api_key or litellm.nlp_cloud_key or get_secret("NLP_CLOUD_API_KEY") or litellm.api_key
api_base = api_base or litellm.api_base or get_secret("NLP_CLOUD_API_BASE") or "https://api.nlpcloud.io/v1/gpu/"
response = nlp_cloud_chat_completion(
model=model,
messages=messages,
api_base=api_base,
model_response=model_response,
print_verbose=print_verbose,
optional_params=optional_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
encoding=get_encoding(),
api_key=nlp_cloud_key,
logging_obj=logging,
)
if "stream" in optional_params and optional_params["stream"] is True:
# don't try to access stream object,
response = CustomStreamWrapper(
response,
model,
custom_llm_provider="nlp_cloud",
logging_obj=logging,
)
if optional_params.get("stream", False) or acompletion is True:
## LOGGING
logging.post_call(
input=messages,
api_key=api_key,
original_response=response,
)
return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
def complete_aleph_alpha(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
api_base = ctx.api_base
api_key: Final = ctx.api_key
litellm_params: Final = ctx.litellm_params
logger_fn: Final = ctx.logger_fn
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response = ctx.model_response
optional_params: Final = ctx.optional_params
aleph_alpha_key: Final = (
api_key
or litellm.aleph_alpha_key
or get_secret("ALEPH_ALPHA_API_KEY")
or get_secret("ALEPHALPHA_API_KEY")
or litellm.api_key
)
api_base = (
api_base or litellm.api_base or get_secret("ALEPH_ALPHA_API_BASE") or "https://api.aleph-alpha.com/complete"
)
model_response = aleph_alpha.completion(
model=model,
messages=messages,
api_base=api_base,
model_response=model_response,
print_verbose=print_verbose,
optional_params=optional_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
encoding=get_encoding(),
default_max_tokens_to_sample=litellm.max_tokens,
api_key=aleph_alpha_key,
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
)
if "stream" in optional_params and optional_params["stream"] is True:
# don't try to access stream object,
return CustomStreamWrapper(
model_response,
model,
custom_llm_provider="aleph_alpha",
logging_obj=logging,
)
return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
def complete_cohere_chat(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key: Final = ctx.api_key
extra_headers: Final = ctx.extra_headers
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
provider_config: Final = ctx.provider_config
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
cohere_key: Final = (
api_key
or litellm.cohere_key
or get_secret_str("COHERE_API_KEY")
or get_secret_str("CO_API_KEY")
or litellm.api_key
)
cohere_route: Final = CohereModelInfo.get_cohere_route(model)
verbose_logger.debug("Cohere route: %s", cohere_route)
# Set API base based on route
if cohere_route == "v2":
api_base = api_base or litellm.api_base or get_secret_str("COHERE_API_BASE") or "https://api.cohere.com/v2/chat"
# Remove v2/ prefix from model name for the actual API call
if "v2/" in model:
model = model.replace("v2/", "")
else:
api_base = api_base or litellm.api_base or get_secret_str("COHERE_API_BASE") or "https://api.cohere.ai/v1/chat"
headers = headers or litellm.headers or {}
if headers is None:
headers = {}
if extra_headers is not None:
headers.update(extra_headers)
verbose_logger.debug("Model: %s, API Base: %s", model, api_base)
verbose_logger.debug("Provider Config: %s", provider_config)
return base_llm_http_handler.completion(
model=model,
stream=stream,
messages=messages,
acompletion=acompletion,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider="cohere_chat",
timeout=timeout,
headers=headers,
encoding=get_encoding(),
api_key=cohere_key,
provider_config=provider_config,
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
)
def complete_maritalk(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
api_base = ctx.api_base
api_key: Final = ctx.api_key
custom_prompt_dict: Final = ctx.custom_prompt_dict
litellm_params: Final = ctx.litellm_params
logger_fn: Final = ctx.logger_fn
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
maritalk_key: Final = api_key or litellm.maritalk_key or get_secret("MARITALK_API_KEY") or litellm.api_key
api_base = api_base or litellm.api_base or get_secret("MARITALK_API_BASE") or "https://chat.maritaca.ai/api"
return openai_like_chat_completion.completion(
model=model,
messages=messages,
api_base=api_base,
model_response=model_response,
print_verbose=print_verbose,
optional_params=optional_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
encoding=get_encoding(),
api_key=maritalk_key,
logging_obj=logging,
custom_llm_provider="maritalk",
custom_prompt_dict=custom_prompt_dict,
)
def complete_huggingface(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
stream: Final = ctx.stream
timeout: Final = ctx.timeout
huggingface_key: Final = (
api_key
or litellm.huggingface_key
or os.environ.get("HF_TOKEN")
or os.environ.get("HUGGINGFACE_API_KEY")
or litellm.api_key
)
hf_headers: Final = headers or litellm.headers
return base_llm_http_handler.completion(
model=model,
messages=messages,
headers=hf_headers,
model_response=model_response,
api_key=huggingface_key,
api_base=api_base,
acompletion=acompletion,
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
timeout=timeout,
client=client,
custom_llm_provider=custom_llm_provider,
encoding=get_encoding(),
stream=stream,
)
def complete_compactifai(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
provider_config: Final = ctx.provider_config
stream: Final = ctx.stream
timeout: Final = ctx.timeout
api_key = api_key or get_secret_str("COMPACTIFAI_API_KEY") or litellm.api_key
api_base = api_base or "https://api.compactif.ai/v1"
## COMPLETION CALL
return base_llm_http_handler.completion(
model=model,
messages=messages,
headers=headers,
model_response=model_response,
api_key=api_key,
api_base=api_base,
acompletion=acompletion,
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
timeout=timeout,
client=client,
custom_llm_provider=custom_llm_provider,
encoding=get_encoding(),
stream=stream,
provider_config=provider_config,
)
def complete_predibase(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
custom_prompt_dict: Final = ctx.custom_prompt_dict
litellm_params: Final = ctx.litellm_params
logger_fn: Final = ctx.logger_fn
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
timeout: Final = ctx.timeout
tenant_id: Final = (
optional_params.pop("tenant_id", None)
or optional_params.pop("predibase_tenant_id", None)
or litellm.predibase_tenant_id
or get_secret("PREDIBASE_TENANT_ID")
)
if tenant_id is None:
raise ValueError(
"Missing Predibase Tenant ID - Required for making the request. Set dynamically (e.g. `completion(..tenant_id=<MY-ID>)`) or in env - `PREDIBASE_TENANT_ID`."
)
api_base = (
api_base
or optional_params.pop("api_base", None)
or optional_params.pop("base_url", None)
or litellm.api_base
or get_secret("PREDIBASE_API_BASE")
)
api_key = api_key or litellm.api_key or litellm.predibase_key or get_secret("PREDIBASE_API_KEY")
_model_response: Final = predibase_chat_completions.completion(
model=model,
messages=messages,
model_response=model_response,
print_verbose=print_verbose,
optional_params=optional_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
encoding=get_encoding(),
logging_obj=logging,
acompletion=acompletion,
api_base=api_base,
custom_prompt_dict=custom_prompt_dict,
api_key=api_key,
tenant_id=tenant_id,
timeout=timeout,
)
if "stream" in optional_params and optional_params["stream"] is True and acompletion is False:
return _model_response
return _model_response
def complete_text_completion_codestral(
ctx: CompletionDispatchContext,
) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
custom_prompt_dict: Final = ctx.custom_prompt_dict
litellm_params: Final = ctx.litellm_params
logger_fn: Final = ctx.logger_fn
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
optional_params: Final = ctx.optional_params
stream: Final = ctx.stream
timeout: Final = ctx.timeout
api_base = (
api_base
or optional_params.pop("api_base", None)
or optional_params.pop("base_url", None)
or litellm.api_base
or "https://codestral.mistral.ai/v1/fim/completions"
)
api_key = api_key or litellm.api_key or get_secret("CODESTRAL_API_KEY")
text_completion_model_response: Final = litellm.TextCompletionResponse(stream=stream)
_model_response: Final = codestral_text_completions.completion(
model=model,
messages=messages,
model_response=text_completion_model_response,
print_verbose=print_verbose,
optional_params=optional_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
encoding=get_encoding(),
logging_obj=logging,
acompletion=acompletion,
api_base=api_base,
custom_prompt_dict=custom_prompt_dict,
api_key=api_key,
timeout=timeout,
)
if "stream" in optional_params and optional_params["stream"] is True and acompletion is False:
return _model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
return _model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
def complete_bytez(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
stream: Final = ctx.stream
timeout: Final = ctx.timeout
api_key = api_key or litellm.bytez_key or get_secret_str("BYTEZ_API_KEY") or litellm.api_key
response: Final = base_llm_http_handler.completion(
model=model,
messages=messages,
headers=headers,
model_response=model_response,
api_key=api_key,
api_base=api_base,
acompletion=acompletion,
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
timeout=timeout,
client=client,
custom_llm_provider=custom_llm_provider,
encoding=get_encoding(),
stream=stream,
provider_config=bytez_transformation,
)
return response

View file

@ -0,0 +1,390 @@
from typing import Final, cast # noqa: TID251 # casts moved verbatim from litellm/main.py
import openai
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.base_llm.chat.transformation import with_attribution_headers
from litellm.llms.custom_httpx.http_handler import http2_enabled
from litellm.native.chat_providers.handlers import (
base_llm_aiohttp_handler,
base_llm_http_handler,
openai_chat_completions,
openai_text_completions,
)
from litellm.native.chat_providers.support import (
dispatch_client_http,
dispatch_client_openai,
dispatch_metadata,
get_encoding,
print_verbose,
provider_config_items,
)
from litellm.responses.bridge_check import resolve_openai_api_base
from litellm.secret_managers.main import get_secret_bool, get_secret_str
from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult
from litellm.utils import get_requester_metadata, get_secret
def complete_text_completion_openai(
ctx: CompletionDispatchContext,
) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = dispatch_client_openai(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
logger_fn: Final = ctx.logger_fn
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
text_completion: Final = ctx.text_completion
timeout: Final = ctx.timeout
openai.api_type = "openai"
api_base = (
api_base
or litellm.api_base
or get_secret("OPENAI_BASE_URL")
or get_secret("OPENAI_API_BASE")
or "https://api.openai.com/v1"
)
openai.api_version = None
# set API KEY
api_key = api_key or litellm.api_key or litellm.openai_key or get_secret("OPENAI_API_KEY")
headers = headers or litellm.headers
## LOAD CONFIG - if set
config: Final = litellm.OpenAITextCompletionConfig.get_config()
for k, v in provider_config_items(config):
if (
k not in optional_params
): # completion(top_k=3) > openai_text_config(top_k=3) <- allows for dynamic variables to be passed in
optional_params[k] = v
if litellm.organization:
openai.organization = litellm.organization
## COMPLETION CALL
_response = openai_text_completions.completion(
model=model,
messages=messages,
headers=headers,
model_response=model_response,
print_verbose=print_verbose,
api_key=api_key,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
acompletion=acompletion,
client=client, # pass AsyncOpenAI, OpenAI client
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
timeout=timeout,
)
if optional_params.get("stream", False) is False and acompletion is False and text_completion is False:
# convert to chat completion response
_response = litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object(
response_object=_response, model_response_object=model_response
)
if optional_params.get("stream", False) or acompletion is True:
## LOGGING
logging.post_call(
input=messages,
api_key=api_key,
original_response=_response,
additional_args={"headers": headers},
)
return _response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
def complete_aiohttp_openai(
ctx: CompletionDispatchContext,
) -> CompletionDispatchResult:
if http2_enabled():
verbose_logger.warning(
"litellm.http2 is enabled but aiohttp_openai/ always uses aiohttp, which has no HTTP/2 client; this request stays on HTTP/1.1"
)
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
extra_headers: Final = ctx.extra_headers
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
stream: Final = ctx.stream
timeout: Final = ctx.timeout
api_base = (
api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there
or litellm.api_base
or get_secret("OPENAI_BASE_URL")
or get_secret("OPENAI_API_BASE")
or "https://api.openai.com/v1"
)
# set API KEY
api_key = (
api_key
or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there
or litellm.openai_key
or get_secret("OPENAI_API_KEY")
)
headers = headers or litellm.headers
if extra_headers is not None:
optional_params["extra_headers"] = extra_headers
return base_llm_aiohttp_handler.completion(
model=model,
messages=messages,
headers=headers,
model_response=model_response,
api_key=api_key,
api_base=api_base,
acompletion=acompletion,
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
timeout=timeout,
client=client,
custom_llm_provider=custom_llm_provider,
encoding=get_encoding(),
stream=stream,
)
def complete_custom_openai(
ctx: CompletionDispatchContext,
) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
custom_prompt_dict: Final = ctx.custom_prompt_dict
extra_headers = ctx.extra_headers
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
logger_fn: Final = ctx.logger_fn
logging: Final = ctx.logging
messages: Final = ctx.messages
metadata: Final = dispatch_metadata(ctx)
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
organization = ctx.organization
provider_config: Final = ctx.provider_config
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
# for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there
api_base = resolve_openai_api_base(api_base)
if custom_llm_provider == "bigmodel" and not api_key:
raise litellm.AuthenticationError(
message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model
)
organization = (
organization
or (None if custom_llm_provider == "bigmodel" else litellm.organization or get_secret("OPENAI_ORGANIZATION"))
or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105
)
openai.organization = organization
# set API KEY
api_key = (
api_key
or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there
or litellm.openai_key
or get_secret("OPENAI_API_KEY")
)
headers = headers or litellm.headers
outbound_headers: Final = (
headers
if provider_config is None
else with_attribution_headers(provider_config.get_attribution_headers(), headers)
)
# Add GitHub Copilot headers (same as /responses endpoint does)
if custom_llm_provider == "github_copilot":
from litellm.llms.github_copilot.authenticator import Authenticator
from litellm.llms.github_copilot.common_utils import (
get_copilot_default_headers,
pin_session_authorization,
)
from litellm.llms.github_copilot.per_user_auth import (
require_github_copilot_user_session,
)
user_session: Final = require_github_copilot_user_session(litellm_params)
copilot_headers: Final = get_copilot_default_headers(
user_session.token if user_session is not None else Authenticator().get_api_key()
)
if extra_headers:
copilot_headers.update(
cast("dict[str, str]", extra_headers) # cast-ok: extra_headers is a str-valued request dict
)
if user_session is not None:
pin_session_authorization(copilot_headers, user_session.token)
extra_headers = copilot_headers
use_base_llm_http_handler: Final = get_secret_bool("EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER")
if extra_headers is not None and not use_base_llm_http_handler:
optional_params["extra_headers"] = extra_headers
if litellm.enable_preview_features and metadata is not None: # [PREVIEW] allow metadata to be passed to OPENAI
openai_metadata: Final = get_requester_metadata(metadata)
if openai_metadata is not None:
optional_params["metadata"] = openai_metadata
## LOAD CONFIG - if set
config: Final = litellm.OpenAIConfig.get_config()
for k, v in provider_config_items(config):
if (
k not in optional_params
): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in
optional_params[k] = v
## COMPLETION CALL
try:
if use_base_llm_http_handler:
response = base_llm_http_handler.completion(
model=model,
messages=messages,
api_base=api_base,
custom_llm_provider=custom_llm_provider,
model_response=model_response,
encoding=get_encoding(),
logging_obj=logging,
optional_params=optional_params,
timeout=timeout,
litellm_params=litellm_params,
shared_session=shared_session,
acompletion=acompletion,
stream=stream,
api_key=api_key,
headers=outbound_headers,
client=client,
provider_config=provider_config,
)
else:
response = openai_chat_completions.completion(
model=model,
messages=messages,
headers=outbound_headers,
model_response=model_response,
print_verbose=print_verbose,
api_key=api_key,
api_base=api_base,
acompletion=acompletion,
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
timeout=timeout,
custom_prompt_dict=custom_prompt_dict,
client=client, # pass AsyncOpenAI, OpenAI client
organization=organization,
custom_llm_provider=custom_llm_provider,
shared_session=shared_session,
)
except Exception as e:
## LOGGING - log the original exception returned
logging.post_call(
input=messages,
api_key=api_key,
original_response=str(e),
additional_args={"headers": outbound_headers},
)
raise e
if optional_params.get("stream", False):
## LOGGING
logging.post_call(
input=messages,
api_key=api_key,
original_response=response,
additional_args={"headers": outbound_headers},
)
return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
def complete_text_completion_inception(
ctx: CompletionDispatchContext,
) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = dispatch_client_openai(ctx)
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logger_fn: Final = ctx.logger_fn
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
text_completion: Final = ctx.text_completion
timeout: Final = ctx.timeout
passed_api_base: Final = api_base or optional_params.pop("api_base", None) or optional_params.pop("base_url", None)
api_base = passed_api_base or get_secret_str("INCEPTION_API_BASE") or "https://api.inceptionlabs.ai/v1"
# FIM is served at `/v1/fim/completions`; the OpenAI client appends
# `/completions`, so point it at the `/v1/fim` base.
api_base = ensure_api_route(api_base, "fim")
# Don't forward the server-managed Inception key to a caller-supplied
# api_base; only resolve it for the default/server base, or when the
# caller passes their own key.
if passed_api_base is None or api_key:
api_key = api_key or litellm.inception_key or get_secret_str("INCEPTION_API_KEY")
_response = openai_text_completions.completion(
model=model,
messages=messages,
model_response=model_response,
print_verbose=print_verbose,
api_key=api_key,
custom_llm_provider="text-completion-inception",
api_base=api_base,
acompletion=acompletion,
client=client,
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
timeout=timeout,
)
if optional_params.get("stream", False) is False and acompletion is False and text_completion is False:
_response = litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object(
response_object=_response, model_response_object=model_response
)
if optional_params.get("stream", False) or acompletion is True:
logging.post_call(
input=messages,
api_key=api_key,
original_response=_response,
additional_args={"headers": headers},
)
return _response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract

View file

@ -0,0 +1,623 @@
from typing import Final
import litellm
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from litellm.llms.databricks.common_utils import DatabricksBase
from litellm.native.chat_providers.handlers import (
base_llm_http_handler,
ovhcloud_transformation,
sap_gen_ai_hub_chat_completions,
watsonx_chat_completion,
)
from litellm.native.chat_providers.support import (
dispatch_client_http,
get_encoding,
print_verbose,
provider_config_items,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult
from litellm.utils import get_secret
def complete_heroku(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
provider_config: Final = ctx.provider_config
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
try:
response: Final = base_llm_http_handler.completion(
model=model,
messages=messages,
headers=headers,
model_response=model_response,
api_key=api_key,
api_base=api_base,
acompletion=acompletion,
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
timeout=timeout,
client=client,
custom_llm_provider=custom_llm_provider,
encoding=get_encoding(),
stream=stream,
provider_config=provider_config,
)
except Exception as e:
logging.post_call(
input=messages,
api_key=api_key,
original_response=str(e),
additional_args={"headers": headers},
)
raise e
return response
def complete_gigachat(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
provider_config: Final = ctx.provider_config
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
api_key = (
api_key
or litellm.api_key
or litellm.gigachat_key
or get_secret("GIGACHAT_API_KEY")
or get_secret("GIGACHAT_CREDENTIALS")
)
headers = headers or litellm.headers or {}
## COMPLETION CALL
try:
response: Final = base_llm_http_handler.completion(
model=model,
messages=messages,
headers=headers,
model_response=model_response,
api_key=api_key,
api_base=api_base,
acompletion=acompletion,
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
timeout=timeout,
client=client,
custom_llm_provider=custom_llm_provider,
encoding=get_encoding(),
stream=stream,
provider_config=provider_config,
)
except Exception as e:
## LOGGING - log the original exception returned
logging.post_call(
input=messages,
api_key=api_key,
original_response=str(e),
additional_args={"headers": headers},
)
raise e
return response
def complete_sap(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
headers = headers or litellm.headers
## LOAD CONFIG - if set
config: Final = litellm.GenAIHubOrchestrationConfig.get_config()
for k, v in provider_config_items(config):
if (
k not in optional_params
): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in
optional_params[k] = v
return sap_gen_ai_hub_chat_completions.completion(
model=model,
messages=messages,
headers=headers,
model_response=model_response,
acompletion=acompletion,
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
timeout=timeout,
shared_session=shared_session,
client=client,
custom_llm_provider=custom_llm_provider,
encoding=get_encoding(),
api_key=api_key,
api_base=api_base,
stream=stream,
)
def complete_oci(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
stream: Final = ctx.stream
timeout: Final = ctx.timeout
return base_llm_http_handler.completion(
model=model,
messages=messages,
headers=headers,
model_response=model_response,
api_key=api_key,
api_base=api_base,
acompletion=acompletion,
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
timeout=timeout,
client=client,
custom_llm_provider=custom_llm_provider,
encoding=get_encoding(),
stream=stream,
)
def complete_databricks(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base: Final = DatabricksBase.resolve_api_base(ctx.api_base)
api_key: Final = DatabricksBase.resolve_api_key(ctx.api_key)
client: Final = dispatch_client_http(ctx)
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
stream: Final = ctx.stream
timeout: Final = ctx.timeout
headers = headers or litellm.headers
## COMPLETION CALL
try:
response: Final = base_llm_http_handler.completion(
model=model,
stream=stream,
messages=messages,
acompletion=acompletion,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
custom_llm_provider="databricks",
timeout=timeout,
headers=headers,
encoding=get_encoding(),
api_key=api_key,
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
client=client,
)
except Exception as e:
## LOGGING - log the original exception returned
logging.post_call(
input=messages,
api_key=api_key,
original_response=str(e),
additional_args={"headers": headers},
)
raise e
if optional_params.get("stream", False):
## LOGGING
logging.post_call(
input=messages,
api_key=api_key,
original_response=response,
additional_args={"headers": headers},
)
return response
def complete_databricks_agent(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
from litellm.llms.databricks.agent.transformation import DatabricksAgentConfig
api_base, api_key = DatabricksAgentConfig.resolve_api_base_and_key(
api_base=ctx.api_base or litellm.api_base,
api_key=ctx.api_key or litellm.api_key,
)
return base_llm_http_handler.completion(
model=ctx.model,
stream=ctx.stream,
messages=ctx.messages,
acompletion=ctx.acompletion,
api_base=api_base,
model_response=ctx.model_response,
optional_params=ctx.optional_params,
litellm_params={**ctx.litellm_params, "api_base": api_base},
shared_session=ctx.shared_session,
custom_llm_provider=ctx.custom_llm_provider,
timeout=ctx.timeout,
headers=ctx.headers or litellm.headers,
encoding=get_encoding(),
api_key=api_key,
logging_obj=ctx.logging,
client=dispatch_client_http(ctx),
)
def complete_datarobot(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
provider_config: Final = ctx.provider_config
stream: Final = ctx.stream
timeout: Final = ctx.timeout
return base_llm_http_handler.completion(
model=model,
messages=messages,
headers=headers,
model_response=model_response,
api_key=api_key,
api_base=api_base,
acompletion=acompletion,
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
timeout=timeout,
client=client,
custom_llm_provider=custom_llm_provider,
encoding=get_encoding(),
stream=stream,
provider_config=provider_config,
)
def complete_watsonx(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_prompt_dict: Final = ctx.custom_prompt_dict
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logger_fn: Final = ctx.logger_fn
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
timeout: Final = ctx.timeout
return watsonx_chat_completion.completion(
model=model,
messages=messages,
headers=headers,
model_response=model_response,
print_verbose=print_verbose,
api_key=api_key,
api_base=api_base,
acompletion=acompletion,
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
timeout=timeout,
custom_prompt_dict=custom_prompt_dict,
client=client, # pass AsyncOpenAI, OpenAI client
encoding=get_encoding(),
custom_llm_provider="watsonx",
)
def complete_watsonx_text(
ctx: CompletionDispatchContext,
) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = dispatch_client_http(ctx)
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
api_key = (
api_key
or optional_params.pop("apikey", None)
or get_secret_str("WATSONX_APIKEY")
or get_secret_str("WATSONX_API_KEY")
or get_secret_str("WX_API_KEY")
)
api_base = (
api_base
or optional_params.pop(
"url",
optional_params.pop("api_base", optional_params.pop("base_url", None)),
)
or get_secret_str("WATSONX_API_BASE")
or get_secret_str("WATSONX_URL")
or get_secret_str("WX_URL")
or get_secret_str("WML_URL")
)
wx_credentials: Final = optional_params.pop(
"wx_credentials",
optional_params.pop("watsonx_credentials", None), # follow {provider}_credentials, same as vertex ai
)
token: str | None = None
if wx_credentials is not None:
api_base = wx_credentials.get("url", api_base)
api_key = wx_credentials.get("apikey", wx_credentials.get("api_key", api_key))
token = wx_credentials.get(
"token",
wx_credentials.get(
"watsonx_token", None
), # follow format of {provider}_token, same as azure - e.g. 'azure_ad_token=..'
)
if token is not None:
optional_params["token"] = token
return base_llm_http_handler.completion(
model=model,
stream=stream,
messages=messages,
acompletion=acompletion,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider="watsonx_text",
timeout=timeout,
headers=headers,
encoding=get_encoding(),
api_key=api_key,
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
client=client,
)
def complete_cloudflare(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
custom_prompt_dict = ctx.custom_prompt_dict
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
api_key = api_key or litellm.cloudflare_api_key or litellm.api_key or get_secret("CLOUDFLARE_API_KEY")
api_base = api_base or litellm.api_base or get_secret("CLOUDFLARE_API_BASE")
custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict
return base_llm_http_handler.completion(
model=model,
stream=stream,
messages=messages,
acompletion=acompletion,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider="cloudflare",
timeout=timeout,
headers=headers,
encoding=get_encoding(),
api_key=api_key,
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
)
def complete_snowflake(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
injected_client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
client: Final = (
injected_client if injected_client is not None else (HTTPHandler(timeout=timeout) if stream is False else None)
) # Keep this here, otherwise, the httpx.client closes and streaming is impossible
try:
response: Final = base_llm_http_handler.completion(
model=model,
messages=messages,
headers=headers,
model_response=model_response,
api_key=api_key,
api_base=api_base,
acompletion=acompletion,
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
timeout=timeout,
client=client,
custom_llm_provider=custom_llm_provider,
encoding=get_encoding(),
stream=stream,
)
except Exception as e:
## LOGGING - log the original exception returned
logging.post_call(
input=messages,
api_key=api_key,
original_response=str(e),
additional_args={"headers": headers},
)
raise e
return response
def complete_gradient_ai(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key: Final = ctx.api_key
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
api_base = litellm.api_base or api_base
return base_llm_http_handler.completion(
model=model,
stream=stream,
messages=messages,
acompletion=acompletion,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider="gradient_ai",
timeout=timeout,
headers=headers,
encoding=get_encoding(),
api_key=api_key,
logging_obj=logging,
)
def complete_ovhcloud(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
stream: Final = ctx.stream
timeout: Final = ctx.timeout
api_key = api_key or litellm.ovhcloud_key or get_secret_str("OVHCLOUD_API_KEY") or litellm.api_key
api_base = (
api_base
or litellm.api_base
or get_secret_str("OVHCLOUD_API_BASE")
or "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1"
)
response: Final = base_llm_http_handler.completion(
model=model,
messages=messages,
headers=headers,
model_response=model_response,
api_key=api_key,
api_base=api_base,
acompletion=acompletion,
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
timeout=timeout,
client=client,
custom_llm_provider=custom_llm_provider,
encoding=get_encoding(),
stream=stream,
provider_config=ovhcloud_transformation,
)
return response

View file

@ -0,0 +1,329 @@
import os
from typing import Final
from pydantic import TypeAdapter
import litellm
from litellm.llms.oobabooga.chat import oobabooga
from litellm.llms.petals.completion import handler as petals_handler
from litellm.llms.vllm.completion import handler as vllm_handler
from litellm.native.chat_providers.handlers import base_llm_http_handler, lemonade_transformation
from litellm.native.chat_providers.support import dispatch_client_http, get_encoding, print_verbose
from litellm.secret_managers.main import get_secret_str
from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult
from litellm.utils import CustomStreamWrapper, get_secret
_OPAQUE_VALUE_ADAPTER: Final = TypeAdapter(object)
def complete_hosted_vllm(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key: Final = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
provider_config: Final = ctx.provider_config
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
api_base = api_base or litellm.api_base or get_secret_str("HOSTED_VLLM_API_BASE")
response: Final = base_llm_http_handler.completion(
model=model,
messages=messages,
api_base=api_base,
custom_llm_provider=custom_llm_provider,
model_response=model_response,
encoding=get_encoding(),
logging_obj=logging,
optional_params=optional_params,
timeout=timeout,
litellm_params=litellm_params,
shared_session=shared_session,
acompletion=acompletion,
stream=stream,
api_key=api_key,
headers=headers,
client=client,
provider_config=provider_config,
)
logging.post_call(input=messages, api_key=api_key, original_response=response)
return response
def complete_oobabooga(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
api_base: Final = ctx.api_base
litellm_params: Final = ctx.litellm_params
logger_fn: Final = ctx.logger_fn
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response = ctx.model_response
optional_params: Final = ctx.optional_params
model_response = oobabooga.completion(
model=model,
messages=messages,
model_response=model_response,
api_base=api_base,
print_verbose=print_verbose,
optional_params=optional_params,
litellm_params=litellm_params,
api_key=None,
logger_fn=logger_fn,
encoding=get_encoding(),
logging_obj=logging,
)
if "stream" in optional_params and optional_params["stream"] is True:
# don't try to access stream object,
return CustomStreamWrapper(
model_response,
model,
custom_llm_provider="oobabooga",
logging_obj=logging,
)
return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
def complete_vllm(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
custom_prompt_dict = ctx.custom_prompt_dict
litellm_params: Final = ctx.litellm_params
logger_fn: Final = ctx.logger_fn
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response = ctx.model_response
optional_params: Final = ctx.optional_params
custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict
model_response = vllm_handler.completion(
model=model,
messages=messages,
custom_prompt_dict=custom_prompt_dict,
model_response=model_response,
print_verbose=print_verbose,
optional_params=optional_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
encoding=get_encoding(),
logging_obj=logging,
)
if "stream" in optional_params and optional_params["stream"] is True: ## [BETA]
# don't try to access stream object,
return CustomStreamWrapper(
model_response,
model,
custom_llm_provider="vllm",
logging_obj=logging,
)
## RESPONSE OBJECT
return model_response
def complete_ollama(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key: Final = ctx.api_key
client: Final = dispatch_client_http(ctx)
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
api_base = litellm.api_base or api_base or get_secret("OLLAMA_API_BASE") or "http://localhost:11434"
if api_key is not None and "Authorization" not in headers:
headers["Authorization"] = f"Bearer {api_key}"
return base_llm_http_handler.completion(
model=model,
stream=stream,
messages=messages,
acompletion=acompletion,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider="ollama",
timeout=timeout,
headers=headers,
encoding=get_encoding(),
api_key=api_key,
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
client=client,
)
def complete_ollama_chat(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = dispatch_client_http(ctx)
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
api_base = litellm.api_base or api_base or get_secret("OLLAMA_API_BASE") or "http://localhost:11434"
api_key = api_key or litellm.ollama_key or os.environ.get("OLLAMA_API_KEY") or litellm.api_key
if api_key is not None and "Authorization" not in headers:
headers["Authorization"] = f"Bearer {api_key}"
return base_llm_http_handler.completion(
model=model,
stream=stream,
messages=messages,
acompletion=acompletion,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider="ollama_chat",
timeout=timeout,
headers=headers,
encoding=get_encoding(),
api_key=api_key,
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
client=client,
)
def complete_triton(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key: Final = ctx.api_key
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
shared_session: Final = ctx.shared_session
stream: Final = ctx.stream
timeout: Final = ctx.timeout
api_base = litellm.api_base or api_base
return base_llm_http_handler.completion(
model=model,
stream=stream,
messages=messages,
acompletion=acompletion,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider=custom_llm_provider,
timeout=timeout,
headers=headers,
encoding=get_encoding(),
api_key=api_key,
logging_obj=logging,
)
def complete_petals(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
api_base = ctx.api_base
client: Final = dispatch_client_http(ctx)
litellm_params: Final = ctx.litellm_params
logger_fn: Final = ctx.logger_fn
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response = ctx.model_response
optional_params: Final = ctx.optional_params
stream = ctx.stream
api_base = api_base or litellm.api_base
stream = optional_params.pop("stream", False)
model_response = petals_handler.completion(
model=model,
messages=messages,
api_base=api_base,
model_response=model_response,
print_verbose=print_verbose,
optional_params=optional_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
encoding=get_encoding(),
logging_obj=logging,
client=client,
)
if stream is True: ## [BETA]
# Fake streaming for petals
resp_string: Final = _OPAQUE_VALUE_ADAPTER.validate_python(model_response["choices"][0]["message"]["content"])
return CustomStreamWrapper(
resp_string,
model,
custom_llm_provider="petals",
logging_obj=logging,
)
return model_response
def complete_lemonade(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key = ctx.api_key
client: Final = dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
messages: Final = ctx.messages
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
stream: Final = ctx.stream
timeout: Final = ctx.timeout
api_key = api_key or litellm.lemonade_key or get_secret_str("LEMONADE_API_KEY") or litellm.api_key
response: Final = base_llm_http_handler.completion(
model=model,
messages=messages,
headers=headers,
model_response=model_response,
api_key=api_key,
api_base=api_base,
acompletion=acompletion,
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
timeout=timeout,
client=client,
custom_llm_provider=custom_llm_provider,
encoding=get_encoding(),
stream=stream,
provider_config=lemonade_transformation,
)
return response

View file

@ -0,0 +1,47 @@
from collections.abc import Iterable, Mapping
from functools import cache
import openai
import litellm
from litellm._lazy_imports import get_default_encoding
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.types.completion import CompletionDispatchContext
def dispatch_metadata(ctx: CompletionDispatchContext) -> Mapping[str, object] | None:
return ctx.metadata
def dispatch_client_http(ctx: CompletionDispatchContext) -> HTTPHandler | AsyncHTTPHandler | None:
return ctx.client
def dispatch_client_azure(
ctx: CompletionDispatchContext,
) -> openai.AzureOpenAI | openai.AsyncAzureOpenAI | HTTPHandler | AsyncHTTPHandler | None:
return ctx.client
def dispatch_client_openai(ctx: CompletionDispatchContext) -> openai.OpenAI | openai.AsyncOpenAI | None:
return ctx.client
def provider_config_items(config: Mapping[str, object]) -> Iterable[tuple[str, object]]:
return config.items()
def print_verbose(print_statement: object) -> None:
try:
verbose_logger.debug(print_statement)
if litellm.set_verbose:
print(print_statement) # noqa: T201
except Exception:
pass
@cache
def get_encoding() -> Tokenizer:
return get_default_encoding()

120
litellm/native/messages.py Normal file
View file

@ -0,0 +1,120 @@
"""Sends an Anthropic Messages request to a provider that speaks Messages natively."""
from collections.abc import AsyncIterator, Coroutine, Mapping
from typing import Final
import litellm
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.messages.native.utils import AnthropicMessagesRequestUtils
from litellm.llms.base_llm.anthropic_messages.transformation import BaseAnthropicMessagesConfig
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.router_utils.reasoning_effort_capability import (
RequestedReasoningEffort,
deployment_reasoning_effort_levels,
keeps_effort,
model_group_from_metadata,
resolve_deployment_reasoning_effort,
with_adaptive_thinking_as_effort,
)
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
from litellm.types.router import GenericLiteLLMParams
__all__ = ("execute_messages",)
def _with_output_config(params: Mapping[str, object], output_config: RequestedReasoningEffort) -> Mapping[str, object]:
rest: Final = {key: value for key, value in params.items() if key != "output_config"}
return rest if output_config is None else {**rest, "output_config": output_config}
def _with_deployment_output_effort(
params: Mapping[str, object],
*,
model: str,
custom_llm_provider: str,
kwargs: Mapping[str, object],
drop_params: bool,
) -> Mapping[str, object]:
deployment_model_info: Final = kwargs.get("model_info")
def resolve(requested: RequestedReasoningEffort) -> RequestedReasoningEffort:
return resolve_deployment_reasoning_effort(
requested,
model=model,
custom_llm_provider=custom_llm_provider,
deployment_model_info=deployment_model_info,
model_group=model_group_from_metadata(kwargs.get("litellm_metadata"), kwargs.get("metadata")),
drop_params=drop_params,
).value
def translate(source: Mapping[str, object]) -> Mapping[str, object]:
return with_adaptive_thinking_as_effort(
source, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params
)
requested_output: Final = as_str_mapping(params.get("output_config"))
caller_output: Final = None if requested_output is None else resolve(requested_output)
caller_params: Final = params if requested_output is None else _with_output_config(params, caller_output)
if deployment_reasoning_effort_levels(deployment_model_info) is None:
return caller_params
if requested_output is not None and keeps_effort(requested_output, caller_output):
return _with_output_config(translate(params), caller_output)
translated: Final = translate(caller_params)
budget_output: Final = as_str_mapping(translated.get("output_config"))
return translated if budget_output is None else _with_output_config(translated, resolve(budget_output))
def execute_messages(
*,
http_handler: BaseLLMHTTPHandler,
model: str,
messages: list[dict[str, object]], # mutable-ok: the HTTP handler takes list[dict] messages
provider_config: BaseAnthropicMessagesConfig,
request_params: Mapping[str, object],
custom_llm_provider: str,
litellm_params: GenericLiteLLMParams,
logging_obj: LiteLLMLoggingObj,
api_key: str | None,
api_base: str | None,
dynamic_api_base: str | None,
client: AsyncHTTPHandler | None,
is_async: bool,
stream: bool | None,
request_kwargs: dict[str, object], # mutable-ok: forwarded as the HTTP handler's kwargs dict
) -> AnthropicMessagesResponse | Coroutine[object, object, AnthropicMessagesResponse | AsyncIterator[bytes]]:
optional_params: Final = AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param(
params=request_params,
model=model,
drop_params=litellm_params.get("drop_params") is True,
custom_llm_provider=custom_llm_provider,
)
resolved_api_base: Final = (
dynamic_api_base
if dynamic_api_base is not None and provider_config.uses_get_llm_provider_api_base()
else api_base
)
return http_handler.anthropic_messages_handler(
model=model,
messages=messages,
anthropic_messages_provider_config=provider_config,
anthropic_messages_optional_request_params=dict(
_with_deployment_output_effort(
optional_params,
model=model,
custom_llm_provider=custom_llm_provider,
kwargs=request_kwargs,
drop_params=litellm.drop_params is True or litellm_params.get("drop_params") is True,
)
),
_is_async=is_async,
client=client,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
logging_obj=logging_obj,
api_key=api_key,
api_base=resolved_api_base,
stream=stream,
kwargs=request_kwargs,
)

View file

@ -0,0 +1,149 @@
from collections.abc import Callable, Mapping
from typing import Final, Protocol
from typing_extensions import assert_never
import litellm
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
from litellm.litellm_core_utils.get_litellm_params import InvalidControlOption, parse_control_options
from litellm.responses.bridge_check import keep_registered_mode
from litellm.types.litellm_params import ControlOptions
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.utils import CustomPricingLiteLLMParams
def resolve_control_options(kwargs: Mapping[str, object], model: str) -> ControlOptions:
control: Final = parse_control_options(kwargs)
match control:
case ControlOptions():
return control
case InvalidControlOption(param=param, message=message):
if litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True:
return ControlOptions()
raise litellm.BadRequestError(message=message, model=model, llm_provider=None, body={"param": param})
case _:
return assert_never(control)
class _AdmissionReservation(LiteLLMBaseModel):
input_tokens: int | None = None
class _AdmissionMetadata(LiteLLMBaseModel):
user_api_key_budget_reservation: _AdmissionReservation | None = None
def admission_input_tokens(kwargs: Mapping[str, object]) -> int | None:
reservations: Final = (
_AdmissionMetadata.model_validate(kwargs.get(key) or {}).user_api_key_budget_reservation
for key in ("litellm_metadata", "metadata")
)
return next(
(
reservation.input_tokens
for reservation in reservations
if reservation and reservation.input_tokens is not None
),
None,
)
def admitted_prompt_token_counter(prompt_tokens: int | None) -> Callable[[], int] | None:
if prompt_tokens is None:
return None
return lambda: prompt_tokens
class _ProxyAuthHeadersProvider(Protocol):
def get_auth_headers(self) -> Mapping[str, str]: ...
def proxy_auth_headers(proxy_auth: _ProxyAuthHeadersProvider) -> Mapping[str, str]:
return proxy_auth.get_auth_headers()
def _build_custom_pricing_entry(
custom_llm_provider: str,
kwargs: dict,
model_info: dict | None = None,
) -> dict:
"""Build a complete model cost entry from kwargs and model_info.
Collects all CustomPricingLiteLLMParams fields present in kwargs and
merges metadata from model_info (mode, supports_prompt_caching, max_tokens)
so that register_model() receives the full pricing configuration.
"""
entry: Final[dict] = {"litellm_provider": custom_llm_provider}
for field_name in CustomPricingLiteLLMParams.model_fields:
value = kwargs.get(field_name)
if value is not None:
entry[field_name] = value
if model_info and isinstance(model_info, dict):
for key in ("mode", "supports_prompt_caching", "max_tokens"):
if key in model_info and model_info[key] is not None:
entry.setdefault(key, model_info[key])
return entry
def _get_router_deployment_id(kwargs: dict) -> str | None:
for metadata_key in ("litellm_metadata", "metadata"):
metadata = kwargs.get(metadata_key) or {}
if not isinstance(metadata, dict):
continue
deployment_model_info = metadata.get("model_info") or {}
if not isinstance(deployment_model_info, dict):
continue
deployment_id = deployment_model_info.get("id")
if deployment_id is not None:
return str(deployment_id)
return None
def register_custom_pricing_for_request(
model: str,
custom_llm_provider: str,
kwargs: dict,
model_info: dict | None,
) -> None:
"""Register per-request custom pricing in litellm.model_cost.
Router-originated requests (identified by the deployment id the router puts
in metadata) get their full pricing registered under that unique id only;
the shared ``{provider}/{model}`` key receives the entry with pricing fields
stripped, mirroring Router._create_deployment. This keeps one deployment's
pricing overrides (e.g. a zero-cost wildcard) from clobbering built-in
pricing used by sibling deployments of the same backend model. Direct SDK
calls keep the legacy behavior of registering the shared key with pricing.
Either way the shared key's ``mode`` follows ``keep_registered_mode``, as on
deployment registration, since a call routes by its own ``model_info``.
"""
entry: Final = _build_custom_pricing_entry(
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
model_info=model_info,
)
shared_key: Final = f"{custom_llm_provider}/{model}"
deployment_id: Final = _get_router_deployment_id(kwargs)
if deployment_id is None:
litellm.register_model({shared_key: keep_registered_mode(shared_key, entry)}, persist_across_reloads=False)
return
shared_entry: Final = keep_registered_mode(
shared_key, CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry)
)
litellm.register_model(
{deployment_id: entry},
persist_across_reloads=False,
warning_display_name=shared_key,
custom_llm_provider=custom_llm_provider,
add_to_provider_model_sets=False,
)
litellm.register_model(
{shared_key: shared_entry},
persist_across_reloads=False,
warning_display_name=shared_key,
custom_llm_provider=custom_llm_provider,
add_to_provider_model_sets=False,
)

185
litellm/native/responses.py Normal file
View file

@ -0,0 +1,185 @@
"""Sends a Responses API request to a provider that speaks Responses natively."""
from collections.abc import Collection, Coroutine, Mapping, Sequence
from typing import TYPE_CHECKING, Annotated, Final
import httpx
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.openai.data_residency import infer_openai_data_residency
from litellm.responses.litellm_completion_transformation.compaction_items import (
with_compaction_items_as_summaries,
)
from litellm.responses.litellm_completion_transformation.transformation import (
with_compaction_items_as_params,
)
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import CostPerToken
if TYPE_CHECKING:
from aiohttp import ClientSession
__all__ = ("execute_responses",)
class _DeploymentModelInfo(BaseModel):
model_config = ConfigDict(frozen=True)
id: object = None
_DEPLOYMENT_MODEL_INFO: Final[TypeAdapter[_DeploymentModelInfo | object]] = TypeAdapter(
Annotated[_DeploymentModelInfo | object, Field(union_mode="left_to_right")]
)
def _is_router_deployment(deployment_model_info: object) -> bool:
"""A Router stamps the deployment's model_info (with its id) onto every call it routes."""
parsed: Final = _DEPLOYMENT_MODEL_INFO.validate_python(deployment_model_info)
return isinstance(parsed, _DeploymentModelInfo) and parsed.id is not None
_COST_PER_TOKEN: Final = TypeAdapter(CostPerToken)
def _per_call_cost_per_token(litellm_params: GenericLiteLLMParams) -> CostPerToken | None:
if litellm_params.input_cost_per_token is None or litellm_params.output_cost_per_token is None:
return None
return _COST_PER_TOKEN.validate_python(
litellm_params.model_dump(
include={
"input_cost_per_token",
"output_cost_per_token",
"cache_read_input_token_cost",
"cache_creation_input_token_cost",
},
exclude_none=True,
)
)
def execute_responses(
*,
http_handler: BaseLLMHTTPHandler,
model: str,
input: str | ResponseInputParam,
provider_config: BaseResponsesAPIConfig,
requested_params: ResponsesAPIOptionalRequestParams,
allowed_openai_params: Sequence[str] | None,
declared_reasoning_efforts: Collection[str],
deployment_model_info: object,
custom_llm_provider: str,
litellm_params: GenericLiteLLMParams,
logging_obj: LiteLLMLoggingObj,
user: str | None,
extra_headers: dict[str, object] | None,
extra_body: Mapping[str, object] | None,
timeout: float | httpx.Timeout | None,
is_async: bool,
stream: bool | None,
client: HTTPHandler | AsyncHTTPHandler | None,
shared_session: "ClientSession | None",
litellm_metadata: dict[str, object] | None,
request_kwargs: dict[str, object],
) -> (
ResponsesAPIResponse
| BaseResponsesAPIStreamingIterator
| Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator]
):
request_params: Final = ResponsesAPIRequestUtils.get_optional_params_responses_api(
model=model,
responses_api_provider_config=provider_config,
response_api_optional_params=requested_params,
allowed_openai_params=allowed_openai_params,
drop_params=normalize_drop_params(request_kwargs.get("drop_params")),
declared_reasoning_efforts=declared_reasoning_efforts,
)
logging_obj.update_from_kwargs(
kwargs=request_kwargs,
model=model,
user=user,
optional_params=dict(request_params),
litellm_params={
**request_params,
**litellm_params.custom_pricing_overrides(),
"aresponses": is_async,
"litellm_call_id": request_kwargs.get("litellm_call_id"),
"model_info": request_kwargs.get("model_info"),
"data_residency": infer_openai_data_residency(custom_llm_provider, litellm_params.api_base),
"metadata": (
request_kwargs["litellm_metadata"]
if "litellm_metadata" in request_kwargs
else request_kwargs.get("metadata")
),
"provider_affinity_header": litellm_params.provider_affinity_header,
},
custom_llm_provider=custom_llm_provider,
)
if not _is_router_deployment(deployment_model_info):
logging_obj.custom_cost_per_token = _per_call_cost_per_token(litellm_params) # rebind-ok: prices the call
logging_obj.custom_cost_per_second = litellm_params.cost_per_second # rebind-ok: prices the call
fake_stream: Final = provider_config.should_fake_stream(
model=model, stream=stream, custom_llm_provider=custom_llm_provider
)
applied_timeout: Final = CompletionTimeout.for_stream(
(StreamKind.FAKED if fake_stream else StreamKind.LIVE) if stream is True else StreamKind.NOT_STREAMED,
CompletionTimeout.as_limit(timeout),
None if stream is not True or litellm_params.stream_timeout is None else float(litellm_params.stream_timeout),
)
logging_obj.record_applied_timeout(applied_timeout)
response: Final = http_handler.response_api_handler(
model=model,
input=with_compaction_items_as_summaries(
with_compaction_items_as_params(ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(input))
),
responses_api_provider_config=provider_config,
response_api_optional_request_params=request_params,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
logging_obj=logging_obj,
extra_headers=extra_headers,
extra_body=extra_body,
timeout=applied_timeout,
_is_async=is_async,
client=client,
fake_stream=fake_stream,
litellm_metadata=litellm_metadata,
shared_session=shared_session,
)
if isinstance(response, Coroutine):
return _stamped_when_done(response, litellm_metadata, custom_llm_provider)
return _stamped(response, litellm_metadata, custom_llm_provider)
def _stamped(
response: ResponsesAPIResponse | BaseResponsesAPIStreamingIterator,
litellm_metadata: Mapping[str, object] | None,
custom_llm_provider: str,
) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator:
if not isinstance(response, ResponsesAPIResponse):
return response
stamped: Final = ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id(
responses_api_response=response,
litellm_metadata=litellm_metadata,
custom_llm_provider=custom_llm_provider,
)
stamped.hidden_params["custom_llm_provider"] = custom_llm_provider
return stamped
async def _stamped_when_done(
response: Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator],
litellm_metadata: Mapping[str, object] | None,
custom_llm_provider: str,
) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator:
return _stamped(await response, litellm_metadata, custom_llm_provider)

View file

@ -0,0 +1,165 @@
"""Which native API format a request reaches its provider in, decided once per request by the client-facing surface."""
from dataclasses import dataclass
from typing import Final, TypeAlias
from litellm.llms.base_llm.anthropic_messages.transformation import BaseAnthropicMessagesConfig
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
__all__ = (
"NativeChat",
"NativeMessages",
"NativeResponses",
"RoutePlan",
"native_messages_config",
"plan_chat_route",
"plan_messages_route",
"plan_responses_route",
)
_MESSAGES_TO_RESPONSES_PROVIDERS: Final = frozenset({"openai"})
_PROVIDER_VALUES: Final = frozenset(provider.value for provider in LlmProviders)
@dataclass(frozen=True, slots=True)
class NativeMessages:
config: BaseAnthropicMessagesConfig
@dataclass(frozen=True, slots=True)
class NativeResponses:
pass
@dataclass(frozen=True, slots=True)
class NativeChat:
pass
RoutePlan: TypeAlias = NativeMessages | NativeResponses | NativeChat
def _deployment_passes_through_anthropic_messages(model_info: object) -> bool:
"""Whether the deployment opted into forwarding /v1/messages untranslated.
The opt-in is ``model_info.supported_endpoints`` containing ``"/v1/messages"``,
declared per deployment in config.yaml and plumbed here as ``kwargs["model_info"]``
by the router.
"""
if not isinstance(model_info, dict):
return False
supported_endpoints: Final = model_info.get("supported_endpoints")
return isinstance(supported_endpoints, (list, tuple)) and "/v1/messages" in supported_endpoints
def _deployment_supports_cache_control_ttl(model_info: object) -> bool:
return isinstance(model_info, dict) and model_info.get("cache_control_ttl") is True
def native_messages_config(
model: str, custom_llm_provider: str | None, model_info: object, api_base: str | None
) -> BaseAnthropicMessagesConfig | None:
provider_config: Final = (
ProviderConfigManager.get_provider_anthropic_messages_config(
model=model,
provider=LlmProviders(custom_llm_provider),
)
if custom_llm_provider is not None and custom_llm_provider in _PROVIDER_VALUES
else None
)
native_config: Final = (
provider_config if provider_config is not None and provider_config.serves_api_base(model, api_base) else None
)
if native_config is not None or not _deployment_passes_through_anthropic_messages(model_info):
return native_config
from litellm.llms.openai_like.messages.transformation import (
OpenAILikeAnthropicMessagesConfig,
)
return OpenAILikeAnthropicMessagesConfig(cache_control_ttl=_deployment_supports_cache_control_ttl(model_info))
def _bridges_to_responses_api(model: str, custom_llm_provider: str, deployment_model_info: object) -> bool:
from litellm.responses.bridge_check import responses_api_bridge_check, str_keyed_mapping
model_info, _ = responses_api_bridge_check(
model=model,
custom_llm_provider=custom_llm_provider,
deployment_model_info=str_keyed_mapping(deployment_model_info),
)
return model_info.get("mode") == "responses"
def _responses_mode_is_lost_by_prefix_strip(
requested_model: str, resolved_model: str, custom_llm_provider: str, deployment_model_info: object
) -> bool:
"""Whether a Responses-only deployment stops looking like one once its provider prefix is stripped.
``litellm.completion`` re-derives the Responses bridge from the stripped id alone, so a
deployment id such as ``perplexity/perplexity/sonar`` (mode ``responses``) is shadowed by the
chat entry ``perplexity/sonar`` and would otherwise be sent to chat/completions.
"""
if requested_model == resolved_model:
return False
return _bridges_to_responses_api(
requested_model, custom_llm_provider, deployment_model_info
) and not _bridges_to_responses_api(resolved_model, custom_llm_provider, deployment_model_info)
def _messages_bridge_to_responses(
custom_llm_provider: str | None,
requested_model: str,
resolved_model: str,
deployment_model_info: object,
force_chat_completions: bool,
stop_requested: bool,
) -> bool:
if force_chat_completions or stop_requested:
return False
if custom_llm_provider in _MESSAGES_TO_RESPONSES_PROVIDERS:
return True
if custom_llm_provider is None:
return False
return _responses_mode_is_lost_by_prefix_strip(
requested_model, resolved_model, custom_llm_provider, deployment_model_info
)
def plan_messages_route(
*,
requested_model: str,
resolved_model: str,
custom_llm_provider: str | None,
model_info: object,
api_base: str | None,
compaction_requested: bool,
force_chat_completions: bool,
stop_requested: bool = False,
) -> RoutePlan:
"""``force_chat_completions`` is ``litellm.use_chat_completions_url_for_anthropic_messages``.
A request with ``stop_sequences`` stays on chat completions.
"""
config: Final = native_messages_config(resolved_model, custom_llm_provider, model_info, api_base)
if config is not None:
return NativeMessages(config)
if not compaction_requested and _messages_bridge_to_responses(
custom_llm_provider, requested_model, resolved_model, model_info, force_chat_completions, stop_requested
):
return NativeResponses()
return NativeChat()
def plan_responses_route(
responses_api_provider_config: BaseResponsesAPIConfig | None, use_chat_completions_api: bool
) -> NativeResponses | NativeChat:
if responses_api_provider_config is None or use_chat_completions_api:
return NativeChat()
return NativeResponses()
def plan_chat_route(responses_api_provider_config: BaseResponsesAPIConfig | None) -> NativeChat | NativeResponses:
"""For a deployment ``responses_api_bridge_check`` resolved to mode ``responses``."""
return NativeChat() if responses_api_provider_config is None else NativeResponses()

View file

@ -48,7 +48,7 @@ def s3_object_reader(bucket_name: str) -> SyncBucketObjectReader:
import boto3
from botocore.credentials import Credentials
from litellm.main import bedrock_converse_chat_completion
from litellm.native.chat_providers.handlers import bedrock_converse_chat_completion
credentials: Final[Credentials] = bedrock_converse_chat_completion.get_credentials()
s3_client: Final = boto3.client(

View file

@ -69,9 +69,9 @@ def _is_openai_backed_api_base(api_base: str) -> bool:
return hostname is not None and (hostname == _OPENAI_API_HOST or hostname.endswith(f".{_OPENAI_API_HOST}"))
def _resolve_openai_api_base(api_base: str | None) -> str:
def resolve_openai_api_base(api_base: str | None) -> str:
"""Effective OpenAI base a chat request will hit: arg > global > env > default. The bridge gate
and the ``_complete_custom_openai`` chat handler MUST resolve this identically, or a custom base
and the ``complete_custom_openai`` chat handler MUST resolve this identically, or a custom base
set via ``litellm.api_base`` or ``OPENAI_BASE_URL``/``OPENAI_API_BASE`` is invisible to the gate,
which then misreads it as the default OpenAI endpoint and bridges a request the backend can't serve."""
return (
@ -243,7 +243,7 @@ def responses_api_bridge_check(
# exactly as the chat handler does, so a custom base set via litellm.api_base or
# OPENAI_BASE_URL/OPENAI_API_BASE isn't misread as the default and bridged to a /responses route it
# lacks. A whitespace-only base collapses to the default too.
resolved_api_base: Final = _resolve_openai_api_base(api_base).strip()
resolved_api_base: Final = resolve_openai_api_base(api_base).strip()
on_foundry_openai_endpoint: Final = custom_llm_provider == "azure_ai" and azure_ai_supports_native_responses(
model, api_base
)

View file

@ -0,0 +1,259 @@
"""Hands a bridged Responses request to the native Responses executor, so the bridge never re-enters `aresponses`."""
from collections.abc import Coroutine, Mapping, Sequence
from typing import Final, NoReturn
import httpx
from aiohttp import ClientSession
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm.constants import request_timeout
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.native.responses import execute_responses
from litellm.responses.main import (
apply_managed_file_id_mapping,
base_llm_http_handler,
reasoning_for_deployment,
requested_reasoning,
resolve_model_provider_for_responses,
resolve_responses_api_provider_config,
with_provider_affinity_header,
with_provider_specific_headers,
)
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.router_utils.reasoning_effort_capability import deployment_reasoning_effort_levels
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import AgenticLoopParams
__all__ = ("anative_responses", "has_native_responses_config", "native_logging_obj", "native_responses")
_STRING_KEYED_DICT: Final = TypeAdapter(dict[str, object])
_CLIENT_HEADERS: Final = TypeAdapter(dict[str, str])
_STRINGS: Final = TypeAdapter(tuple[str, ...])
def _optional_str(value: object) -> str | None:
return value if isinstance(value, str) else None
def _string_keyed_dict(value: object) -> dict[str, object] | None:
try:
return _STRING_KEYED_DICT.validate_python(value) if value is not None else None
except ValidationError:
return None
def _client_headers(value: object) -> Mapping[str, str] | None:
try:
return _CLIENT_HEADERS.validate_python(value) if isinstance(value, dict) else None
except ValidationError:
return None
def _shared_session(value: object) -> ClientSession | None:
return value if isinstance(value, ClientSession) else None
def _http_client(value: object) -> HTTPHandler | AsyncHTTPHandler | None:
return value if isinstance(value, (HTTPHandler, AsyncHTTPHandler)) else None
def _timeout(value: object) -> float | httpx.Timeout:
return value if isinstance(value, (float, int, httpx.Timeout)) and value else request_timeout
def _allowed_openai_params(value: object) -> Sequence[str] | None:
try:
return _STRINGS.validate_python(value) if value is not None else None
except ValidationError:
return None
def _responses_target(
model: str, forwarded_kwargs: Mapping[str, object], litellm_params: GenericLiteLLMParams
) -> tuple[str, str, BaseResponsesAPIConfig] | None:
provider_model, custom_llm_provider = resolve_model_provider_for_responses(
model=model,
custom_llm_provider=_optional_str(forwarded_kwargs.get("custom_llm_provider")),
litellm_params=litellm_params,
local_vars={},
)
if custom_llm_provider is None:
return None
provider_config: Final = resolve_responses_api_provider_config(
provider_model, custom_llm_provider, forwarded_kwargs.get("model_info"), litellm_params.api_base
)
return None if provider_config is None else (provider_model, custom_llm_provider, provider_config)
def has_native_responses_config(model: str, forwarded_kwargs: Mapping[str, object]) -> bool:
return _responses_target(model, forwarded_kwargs, GenericLiteLLMParams(**forwarded_kwargs)) is not None
def _native_responses_call(
request: Mapping[str, object],
model: str,
forwarded_kwargs: Mapping[str, object],
logging_obj: LiteLLMLoggingObj,
is_async: bool,
) -> (
ResponsesAPIResponse
| BaseResponsesAPIStreamingIterator
| Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator]
):
"""Resolve the deployment and hand the translated request to the native Responses executor."""
logging_obj.record_agentic_loop_params(AgenticLoopParams(model=model))
litellm_params: Final = GenericLiteLLMParams(**forwarded_kwargs)
target: Final = _responses_target(model, forwarded_kwargs, litellm_params)
if target is None:
raise ValueError(f"No Responses API config for {model}")
provider_model, custom_llm_provider, provider_config = target
request_input, request_tools = apply_managed_file_id_mapping(
input=request["input"],
tools=request.get("tools"),
kwargs=dict(forwarded_kwargs),
local_vars={},
)
params: Final = {
**forwarded_kwargs,
**request,
"input": request_input,
"tools": request_tools,
"custom_llm_provider": custom_llm_provider,
}
deployment_model_info: Final = forwarded_kwargs.get("model_info")
reasoning: Final = reasoning_for_deployment(
requested_reasoning(params, provider_model, custom_llm_provider),
provider_model,
custom_llm_provider,
deployment_model_info,
forwarded_kwargs,
)
return execute_responses(
http_handler=base_llm_http_handler,
model=provider_model,
input=request_input,
provider_config=provider_config,
requested_params=ResponsesAPIRequestUtils.get_requested_response_api_optional_param(
{key: value for key, value in {**params, "reasoning": reasoning}.items() if key != "reasoning_effort"}
),
allowed_openai_params=_allowed_openai_params(forwarded_kwargs.get("allowed_openai_params")),
declared_reasoning_efforts=deployment_reasoning_effort_levels(deployment_model_info) or (),
deployment_model_info=deployment_model_info,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
logging_obj=logging_obj,
user=_optional_str(request.get("user")),
extra_headers=with_provider_affinity_header(
with_provider_specific_headers(
ResponsesAPIRequestUtils.merge_client_forwarded_headers(
extra_headers=_string_keyed_dict(forwarded_kwargs.get("extra_headers")),
client_headers=_client_headers(forwarded_kwargs.get("headers")),
),
litellm_params,
custom_llm_provider,
),
litellm_params,
_string_keyed_dict(request.get("metadata")),
forwarded_kwargs,
provider_model,
custom_llm_provider,
),
extra_body=_string_keyed_dict(forwarded_kwargs.get("extra_body")),
timeout=_timeout(forwarded_kwargs.get("timeout")),
is_async=is_async,
stream=request.get("stream") is True,
client=_http_client(forwarded_kwargs.get("client")),
shared_session=_shared_session(forwarded_kwargs.get("shared_session")),
litellm_metadata=_string_keyed_dict(forwarded_kwargs.get("litellm_metadata")) or {},
request_kwargs=dict(forwarded_kwargs),
)
_OPENAI_CHAT_COMPLETIONS_MODEL_PREFIX: Final = "openai/chat_completions/"
def _requests_chat_completions_api(params: Mapping[str, object]) -> bool:
model: Final = params.get("model")
return bool(params.get("use_chat_completions_api")) or (
isinstance(model, str) and model.startswith(_OPENAI_CHAT_COMPLETIONS_MODEL_PREFIX)
)
def _reads_responses_cache(params: Mapping[str, object]) -> bool:
return litellm.cache is not None or params.get("caching") is True
def native_logging_obj(
logging_obj: LiteLLMLoggingObj | None, forwarded_kwargs: Mapping[str, object], request: Mapping[str, object]
) -> LiteLLMLoggingObj | None:
"""The logging object when the native executor can serve the call, None when only `aresponses` can.
The executor skips the prompt management hooks `aresponses` runs, so a call that needs them re-enters it.
So does a deployment that asks for the chat completions API, which `aresponses` bridges to chat, and a call
that reads the response cache, which `aresponses` keys and fills.
"""
if logging_obj is None:
return None
params: Final = {**forwarded_kwargs, **request}
if _requests_chat_completions_api(params) or _reads_responses_cache(params):
return None
if logging_obj.should_run_prompt_management_hooks(
non_default_params=params, prompt_id=_optional_str(params.get("prompt_id"))
):
return None
return logging_obj
def _raise_mapped(error: Exception, model: str, forwarded_kwargs: Mapping[str, object]) -> NoReturn:
raise litellm.exception_type(
model=model,
custom_llm_provider=_optional_str(forwarded_kwargs.get("custom_llm_provider")),
original_exception=error,
extra_kwargs=dict(forwarded_kwargs),
) from error
def _priced(
response: ResponsesAPIResponse | BaseResponsesAPIStreamingIterator, logging_obj: LiteLLMLoggingObj
) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator:
"""The response, priced as `aresponses` would have priced it before the bridge translates it."""
if isinstance(response, ResponsesAPIResponse):
logging_obj.price_bridged_provider_response(response)
else:
logging_obj.price_bridged_provider_stream(response)
return response
def native_responses(
request: Mapping[str, object],
model: str,
forwarded_kwargs: Mapping[str, object],
logging_obj: LiteLLMLoggingObj,
) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator:
try:
response: Final = _native_responses_call(request, model, forwarded_kwargs, logging_obj, is_async=False)
except Exception as e: # noqa: BLE001 # every provider failure maps to the litellm exception aresponses would raise
_raise_mapped(e, model, forwarded_kwargs)
if isinstance(response, Coroutine):
raise TypeError("The sync Responses executor returned a coroutine")
return _priced(response, logging_obj)
async def anative_responses(
request: Mapping[str, object],
model: str,
forwarded_kwargs: Mapping[str, object],
logging_obj: LiteLLMLoggingObj,
) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator:
try:
response: Final = _native_responses_call(request, model, forwarded_kwargs, logging_obj, is_async=True)
settled: Final = await response if isinstance(response, Coroutine) else response
except Exception as e: # noqa: BLE001 # every provider failure maps to the litellm exception aresponses would raise
_raise_mapped(e, model, forwarded_kwargs)
return _priced(settled, logging_obj)

View file

@ -4,7 +4,7 @@ Handler for transforming responses api requests to litellm.completion requests
from collections.abc import Coroutine, Mapping, Sequence
from dataclasses import dataclass
from typing import Final
from typing import TYPE_CHECKING, Final
from pydantic import TypeAdapter
@ -30,6 +30,9 @@ from litellm.types.llms.openai import (
)
from litellm.types.utils import ModelResponse
if TYPE_CHECKING:
from litellm.native.chat_plan import PlannedChat
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(dict[str, object])
@ -55,6 +58,29 @@ def _bridged_request(
)
def _plan_bridged_chat(request: Mapping[str, object]) -> "PlannedChat | None":
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.native.chat_plan import plan_chat
logging_obj: Final = request.get("litellm_logging_obj")
return plan_chat(request, logging_obj) if isinstance(logging_obj, Logging) else None
def _price_planned_chat_response(
request: Mapping[str, object], response: ModelResponse | litellm.CustomStreamWrapper
) -> None:
"""Prices the chat response as `acompletion` would have, before it becomes a Responses object."""
from litellm.litellm_core_utils.litellm_logging import Logging
logging_obj: Final = request.get("litellm_logging_obj")
if not isinstance(logging_obj, Logging):
return
if isinstance(response, ModelResponse):
logging_obj.price_bridged_provider_response(response)
else:
logging_obj.price_bridged_provider_stream(response)
class LiteLLMCompletionTransformationHandler:
def response_api_handler(
self,
@ -116,10 +142,18 @@ class LiteLLMCompletionTransformationHandler:
**kwargs,
)
litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = litellm.completion(
**{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True},
completion_args: Final = {**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True}
from litellm.native.chat_plan import execute_planned_chat
planned_chat: Final = _plan_bridged_chat(_REQUEST_KWARGS_ADAPTER.validate_python(completion_args))
litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = (
litellm.completion(**completion_args) if planned_chat is None else execute_planned_chat(planned_chat)
)
if planned_chat is not None:
_price_planned_chat_response(completion_args, litellm_completion_response)
if isinstance(litellm_completion_response, ModelResponse):
responses_api_response: Final[ResponsesAPIResponse] = (
LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response(
@ -139,6 +173,7 @@ class LiteLLMCompletionTransformationHandler:
responses_api_request=bridged_request,
custom_llm_provider=custom_llm_provider,
litellm_metadata=kwargs.get("litellm_metadata", {}),
logs_completion=planned_chat is not None,
)
raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}")
@ -208,9 +243,20 @@ class LiteLLMCompletionTransformationHandler:
input_replaces_history=holds_compaction_summary(with_compaction_items_as_params(request_input)),
)
litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = await litellm.acompletion(
**{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True},
acompletion_args: Final = {**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True}
from litellm.native.chat_plan import aexecute_planned_chat
planned_chat: Final = _plan_bridged_chat(_REQUEST_KWARGS_ADAPTER.validate_python(acompletion_args))
litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = (
await litellm.acompletion(**acompletion_args)
if planned_chat is None
else await aexecute_planned_chat(planned_chat)
)
if planned_chat is not None:
_price_planned_chat_response(acompletion_args, litellm_completion_response)
if isinstance(litellm_completion_response, ModelResponse):
responses_api_response: Final[ResponsesAPIResponse] = (
LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response(
@ -230,5 +276,6 @@ class LiteLLMCompletionTransformationHandler:
responses_api_request=responses_api_request,
custom_llm_provider=litellm_completion_request.get("custom_llm_provider"),
litellm_metadata=kwargs.get("litellm_metadata", {}),
logs_completion=planned_chat is not None,
)
raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}")

View file

@ -2,13 +2,16 @@ import asyncio
import json
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from itertools import chain
from itertools import chain, groupby
from typing import TYPE_CHECKING, Any, Final, cast
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER
from litellm.litellm_core_utils.cold_storage_handler import ColdStorageHandler
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.responses.litellm_completion_transformation.client_tool_search import (
bridge_client_tool_search,
client_tool_search_loaded_tools,
@ -41,6 +44,7 @@ else:
# Cold Storage Handler
########################################################
COLD_STORAGE_HANDLER: Final = ColdStorageHandler()
_STORED_REQUEST: Final = TypeAdapter(dict[str, object])
########################################################
@ -181,6 +185,82 @@ def _instruction_items_before(
)
_OBJECT_LIST: Final = TypeAdapter(list[object])
def _stored_web_search_results(stored_response: object) -> Sequence[Mapping[str, object]]:
"""The Anthropic ``web_search_tool_result`` blocks a bridged response kept in its ``provider_specific_fields``."""
response: Final = as_str_mapping(stored_response)
fields: Final = None if response is None else as_str_mapping(response.get("provider_specific_fields"))
results: Final = None if fields is None else fields.get("web_search_results")
if not isinstance(results, list):
return ()
return tuple(result for result in map(as_str_mapping, _OBJECT_LIST.validate_python(results)) if result is not None)
def _server_web_search_call_id(item: object, web_search_results: Sequence[Mapping[str, object]]) -> str | None:
"""The ``server_tool_use`` id behind a stored ``web_search_call`` item, when its result block was kept."""
if not isinstance(item, Mapping) or item.get("type") != "web_search_call":
return None
item_id: Final = item.get("id")
call_id: Final = item_id.removeprefix("ws_") if isinstance(item_id, str) else None
if call_id and any(result.get("tool_use_id") == call_id for result in web_search_results):
return call_id
return None
def _server_web_search_message(items: Sequence[object], web_search_results: Sequence[Mapping[str, object]]) -> Message:
"""Replay stored web search calls as the assistant tool calls the chat completion carried,
so an Anthropic target rebuilds its ``server_tool_use`` / ``web_search_tool_result`` blocks."""
calls: Final = tuple(
(call_id, item)
for item in map(as_str_mapping, items)
if item is not None and (call_id := _server_web_search_call_id(item, web_search_results)) is not None
)
call_ids: Final = frozenset(call_id for call_id, _ in calls)
return Message(
role="assistant",
content=None,
tool_calls=[
{
"id": call_id,
"type": "function",
"function": {"name": "web_search", "arguments": json.dumps({"query": _web_search_query(item)})},
}
for call_id, item in calls
],
provider_specific_fields={
"web_search_results": [result for result in web_search_results if result.get("tool_use_id") in call_ids]
},
)
def _web_search_query(item: Mapping[str, object]) -> str:
action: Final = item.get("action")
query: Final = action.get("query") if isinstance(action, Mapping) else None
return query if isinstance(query, str) else ""
_CHAT_MESSAGE_CONTENT_ITEM_TYPES: Final = ("reasoning", "message")
def _in_chat_message_order(
output: ResponseInputParam, web_search_results: Sequence[Mapping[str, object]]
) -> ResponseInputParam:
"""A turn with server web searches in the order its chat completion message replayed it: reasoning, then text,
then the tool calls as they happened. Any other turn keeps its output order."""
if not any(_server_web_search_call_id(item, web_search_results) is not None for item in output):
return output
return sorted(
output,
key=lambda item: (
_CHAT_MESSAGE_CONTENT_ITEM_TYPES.index(item_type)
if (item_type := item.get("type")) in _CHAT_MESSAGE_CONTENT_ITEM_TYPES
else len(_CHAT_MESSAGE_CONTENT_ITEM_TYPES)
),
)
def _stored_instructions(proxy_server_request: Mapping[str, object] | None) -> str | None:
instructions: Final = None if proxy_server_request is None else proxy_server_request.get("instructions")
return instructions if isinstance(instructions, str) and instructions else None
@ -371,15 +451,23 @@ class ResponsesSessionHandler:
############################################################
_response_output: Final[object] = spend_log.get("response", "{}")
if (responses_api_output := _stored_responses_api_output(_response_output)) is not None:
chat_completion_message_history.extend(
LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
input=bridge_client_tool_search(responses_api_output, None).input,
responses_api_request={},
replay_reasoning=True,
item_cache_control=item_cache_control,
only_since_compaction=True,
web_search_results: Final = _stored_web_search_results(_response_output)
for is_server_web_search, items in groupby(
_in_chat_message_order(responses_api_output, web_search_results),
key=lambda item: _server_web_search_call_id(item, web_search_results) is not None,
):
if is_server_web_search:
chat_completion_message_history.append(_server_web_search_message(tuple(items), web_search_results))
continue
chat_completion_message_history.extend(
LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
input=bridge_client_tool_search(list(items), None).input,
responses_api_request={},
replay_reasoning=True,
item_cache_control=item_cache_control,
only_since_compaction=True,
)
)
)
elif isinstance(_response_output, dict) and _response_output and _response_output != {}:
# transform `ChatCompletion Response` to `ResponsesAPIResponse`
model_response: Final = ModelResponse(**_response_output)
@ -397,30 +485,23 @@ class ResponsesSessionHandler:
Get the parsed proxy server request from the spend log
"""
proxy_server_request: Final[str | dict] = spend_log.get("proxy_server_request") or "{}"
proxy_server_request_dict: dict[str, object] | None = None
if isinstance(proxy_server_request, dict):
proxy_server_request_dict = proxy_server_request
else:
proxy_server_request_dict = json.loads(proxy_server_request)
proxy_server_request_dict: Final = _STORED_REQUEST.validate_python(
proxy_server_request if isinstance(proxy_server_request, dict) else json.loads(proxy_server_request)
)
############################################################
# Check if user has setup cold storage for session handling
############################################################
if ResponsesSessionHandler._should_check_cold_storage_for_full_payload(proxy_server_request_dict):
# Try to get cold storage object key from spend log metadata
_proxy_server_request_dict: dict | None = None
cold_storage_object_key = ResponsesSessionHandler._get_cold_storage_object_key_from_spend_log(spend_log)
if cold_storage_object_key:
# Use the object key directly from metadata
_proxy_server_request_dict = (
await ResponsesSessionHandler.get_proxy_server_request_from_cold_storage_with_object_key(
object_key=cold_storage_object_key,
)
)
if _proxy_server_request_dict:
proxy_server_request_dict = _proxy_server_request_dict
return proxy_server_request_dict
if not ResponsesSessionHandler._should_check_cold_storage_for_full_payload(proxy_server_request_dict):
return proxy_server_request_dict
cold_storage_object_key: Final = ResponsesSessionHandler._get_cold_storage_object_key_from_spend_log(spend_log)
if not cold_storage_object_key:
return proxy_server_request_dict
return (
await ResponsesSessionHandler.get_proxy_server_request_from_cold_storage_with_object_key(
object_key=cold_storage_object_key,
)
) or proxy_server_request_dict
@staticmethod
def _get_cold_storage_object_key_from_spend_log(
@ -462,14 +543,10 @@ class ResponsesSessionHandler:
"""
verbose_proxy_logger.debug("inside get_proxy_server_request_from_cold_storage_with_object_key...")
proxy_server_request_dict: Final = (
await COLD_STORAGE_HANDLER.get_proxy_server_request_from_cold_storage_with_object_key(
object_key=object_key,
)
return await COLD_STORAGE_HANDLER.get_proxy_server_request_from_cold_storage_with_object_key(
object_key=object_key,
)
return proxy_server_request_dict
@staticmethod
def _should_check_cold_storage_for_full_payload(
proxy_server_request_dict: Mapping[str, object] | None,

View file

@ -9,6 +9,7 @@ import litellm
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items
from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params, served_from_cache
from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.main import stream_chunk_builder
from litellm.responses.litellm_completion_transformation.compaction_items import compaction_output_item
from litellm.responses.litellm_completion_transformation.custom_tools import (
@ -26,7 +27,11 @@ from litellm.responses.litellm_completion_transformation.tool_search import (
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator, stamp_responses_usage_cost
from litellm.responses.streaming_iterator import (
ResponsesAPIStreamingIterator,
dispatch_stream_success,
stamp_responses_usage_cost,
)
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.llms.openai import (
PART_UNION_TYPES,
@ -119,6 +124,10 @@ def _delta_carries_reasoning_and_answer(delta: ChatCompletionDelta) -> bool:
return has_reasoning and bool(delta.content or delta.tool_calls)
def _logged_by_this_stream(_assembled: object) -> None:
return None
class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
"""
Async iterator for processing streaming responses from the Responses API.
@ -132,9 +141,14 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
responses_api_request: ResponsesAPIOptionalRequestParams,
custom_llm_provider: str | None = None,
litellm_metadata: dict | None = None,
logs_completion: bool = False,
):
"""``logs_completion`` makes this stream, not the chat stream it wraps, log the request as a Responses row."""
self.model: str = model
self.litellm_custom_stream_wrapper: litellm.CustomStreamWrapper = litellm_custom_stream_wrapper
self._logs_completion = logs_completion
if logs_completion:
litellm_custom_stream_wrapper.hand_completion_to(_logged_by_this_stream)
self.request_input: str | ResponseInputParam = request_input
self.responses_api_request: ResponsesAPIOptionalRequestParams = responses_api_request
self.custom_llm_provider: str | None = custom_llm_provider
@ -143,7 +157,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
# repeated Pydantic attribute access in end-of-stream assembly.
self.collected_chat_completion_chunks: list[dict[str, object]] = []
self.finished: bool = False
self.litellm_logging_obj = litellm_custom_stream_wrapper.logging_obj
self.litellm_logging_obj: LiteLLMLoggingObj = litellm_custom_stream_wrapper.logging_obj
self.sent_response_created_event: bool = False
self.sent_response_in_progress_event: bool = False
self.sent_output_item_added_event: bool = False
@ -1120,6 +1134,15 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
# Latch so wrappers (FallbackResponsesStreamWrapper) + proxy
# container-ownership hook can read completed_response.
self.completed_response = response_completed_event
if self._logs_completion:
dispatch_stream_success(
self.litellm_logging_obj,
response_completed_event,
start_time=None,
end_time=None,
cache_hit=None,
is_async=not sync_mode,
)
return response_completed_event
else:
if sync_mode:
@ -1493,6 +1516,12 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
def bill_failure_after_earlier_rounds(self, earlier: BilledRoundsSpend) -> None:
self.litellm_custom_stream_wrapper.bill_failure_after_earlier_rounds(earlier)
def log_rounds_as_chat(self) -> None:
"""An MCP auto-execute bills its rounds from each chat stream's assembled response, so the chat stream logs."""
if self._logs_completion:
self._logs_completion = False
self.litellm_custom_stream_wrapper.take_completion_back()
def partial_response_on_disconnect(self) -> ResponseIncompleteEvent | None:
"""
The response.incomplete event to bill when the client went away mid-stream.

View file

@ -6,10 +6,10 @@ from contextlib import contextmanager
from dataclasses import dataclass
from functools import partial
from types import MappingProxyType
from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast, get_type_hints
from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast, get_type_hints
import httpx
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
from pydantic import BaseModel, TypeAdapter, ValidationError
from typing_extensions import (
TypeIs, # noqa: TID251 # narrows a client input item before its mark is read
assert_never,
@ -29,7 +29,6 @@ from litellm.integrations.anthropic_cache_control_hook import (
configured_injection_points,
)
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params
from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -43,6 +42,8 @@ from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfi
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.openai.common_utils import get_openai_credentials
from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig
from litellm.native.responses import execute_responses
from litellm.native.route_plan import NativeChat, plan_responses_route
from litellm.responses.litellm_completion_transformation.compaction_items import (
with_compaction_items_as_summaries,
)
@ -92,7 +93,6 @@ from litellm.types.utils import (
ADDRESSED_RESPONSE_ID_FIELD,
AgenticLoopParams,
CallTypes,
CostPerToken,
all_litellm_params,
)
from litellm.utils import (
@ -565,48 +565,6 @@ async def aresponses_api_with_mcp(
return response
class _DeploymentModelInfo(BaseModel):
model_config = ConfigDict(frozen=True)
id: object = None
_DEPLOYMENT_MODEL_INFO: Final[TypeAdapter[_DeploymentModelInfo | object]] = TypeAdapter(
Annotated[_DeploymentModelInfo | object, Field(union_mode="left_to_right")]
)
def _is_router_deployment(deployment_model_info: object) -> bool:
"""A Router stamps the deployment's model_info (with its id) onto every call it routes."""
parsed: Final = _DEPLOYMENT_MODEL_INFO.validate_python(deployment_model_info)
return isinstance(parsed, _DeploymentModelInfo) and parsed.id is not None
_COST_PER_TOKEN: Final = TypeAdapter(CostPerToken)
def _per_call_cost_per_token(litellm_params: GenericLiteLLMParams) -> CostPerToken | None:
if litellm_params.input_cost_per_token is None or litellm_params.output_cost_per_token is None:
return None
return _COST_PER_TOKEN.validate_python(
litellm_params.model_dump(
include={
"input_cost_per_token",
"output_cost_per_token",
"cache_read_input_token_cost",
"cache_creation_input_token_cost",
},
exclude_none=True,
)
)
def _bridges_to_chat_completions(
responses_api_provider_config: BaseResponsesAPIConfig | None, use_chat_completions_api: bool
) -> bool:
"""Whether the request reaches its provider as a chat completion, not a Responses call."""
return responses_api_provider_config is None or use_chat_completions_api is True
_RESPONSES_ONLY_REQUEST_FIELDS_NEVER_BRIDGED: Final = frozenset({"client_metadata"})
@ -693,7 +651,7 @@ def _encrypted_task_support_failure(
) -> _ResponsesCompatibilityFailure | None:
if (
responses_api_provider_config is None
or _bridges_to_chat_completions(responses_api_provider_config, use_chat_completions_api)
or isinstance(plan_responses_route(responses_api_provider_config, use_chat_completions_api), NativeChat)
or not responses_api_provider_config.supports_encrypted_agent_messages()
):
return "encrypted_task_unsupported"
@ -737,7 +695,7 @@ class _AsyncPromptManagementOutcome:
deployment_model_info: object
def _resolve_responses_api_provider_config(
def resolve_responses_api_provider_config(
model: str, custom_llm_provider: str, model_info: object, api_base: str | None
) -> BaseResponsesAPIConfig | None:
provider_config: Final = ProviderConfigManager.get_provider_responses_api_config(
@ -770,7 +728,7 @@ def _will_bridge_to_chat_completions(
api_base: str | None,
litellm_params: GenericLiteLLMParams | None = None,
) -> bool:
"""``_bridges_to_chat_completions`` for callers running before the provider config is resolved.
"""``plan_responses_route`` for callers running before the provider config is resolved.
Resolving the config is a pure lookup, so this asks the same question the dispatch
asks rather than restating its condition, with the model name the dispatch hands the
@ -783,14 +741,17 @@ def _will_bridge_to_chat_completions(
normalized_model: Final = _normalize_openai_chat_completions_responses_model(model)
if custom_llm_provider is None:
return True
return _bridges_to_chat_completions(
_resolve_responses_api_provider_config(
_dispatched_model_name(normalized_model[0], custom_llm_provider, api_base, litellm_params),
custom_llm_provider,
model_info,
api_base,
return isinstance(
plan_responses_route(
resolve_responses_api_provider_config(
_dispatched_model_name(normalized_model[0], custom_llm_provider, api_base, litellm_params),
custom_llm_provider,
model_info,
api_base,
),
use_chat_completions_api or normalized_model[1],
),
use_chat_completions_api or normalized_model[1],
NativeChat,
)
@ -1251,7 +1212,88 @@ def _strip_responses_routing_prefix(model: str) -> str:
return model[len(_RESPONSES_ROUTING_PREFIX) :]
def _resolve_model_provider_for_responses(
def with_provider_specific_headers(
extra_headers: dict[str, object] | None, litellm_params: GenericLiteLLMParams, custom_llm_provider: str | None
) -> dict[str, object] | None:
from litellm.litellm_core_utils.core_helpers import attribute_or_none
return ResponsesAPIRequestUtils.merge_provider_specific_headers(
extra_headers=extra_headers,
provider_specific_header=attribute_or_none(litellm_params, "provider_specific_header"),
custom_llm_provider=custom_llm_provider,
)
def with_provider_affinity_header(
extra_headers: dict[str, object] | None,
litellm_params: GenericLiteLLMParams,
metadata: Mapping[str, object] | None,
kwargs: Mapping[str, object],
model: str,
custom_llm_provider: str | None,
) -> dict[str, object] | None:
affinity_api_base: Final = (
get_openai_credentials(api_base=litellm_params.api_base).api_base
if custom_llm_provider == "openai"
else litellm_params.api_base
)
default_affinity_header: Final = get_provider_affinity_header(affinity_api_base)
if litellm_params.provider_affinity_header is None and default_affinity_header is None:
return extra_headers
try:
return add_provider_affinity_header(
headers=extra_headers or MappingProxyType({}),
litellm_params=MappingProxyType(
{
"provider_affinity_header": litellm_params.provider_affinity_header,
"litellm_session_id": kwargs.get("litellm_session_id"),
"session_id": kwargs.get("session_id"),
"metadata": metadata,
"litellm_metadata": kwargs.get("litellm_metadata"),
}
),
default_header=default_affinity_header,
)
except ValueError as affinity_error:
raise litellm.BadRequestError(
message=str(affinity_error), model=model, llm_provider=custom_llm_provider
) from affinity_error
def requested_reasoning(params: Mapping[str, object], model: str, custom_llm_provider: str | None) -> Reasoning | None:
current_reasoning: Final = cast( # cast-ok: prompt-managed reasoning arrives as a plain dict
Reasoning | None, params.get("reasoning")
)
reasoning_effort: Final = params.get("reasoning_effort")
if current_reasoning is None and reasoning_effort is not None:
return LiteLLMResponsesTransformationHandler().map_reasoning_effort(
reasoning_effort, model, custom_llm_provider
)
return current_reasoning
def reasoning_for_deployment(
reasoning: Reasoning | None,
model: str,
custom_llm_provider: str | None,
deployment_model_info: object,
kwargs: Mapping[str, object],
) -> Reasoning | None:
"""The requested reasoning with its effort resolved onto the deployment's declared reasoning_effort_levels."""
return cast( # cast-ok: the resolver hands a Reasoning back in the shape it was given
Reasoning | None,
resolve_deployment_reasoning_effort(
reasoning,
model=model,
custom_llm_provider=custom_llm_provider,
deployment_model_info=deployment_model_info,
model_group=model_group_from_metadata(kwargs.get("litellm_metadata")),
drop_params=litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True,
).value,
)
def resolve_model_provider_for_responses(
model: str,
custom_llm_provider: str | None,
litellm_params: GenericLiteLLMParams,
@ -1276,7 +1318,7 @@ def _resolve_model_provider_for_responses(
return _strip_responses_routing_prefix(provider_model), resolved_provider
def _apply_managed_file_id_mapping(
def apply_managed_file_id_mapping(
input: str | ResponseInputParam,
tools: Iterable[ToolParam] | None,
kwargs: dict[str, Any],
@ -1523,7 +1565,6 @@ def responses(
try:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
_is_async: Final = kwargs.pop("aresponses", False) is True
skip_mcp_handler: Final = kwargs.pop("_skip_mcp_handler", False)
require_encrypted_task_support: Final = kwargs.pop("_require_encrypted_task_support", False) is True
@ -1576,44 +1617,12 @@ def responses(
)
litellm_logging_obj.record_agentic_loop_params(AgenticLoopParams(model=model))
from litellm.litellm_core_utils.core_helpers import attribute_or_none
litellm_params: Final = GenericLiteLLMParams(extra_headers=extra_headers, **kwargs)
provider_headers: Final = ResponsesAPIRequestUtils.merge_provider_specific_headers(
extra_headers=extra_headers,
provider_specific_header=attribute_or_none(litellm_params, "provider_specific_header"),
custom_llm_provider=custom_llm_provider,
)
provider_headers: Final = with_provider_specific_headers(extra_headers, litellm_params, custom_llm_provider)
local_vars["extra_headers"] = provider_headers
affinity_api_base: Final = (
get_openai_credentials(api_base=litellm_params.api_base).api_base
if custom_llm_provider == "openai"
else litellm_params.api_base
effective_extra_headers: Final = with_provider_affinity_header(
provider_headers, litellm_params, metadata, kwargs, model, custom_llm_provider
)
default_affinity_header: Final = get_provider_affinity_header(affinity_api_base)
try:
effective_extra_headers: Final = (
add_provider_affinity_header(
headers=provider_headers or MappingProxyType({}),
litellm_params=MappingProxyType(
{
"provider_affinity_header": litellm_params.provider_affinity_header,
"litellm_session_id": kwargs.get("litellm_session_id"),
"session_id": kwargs.get("session_id"),
"metadata": metadata,
"litellm_metadata": kwargs.get("litellm_metadata"),
}
),
default_header=default_affinity_header,
)
if litellm_params.provider_affinity_header is not None or default_affinity_header is not None
else provider_headers
)
except ValueError as affinity_error:
raise litellm.BadRequestError(
message=str(affinity_error), model=model, llm_provider=custom_llm_provider
) from affinity_error
#########################################################
# MOCK RESPONSE LOGIC
@ -1621,7 +1630,7 @@ def responses(
if litellm_params.mock_response and isinstance(litellm_params.mock_response, str):
return mock_responses_api_response(mock_response=litellm_params.mock_response)
model, custom_llm_provider = _resolve_model_provider_for_responses(
model, custom_llm_provider = resolve_model_provider_for_responses(
model=model,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
@ -1631,7 +1640,7 @@ def responses(
#########################################################
# Update input and tools with provider-specific file IDs if managed files are used
#########################################################
input, tools = _apply_managed_file_id_mapping(input=input, tools=tools, kwargs=kwargs, local_vars=local_vars)
input, tools = apply_managed_file_id_mapping(input=input, tools=tools, kwargs=kwargs, local_vars=local_vars)
#########################################################
# Native MCP Responses API
@ -1674,7 +1683,7 @@ def responses(
if custom_llm_provider is None:
responses_api_provider_config = None
else:
responses_api_provider_config = _resolve_responses_api_provider_config(
responses_api_provider_config = resolve_responses_api_provider_config(
model, custom_llm_provider, deployment_model_info, litellm_params.api_base
)
@ -1690,27 +1699,9 @@ def responses(
_raise_responses_compatibility_failure(compatibility_failure, model, custom_llm_provider)
local_vars.update(kwargs)
current_reasoning: Final = cast( # cast-ok: prompt-managed reasoning arrives as a plain dict
Reasoning | None, local_vars.get("reasoning")
)
reasoning_effort: Final[object] = local_vars.get("reasoning_effort")
request_reasoning: Final = (
LiteLLMResponsesTransformationHandler().map_reasoning_effort(reasoning_effort, model, custom_llm_provider)
if current_reasoning is None and reasoning_effort is not None
else current_reasoning
)
routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict
deployment_reasoning: Final = cast( # cast-ok: the resolver hands a Reasoning back in the shape it was given
Reasoning | None,
resolve_deployment_reasoning_effort(
request_reasoning,
model=model,
custom_llm_provider=custom_llm_provider,
deployment_model_info=deployment_model_info,
model_group=model_group_from_metadata(routing_kwargs.get("litellm_metadata")),
drop_params=litellm.drop_params is True
or normalize_drop_params(routing_kwargs.get("drop_params")) is True,
).value,
request_reasoning: Final = requested_reasoning(local_vars, model, custom_llm_provider)
deployment_reasoning: Final = reasoning_for_deployment(
request_reasoning, model, custom_llm_provider, deployment_model_info, kwargs
)
response_api_optional_params: Final[ResponsesAPIOptionalRequestParams] = (
ResponsesAPIRequestUtils.get_requested_response_api_optional_param(
@ -1756,7 +1747,7 @@ def responses(
if _file_search_dispatch is not None:
return _file_search_dispatch
if _bridges_to_chat_completions(responses_api_provider_config, use_chat_completions_api):
if isinstance(plan_responses_route(responses_api_provider_config, use_chat_completions_api), NativeChat):
bridge_kwargs: Final = _bridge_kwargs(kwargs, responses_api_provider_config, allowed_openai_params)
if isinstance(bridge_kwargs, _UntranslatableResponsesFields):
_raise_untranslatable_responses_fields(bridge_kwargs, model, custom_llm_provider)
@ -1775,88 +1766,32 @@ def responses(
**bridge_kwargs,
)
# Get optional parameters for the responses API
request_drop_params: Final = kwargs.get("drop_params")
responses_api_request_params: Final[dict] = ResponsesAPIRequestUtils.get_optional_params_responses_api(
model=model,
responses_api_provider_config=responses_api_provider_config,
response_api_optional_params=response_api_optional_params,
allowed_openai_params=allowed_openai_params,
drop_params=normalize_drop_params(request_drop_params),
declared_reasoning_efforts=deployment_reasoning_effort_levels(deployment_model_info) or (),
)
litellm_logging_obj.update_from_kwargs(
kwargs=kwargs,
model=model,
user=user,
optional_params=dict(responses_api_request_params),
litellm_params={
**responses_api_request_params,
**litellm_params.custom_pricing_overrides(),
"aresponses": _is_async,
"litellm_call_id": litellm_call_id,
"model_info": kwargs.get("model_info"),
"data_residency": infer_openai_data_residency(custom_llm_provider, litellm_params.api_base),
"metadata": (kwargs["litellm_metadata"] if "litellm_metadata" in kwargs else kwargs.get("metadata")),
"provider_affinity_header": litellm_params.provider_affinity_header,
},
custom_llm_provider=custom_llm_provider,
)
if not _is_router_deployment(deployment_model_info):
litellm_logging_obj.custom_cost_per_token = _per_call_cost_per_token(litellm_params)
litellm_logging_obj.custom_cost_per_second = litellm_params.cost_per_second
# Decode any litellm-encoded encrypted-content item IDs back to their original IDs
input = with_compaction_items_as_summaries(
with_compaction_items_as_params(ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(input))
)
# Call the handler with _is_async flag instead of directly calling the async handler
if custom_llm_provider is None:
if custom_llm_provider is None or responses_api_provider_config is None:
raise ValueError("custom_llm_provider is required but passed as None")
fake_stream: Final = responses_api_provider_config.should_fake_stream(
model=model, stream=stream, custom_llm_provider=custom_llm_provider
)
applied_timeout: Final = CompletionTimeout.for_stream(
(StreamKind.FAKED if fake_stream else StreamKind.LIVE) if stream is True else StreamKind.NOT_STREAMED,
CompletionTimeout.as_limit(timeout or request_timeout),
None
if stream is not True or litellm_params.stream_timeout is None
else float(litellm_params.stream_timeout),
)
litellm_logging_obj.record_applied_timeout(applied_timeout)
response = base_llm_http_handler.response_api_handler(
return execute_responses(
http_handler=base_llm_http_handler,
model=model,
input=input,
responses_api_provider_config=responses_api_provider_config,
response_api_optional_request_params=responses_api_request_params,
provider_config=responses_api_provider_config,
requested_params=response_api_optional_params,
allowed_openai_params=allowed_openai_params,
declared_reasoning_efforts=deployment_reasoning_effort_levels(deployment_model_info) or (),
deployment_model_info=deployment_model_info,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
logging_obj=litellm_logging_obj,
user=user,
extra_headers=effective_extra_headers,
extra_body=extra_body,
timeout=applied_timeout,
_is_async=_is_async,
timeout=timeout or request_timeout,
is_async=_is_async,
stream=stream,
client=kwargs.get("client"),
fake_stream=fake_stream,
litellm_metadata=kwargs.get("litellm_metadata", {}),
shared_session=kwargs.get("shared_session"),
litellm_metadata=kwargs.get("litellm_metadata", {}),
request_kwargs=kwargs,
)
# Update the responses_api_response_id with the model_id
if isinstance(response, ResponsesAPIResponse):
response = ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id(
responses_api_response=response,
litellm_metadata=kwargs.get("litellm_metadata", {}),
custom_llm_provider=custom_llm_provider,
)
# Stamp custom_llm_provider so callbacks can identify the provider
# (mirrors litellm/main.py:1371 for chat completions)
response.hidden_params["custom_llm_provider"] = custom_llm_provider
return response
except Exception as e:
raise litellm.exception_type(
model=model,
@ -2694,7 +2629,7 @@ def compact_responses(
# get llm provider logic
litellm_params: Final = GenericLiteLLMParams(**kwargs)
model, custom_llm_provider = _resolve_model_provider_for_responses(
model, custom_llm_provider = resolve_model_provider_for_responses(
model=model,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,

View file

@ -475,6 +475,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator):
if not self.should_auto_execute:
return None
if isinstance(self.base_iterator, LiteLLMCompletionStreamingIterator):
self.base_iterator.log_rounds_as_chat()
return StreamedRoundBilling(self.base_iterator.litellm_logging_obj)
if isinstance(self.base_iterator, BaseResponsesAPIStreamingIterator):
return StreamedRoundBilling(self.base_iterator.logging_obj)
@ -1046,6 +1047,8 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator):
self._cached_response_id = None
self._round_index += 1
if self._round_billing is not None:
if isinstance(follow_up_response, LiteLLMCompletionStreamingIterator):
follow_up_response.log_rounds_as_chat()
self._round_billing.record_billed_rounds(follow_up_response)
except Exception as e:

View file

@ -57,6 +57,7 @@ from litellm.litellm_core_utils.llm_response_utils.get_headers import provider_r
from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
update_response_metadata,
)
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.litellm_core_utils.thread_pool_executor import executor
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
from litellm.responses.litellm_completion_transformation.transformation import (
@ -366,6 +367,50 @@ def _mid_stream_fallback_eligible(mapped_exception: Exception) -> bool:
PRE_OUTPUT_LIFECYCLE_EVENT_TYPES: Final = frozenset({"response.created", "response.in_progress", "response.queued"})
def dispatch_stream_success(
logging_obj: LiteLLMLoggingObj,
logging_response: object,
*,
start_time: datetime | None,
end_time: datetime | None,
cache_hit: bool | None,
is_async: bool,
client_detached: bool = False,
) -> None:
if not is_async:
run_async_function(
async_function=logging_obj.async_success_handler,
result=logging_response,
start_time=start_time,
end_time=end_time,
cache_hit=cache_hit,
)
executor.submit(
logging_obj.success_handler,
result=logging_response,
cache_hit=cache_hit,
start_time=start_time,
end_time=end_time,
)
return
logging_coroutine: Final = logging_obj.dispatch_success_handlers(
logging_response,
start_time=start_time,
end_time=end_time,
cache_hit=cache_hit,
prefer_async_handlers=True,
)
if client_detached:
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=logging_coroutine)
return
if getattr(logging_obj, "_on_deferred_stream_complete", None) is not None:
# End-of-stream guardrail scans write guardrail_information after
# the terminal event; dispatching now would snapshot metadata early.
logging_obj.hold_stream_completion((logging_coroutine,))
return
spawn_logging_task(asyncio.get_running_loop(), logging_coroutine, task_name="Responses stream success logging")
class BaseResponsesAPIStreamingIterator:
"""
Base class for streaming iterators that process responses from the Responses API.
@ -377,6 +422,7 @@ class BaseResponsesAPIStreamingIterator:
_started_tool_call: bool = False
_earlier_rounds_spend: BilledRoundsSpend | None = None
_replays_stored_stream: bool = False
_on_complete: Callable[[object], None] | None = None
def __init__(
self,
@ -430,6 +476,10 @@ class BaseResponsesAPIStreamingIterator:
) # GUARANTEE OPENAI HEADERS IN RESPONSE
self._raw_response_headers: Mapping[str, str] = MappingProxyType(dict(self.response.headers or {}))
def hand_completion_to(self, on_complete: Callable[[object], None]) -> None:
"""A bridge whose own client stream logs the request takes the completed response instead of the logger."""
self._on_complete = on_complete
def _check_max_streaming_duration(self) -> None:
"""Raise litellm.Timeout if the stream has exceeded LITELLM_MAX_STREAMING_DURATION_SECONDS."""
if LITELLM_MAX_STREAMING_DURATION_SECONDS is None:
@ -634,37 +684,16 @@ class BaseResponsesAPIStreamingIterator:
self._restore_provider_response_headers(logging_response)
end_time: Final = datetime.now()
if is_async:
logging_coroutine: Final = self.logging_obj.dispatch_success_handlers(
if self._on_complete is not None:
self._on_complete(logging_response)
else:
dispatch_stream_success(
self.logging_obj,
logging_response,
start_time=self.start_time,
end_time=end_time,
cache_hit=self._completed_response_cache_hit,
prefer_async_handlers=True,
)
deferred_dispatch_armed: Final = getattr(self.logging_obj, "_on_deferred_stream_complete", None) is not None
if deferred_dispatch_armed:
# End-of-stream guardrail scans write guardrail_information after
# the terminal event; dispatching now would snapshot metadata early.
self.logging_obj._deferred_stream_complete_args = (logging_coroutine,)
else:
spawn_logging_task(
asyncio.get_running_loop(), logging_coroutine, task_name="Responses stream success logging"
)
else:
run_async_function(
async_function=self.logging_obj.async_success_handler,
result=logging_response,
start_time=self.start_time,
end_time=end_time,
cache_hit=self._completed_response_cache_hit,
)
executor.submit(
self.logging_obj.success_handler,
result=logging_response,
cache_hit=self._completed_response_cache_hit,
start_time=self.start_time,
end_time=end_time,
is_async=is_async,
)
self._run_post_success_hooks(end_time=end_time)

View file

@ -49,7 +49,9 @@ class _UsageExtraFields(ExtraItemsTypedDict, total=False, extra_items=object):
pass
def _apply_nested_drop_params(params: dict[str, object], additional_drop_params: list[str] | None) -> dict[str, object]:
def _apply_nested_drop_params(
params: dict[str, object], additional_drop_params: Sequence[str] | None
) -> dict[str, object]:
nested_paths: Final = tuple(path for path in additional_drop_params or () if is_nested_path(path))
return reduce(lambda acc, path: delete_nested_value(acc, path), nested_paths, params)
@ -224,7 +226,7 @@ class ResponsesAPIRequestUtils:
@staticmethod
def merge_client_forwarded_headers(
extra_headers: dict[str, object] | None,
client_headers: dict[str, str] | None,
client_headers: Mapping[str, str] | None,
) -> dict[str, object] | None:
"""
Merge headers forwarded by the proxy (`headers` kwarg, set when
@ -259,8 +261,8 @@ class ResponsesAPIRequestUtils:
@staticmethod
def _check_valid_arg(
supported_params: list[str] | None,
non_default_params: dict,
supported_params: Sequence[str] | None,
non_default_params: Mapping[str, object],
drop_params: bool | None,
custom_llm_provider: str | None,
model: str,
@ -285,7 +287,7 @@ class ResponsesAPIRequestUtils:
model: str,
responses_api_provider_config: BaseResponsesAPIConfig,
response_api_optional_params: ResponsesAPIOptionalRequestParams,
allowed_openai_params: list[str] | None = None,
allowed_openai_params: Sequence[str] | None = None,
drop_params: bool | None = None,
declared_reasoning_efforts: Collection[str] | None = None,
) -> dict:
@ -311,7 +313,7 @@ class ResponsesAPIRequestUtils:
non_default_params: Final = cast(dict, response_api_optional_params)
# Check for unsupported parameters
ResponsesAPIRequestUtils._check_valid_arg(
supported_params=supported_params + (allowed_openai_params or []),
supported_params=[*supported_params, *(allowed_openai_params or ())],
non_default_params=non_default_params,
drop_params=should_drop_params,
custom_llm_provider=responses_api_provider_config.custom_llm_provider,
@ -340,7 +342,7 @@ class ResponsesAPIRequestUtils:
return apply_openai_param_overrides(
optional_params=params_with_normalized_stream_options,
non_default_params=non_default_params,
allowed_openai_params=allowed_openai_params or [],
allowed_openai_params=list(allowed_openai_params or ()),
)
@staticmethod
@ -1311,7 +1313,7 @@ class ResponseAPILoggingUtils:
output_tokens: Final = usage_input.get("output_tokens")
if isinstance(input_tokens, int) and isinstance(output_tokens, int):
usage_input["total_tokens"] = input_tokens + output_tokens
response_api_usage = ResponseAPIUsage(**usage_input)
response_api_usage = ResponseAPIUsage.model_validate(usage_input)
else:
response_api_usage = usage_input
prompt_tokens: Final[int] = response_api_usage.input_tokens or 0

View file

@ -245,7 +245,7 @@ class CompletionDispatchContext:
_CompletionDispatchContext = CompletionDispatchContext
_CompletionDispatchResult = Union[
CompletionDispatchResult = Union[
Coroutine[object, object, Union["ModelResponse", "CustomStreamWrapper"]],
"ModelResponse",
"CustomStreamWrapper",

View file

@ -9251,11 +9251,11 @@ def validate_and_fix_openai_messages(messages: list):
return validate_chat_completion_user_messages(messages=new_messages)
def validate_and_fix_openai_tools(tools: list | None) -> list[dict] | None:
def validate_and_fix_openai_tools(tools: list | None) -> list[dict[str, object]] | None:
"""
Ensure tools is List[dict] and not List[BaseModel]
"""
new_tools: Final = []
new_tools: Final[list[dict[str, object]]] = []
if tools is None:
return tools
for tool in tools:
@ -9343,9 +9343,9 @@ def validate_chat_completion_user_messages(messages: list[AllMessageValues]):
def validate_chat_completion_tool_choice(
tool_choice: dict | str | None,
tool_choice: dict[str, object] | str | None,
model: str = "",
) -> dict | str | None:
) -> dict[str, object] | str | None:
"""
Confirm the tool choice is passed in the OpenAI format.
@ -9376,7 +9376,7 @@ def validate_chat_completion_tool_choice(
)
def validate_openai_optional_params(stop: str | list[str] | None = None, **kwargs) -> str | list[str] | None:
def validate_openai_optional_params(stop: str | list[str] | None = None, **kwargs: object) -> str | list[str] | None:
"""
Validates and fixes OpenAI optional parameters.
@ -11426,8 +11426,8 @@ def reasoning_effort_with_summary_alias(
def strip_reasoning_summary_aliases_from_optional_params(
optional_params: dict,
) -> tuple[dict, object | None]:
optional_params: Mapping[str, object],
) -> tuple[dict[str, object], object | None]:
"""Copy optional_params; remove reasoningSummary aliases from top-level and extra_body."""
op: Final = dict(optional_params)
rs_val = op.pop("reasoningSummary", None)

View file

@ -0,0 +1,58 @@
import ast
import sys
from collections.abc import Iterator
from pathlib import Path
from typing import Final
PACKAGE: Final = Path("litellm/native")
FORBIDDEN: Final = "litellm.main"
def _is_forbidden(module: str) -> bool:
return module == FORBIDDEN or module.startswith(f"{FORBIDDEN}.")
def _from_import_targets(package: tuple[str, ...], node: ast.ImportFrom) -> Iterator[str]:
base: Final = package[: len(package) - node.level + 1] if node.level else ()
module: Final = ".".join((*base, node.module) if node.module else base)
yield module
yield from (f"{module}.{alias.name}" for alias in node.names)
def _imported(package: tuple[str, ...], node: ast.AST) -> Iterator[str]:
match node:
case ast.Import(names=names):
yield from (alias.name for alias in names)
case ast.ImportFrom():
yield from _from_import_targets(package, node)
case ast.Attribute(value=ast.Name(id="litellm"), attr=attr):
yield f"litellm.{attr}"
case _:
return
def violations(path: Path, source: str) -> tuple[str, ...]:
package: Final = (
("litellm", "native", *path.relative_to(PACKAGE).parent.parts) if path.is_relative_to(PACKAGE) else ()
)
tree: Final = ast.parse(source, filename=str(path))
return tuple(
f"{path}:{node.lineno}: litellm.native must not depend on litellm.main"
for node in ast.walk(tree)
if any(_is_forbidden(target) for target in _imported(package, node))
)
def main() -> int:
findings: Final = tuple(
finding for path in sorted(PACKAGE.rglob("*.py")) for finding in violations(path, path.read_text())
)
if findings:
sys.stderr.write("\n".join(findings) + "\n")
return 1
sys.stdout.write("litellm.native imports: passed\n")
return 0
if __name__ == "__main__":
sys.exit(main())

View file

@ -107,6 +107,9 @@ litellm_py_files=$(scope_match "$litellm_py_pattern")
if [ -n "$(scope_match '^(litellm/proxy/mcp_server/|scripts/check_mcp_operation_boundary\.py)')" ]; then
uv run --no-sync python scripts/check_mcp_operation_boundary.py || exit 1
fi
if [ -n "$(scope_match '^(litellm/native/|scripts/check_native_imports\.py)')" ]; then
uv run --no-sync python scripts/check_native_imports.py || exit 1
fi
e2e_py_files=$(scope_match "$e2e_py_pattern")
test_tree_files=$(scope_match "$test_tree_pattern")
# ruff format (and CI's format step) skip enterprise; the rest of make lint covers it.

View file

@ -1,15 +1,26 @@
import asyncio
import json
import math
from collections.abc import Mapping, Sequence
from datetime import datetime
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
import respx
from openai.types.chat import ChatCompletion
from pydantic import TypeAdapter
import litellm
from litellm.caching.caching import Cache
from litellm.completion_extras.litellm_responses_transformation.handler import (
ResponsesToCompletionBridgeHandler,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.responses import bridge_executor
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import ModelResponse
@ -41,6 +52,11 @@ def test_is_preformatted_cached_chat_stream_false_wrong_type():
)
@pytest.fixture
def _cache_on(monkeypatch):
monkeypatch.setattr(litellm, "cache", MagicMock(spec=Cache))
def _bridge_kwargs(stream: bool):
logging_obj = LiteLLMLogging(
litellm_call_id="test-call",
@ -63,7 +79,7 @@ def _bridge_kwargs(stream: bool):
}
def test_completion_returns_cached_model_response_directly():
def test_completion_returns_cached_model_response_directly(_cache_on):
"""Non-streaming bridge cache hit: responses() returns a ModelResponse -> bridge returns it as-is."""
cached = ModelResponse(id="chatcmpl-cached-nonstream", model="gpt-5.4")
bridge = ResponsesToCompletionBridgeHandler()
@ -82,7 +98,7 @@ def test_completion_returns_cached_model_response_directly():
@pytest.mark.asyncio
async def test_acompletion_returns_cached_model_response_directly():
async def test_acompletion_returns_cached_model_response_directly(_cache_on):
cached = ModelResponse(id="chatcmpl-cached-nonstream-async", model="gpt-5.4")
bridge = ResponsesToCompletionBridgeHandler()
@ -99,7 +115,7 @@ async def test_acompletion_returns_cached_model_response_directly():
assert result is cached
def test_completion_skips_rewrapping_preformatted_cached_chat_stream():
def test_completion_skips_rewrapping_preformatted_cached_chat_stream(_cache_on):
"""Streaming bridge cache hit returning CustomStreamWrapper(cached_response) -> bridge skips re-wrapping."""
stream = MagicMock(spec=CustomStreamWrapper)
stream.custom_llm_provider = "cached_response"
@ -124,7 +140,7 @@ def test_completion_skips_rewrapping_preformatted_cached_chat_stream():
assert result is stream
def test_completion_preserves_top_level_stream_flag_in_responses_request():
def test_completion_preserves_top_level_stream_flag_in_responses_request(_cache_on):
stream = MagicMock(spec=CustomStreamWrapper)
stream.custom_llm_provider = "cached_response"
bridge = ResponsesToCompletionBridgeHandler()
@ -152,7 +168,7 @@ def test_completion_preserves_top_level_stream_flag_in_responses_request():
@pytest.mark.asyncio
async def test_acompletion_skips_rewrapping_preformatted_cached_chat_stream():
async def test_acompletion_skips_rewrapping_preformatted_cached_chat_stream(_cache_on):
stream = MagicMock(spec=CustomStreamWrapper)
stream.custom_llm_provider = "cached_response"
bridge = ResponsesToCompletionBridgeHandler()
@ -177,7 +193,7 @@ async def test_acompletion_skips_rewrapping_preformatted_cached_chat_stream():
@pytest.mark.asyncio
async def test_acompletion_preserves_top_level_stream_flag_in_responses_request():
async def test_acompletion_preserves_top_level_stream_flag_in_responses_request(_cache_on):
stream = MagicMock(spec=CustomStreamWrapper)
stream.custom_llm_provider = "cached_response"
bridge = ResponsesToCompletionBridgeHandler()
@ -219,7 +235,7 @@ def _completed_chat_response() -> ModelResponse:
@pytest.mark.asyncio
async def test_acompletion_streams_completed_model_response():
async def test_acompletion_streams_completed_model_response(_cache_on):
"""A streaming request whose bridge call comes back already completed must still be
handed back as an async-iterable stream. Returning the bare ModelResponse crashed the
proxy's SSE generator with "'async for' requires an object with __aiter__ method".
@ -245,7 +261,7 @@ async def test_acompletion_streams_completed_model_response():
assert [c for c in chunks if c.choices[0].finish_reason], "stream never emitted a finish_reason"
def test_completion_streams_completed_model_response():
def test_completion_streams_completed_model_response(_cache_on):
completed = _completed_chat_response()
bridge = ResponsesToCompletionBridgeHandler()
@ -288,7 +304,7 @@ def _upstream_model_for(handed_model: str, custom_llm_provider: str) -> str:
def test_completion_keeps_provider_native_model_id_through_responses(
custom_llm_provider, bridge_model, expected_upstream_model
):
"""responses() resolves the provider itself, so the bridge must not hand it an already-stripped model."""
"""The Responses executor resolves the provider itself, so the bridge must not hand it an already-stripped model."""
cached = ModelResponse(id="chatcmpl-cached", model=bridge_model)
bridge = ResponsesToCompletionBridgeHandler()
kwargs = _bridge_kwargs(stream=False)
@ -301,11 +317,11 @@ def test_completion_keeps_provider_native_model_id_through_responses(
"transform_request",
return_value={"model": bridge_model, "input": "hi"},
),
patch("litellm.responses", return_value=cached) as responses_call,
patch.object(bridge_executor, "native_responses", return_value=cached) as responses_call,
):
bridge.completion(**kwargs)
handed_model = responses_call.call_args.kwargs["model"]
handed_model = responses_call.call_args.args[1]
assert _upstream_model_for(handed_model, custom_llm_provider) == expected_upstream_model
@ -329,9 +345,306 @@ async def test_acompletion_keeps_provider_native_model_id_through_responses(
"transform_request",
return_value={"model": bridge_model, "input": "hi"},
),
patch("litellm.aresponses", new=AsyncMock(return_value=cached)) as responses_call,
patch.object(bridge_executor, "anative_responses", new=AsyncMock(return_value=cached)) as responses_call,
):
await bridge.acompletion(**kwargs)
handed_model = responses_call.call_args.kwargs["model"]
handed_model = responses_call.call_args.args[1]
assert _upstream_model_for(handed_model, custom_llm_provider) == expected_upstream_model
class _DeploymentHookCounter(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.calls = 0
async def async_pre_call_deployment_hook(self, kwargs, call_type) -> None:
self.calls += 1
@pytest.mark.asyncio
async def test_bridged_chat_call_runs_deployment_hooks_once(monkeypatch):
counter = _DeploymentHookCounter()
monkeypatch.setattr(litellm, "callbacks", [counter])
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
litellm.register_model(
{"openai/bridge-only-model": {"litellm_provider": "openai", "mode": "responses", "max_tokens": 100}}
)
with respx.mock(assert_all_called=False) as upstream:
responses_route = upstream.post("https://example.invalid/v1/responses").mock(
return_value=httpx.Response(
200,
json={
"id": "resp_bridge",
"object": "response",
"created_at": 1,
"status": "completed",
"model": "bridge-only-model",
"output": [
{
"type": "message",
"id": "msg_bridge",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "from upstream", "annotations": []}],
}
],
"usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
},
)
)
response = await litellm.acompletion(
model="openai/bridge-only-model",
api_base="https://example.invalid/v1",
api_key="x",
messages=[{"role": "user", "content": "hi"}],
)
assert response.choices[0].message.content == "from upstream"
assert responses_route.call_count == 1
assert counter.calls == 1
@pytest.mark.parametrize(
"bridge_model, tools",
[
("gpt-5.4", [{"type": "mcp", "server_label": "docs", "server_url": "litellm_proxy"}]),
("gpt-5.4", [{"type": "file_search", "vector_store_ids": ["vs_1"]}]),
("chat_completions/gpt-5.4", None),
],
)
def test_completion_keeps_responses_surface_for_what_only_it_serves(bridge_model, tools):
cached = ModelResponse(id="chatcmpl-surface", model="gpt-5.4")
bridge = ResponsesToCompletionBridgeHandler()
kwargs = _bridge_kwargs(stream=False)
kwargs["model"] = bridge_model
with (
patch.object(
bridge.transformation_handler,
"transform_request",
return_value={"model": bridge_model, "input": "hi", "tools": tools},
),
patch("litellm.responses", return_value=cached) as responses_call,
patch.object(bridge_executor, "native_responses") as executor_call,
):
result = bridge.completion(**kwargs)
assert result is cached
assert responses_call.call_count == 1
assert executor_call.call_count == 0
_LOGGED_ROW: Final = TypeAdapter(dict[str, object])
_LOGGED_CHAT: Final = TypeAdapter(ChatCompletion)
class _SuccessRowRecorder(CustomLogger):
rows: tuple[dict[str, object], ...] = ()
async def async_log_success_event(
self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime
) -> None:
self.rows = (*self.rows, _LOGGED_ROW.validate_python(kwargs["standard_logging_object"]))
def _responses_sse(events: Sequence[Mapping[str, object]]) -> bytes:
return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events).encode()
@pytest.mark.asyncio
async def test_bridged_chat_stream_logs_one_chat_row_billed_from_provider_usage(monkeypatch: pytest.MonkeyPatch):
recorder: Final = _SuccessRowRecorder()
monkeypatch.setattr(litellm, "callbacks", [recorder])
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
prices: Final = {"input_cost_per_token": 1e-6, "output_cost_per_token": 4e-6, "cache_read_input_token_cost": 1e-7}
litellm.register_model(
{
"openai/bridge-stream-model": {
"litellm_provider": "openai",
"mode": "responses",
"max_tokens": 100,
**prices,
}
}
)
usage: Final[dict[str, object]] = {
"input_tokens": 100,
"input_tokens_details": {"cached_tokens": 40},
"output_tokens": 20,
"output_tokens_details": {"reasoning_tokens": 5},
"total_tokens": 120,
}
message: Final[dict[str, object]] = {
"type": "message",
"id": "msg_stream",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "streamed", "annotations": []}],
}
response: Final[dict[str, object]] = {
"id": "resp_stream",
"object": "response",
"created_at": 1,
"model": "bridge-stream-model",
"output": [],
"parallel_tool_calls": True,
"tool_choice": "auto",
"tools": [],
}
sse: Final = _responses_sse(
[
{"type": "response.created", "sequence_number": 0, "response": {**response, "status": "in_progress"}},
{"type": "response.output_item.added", "sequence_number": 1, "output_index": 0, "item": message},
{
"type": "response.output_text.delta",
"sequence_number": 2,
"item_id": "msg_stream",
"output_index": 0,
"content_index": 0,
"delta": "streamed",
},
{"type": "response.output_item.done", "sequence_number": 3, "output_index": 0, "item": message},
{
"type": "response.completed",
"sequence_number": 4,
"response": {**response, "status": "completed", "output": [message], "usage": usage},
},
]
)
with respx.mock(assert_all_called=False) as upstream:
responses_route = upstream.post("https://example.invalid/v1/responses").mock(
return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, content=sse)
)
stream = await litellm.acompletion(
model="openai/bridge-stream-model",
api_base="https://example.invalid/v1",
api_key="x",
messages=[{"role": "user", "content": "hi"}],
stream=True,
)
assert isinstance(stream, CustomStreamWrapper)
text = "".join([chunk.choices[0].delta.content or "" async for chunk in stream if chunk.choices])
for _ in range(100):
if recorder.rows:
break
await asyncio.sleep(0.01)
await asyncio.sleep(0.1)
assert text == "streamed"
assert responses_route.call_count == 1
assert len(recorder.rows) == 1
row: Final = recorder.rows[0]
logged: Final = _LOGGED_CHAT.validate_python(row["response"])
assert logged.object == "chat.completion"
assert logged.choices[0].message.content == "streamed"
assert row["prompt_tokens"] == 100
assert row["completion_tokens"] == 20
cost: Final = row["response_cost"]
assert isinstance(cost, float)
assert math.isclose(
cost,
60 * prices["input_cost_per_token"]
+ 40 * prices["cache_read_input_token_cost"]
+ 20 * prices["output_cost_per_token"],
)
_BUILT_IN_TOOL_CALLS: Final[Mapping[str, tuple[Sequence[Mapping[str, object]], Sequence[Mapping[str, object]]]]] = {
"web_search": (
tuple(
{
"type": "web_search_call",
"id": f"ws_{i}",
"status": "completed",
"action": {"type": "search", "query": "q"},
}
for i in range(3)
),
(),
),
"image_generation": (
(
{
"type": "image_generation_call",
"id": "ig_1",
"status": "completed",
"result": "aGk=",
"quality": "low",
"size": "1024x1024",
},
),
({"type": "image_generation", "model": "gpt-image-1"},),
),
}
def _responses_body_with(tool_call: str) -> dict[str, object]:
output_items, tools = _BUILT_IN_TOOL_CALLS[tool_call]
message: Final[dict[str, object]] = {
"type": "message",
"id": "msg_tool",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "done", "annotations": []}],
}
return {
"id": "resp_tool",
"object": "response",
"created_at": 1,
"status": "completed",
"model": "gpt-4.1",
"output": [*output_items, message],
"parallel_tool_calls": True,
"tool_choice": "auto",
"tools": list(tools),
"usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
}
async def _rows_for(recorder: _SuccessRowRecorder, count: int) -> tuple[dict[str, object], ...]:
for _ in range(200):
if len(recorder.rows) >= count:
break
await asyncio.sleep(0.01)
return recorder.rows
@pytest.mark.asyncio
@pytest.mark.parametrize("tool_call", sorted(_BUILT_IN_TOOL_CALLS))
async def test_bridged_chat_call_is_billed_for_the_built_in_tool_calls_the_provider_made(
monkeypatch: pytest.MonkeyPatch, tool_call: str
):
recorder: Final = _SuccessRowRecorder()
monkeypatch.setattr(litellm, "callbacks", [recorder])
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
with respx.mock(assert_all_called=False) as upstream:
upstream.post("https://example.invalid/v1/responses").mock(
return_value=httpx.Response(200, json=_responses_body_with(tool_call))
)
bridged = await litellm.acompletion(
model="openai/responses/gpt-4.1",
api_base="https://example.invalid/v1",
api_key="x",
messages=[{"role": "user", "content": "hi"}],
)
await _rows_for(recorder, 1)
await litellm.aresponses(model="openai/gpt-4.1", api_base="https://example.invalid/v1", api_key="x", input="hi")
rows: Final = await _rows_for(recorder, 2)
bridged_row, native_row = rows
assert bridged_row["call_type"] == "acompletion"
assert native_row["call_type"] == "aresponses"
native_cost: Final = native_row["response_cost"]
assert isinstance(native_cost, float)
assert bridged_row["response_cost"] == pytest.approx(native_cost)
assert bridged._hidden_params["response_cost"] == pytest.approx(native_cost)
breakdown: Final = _LOGGED_ROW.validate_python(bridged_row["cost_breakdown"])
tool_usage_cost: Final = breakdown["tool_usage_cost"]
assert isinstance(tool_usage_cost, float) and tool_usage_cost > 0

View file

@ -13,6 +13,7 @@ from litellm.completion_extras.litellm_responses_transformation.handler import (
ResponsesToCompletionBridgeHandler,
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.responses import bridge_executor
from litellm.types.utils import ModelResponse
MODEL = "openai.gpt-5.5"
@ -73,11 +74,11 @@ def test_completion_forwards_custom_llm_provider_to_responses():
bridge = ResponsesToCompletionBridgeHandler()
cached = ModelResponse(id="chatcmpl-cached", model="gpt-5.5")
with patch("litellm.responses", return_value=cached) as fake_responses:
with patch.object(bridge_executor, "native_responses", return_value=cached) as fake_responses:
result = bridge.completion(**_openai_kwargs())
assert result is cached
assert fake_responses.call_args.kwargs["custom_llm_provider"] == "openai"
assert fake_responses.call_args.args[2]["custom_llm_provider"] == "openai"
@pytest.mark.asyncio
@ -85,13 +86,13 @@ async def test_acompletion_forwards_custom_llm_provider_to_aresponses():
bridge = ResponsesToCompletionBridgeHandler()
cached = ModelResponse(id="chatcmpl-cached", model="gpt-5.5")
async def _fake_aresponses(**kwargs):
_fake_aresponses.kwargs = kwargs
async def _fake_aresponses(request, model, forwarded_kwargs, logging_obj):
_fake_aresponses.kwargs = forwarded_kwargs
return cached
_fake_aresponses.kwargs = {}
with patch("litellm.aresponses", _fake_aresponses):
with patch.object(bridge_executor, "anative_responses", _fake_aresponses):
result = await bridge.acompletion(**_openai_kwargs())
assert result is cached
@ -103,13 +104,13 @@ async def test_acompletion_forwards_aws_region_name_to_aresponses():
bridge = ResponsesToCompletionBridgeHandler()
cached = ModelResponse(id="chatcmpl-cached", model=MODEL)
async def _fake_aresponses(**kwargs):
_fake_aresponses.kwargs = kwargs
async def _fake_aresponses(request, model, forwarded_kwargs, logging_obj):
_fake_aresponses.kwargs = forwarded_kwargs
return cached
_fake_aresponses.kwargs = {}
with patch("litellm.aresponses", _fake_aresponses):
with patch.object(bridge_executor, "anative_responses", _fake_aresponses):
result = await bridge.acompletion(**_bedrock_mantle_kwargs())
assert result is cached

View file

@ -31,11 +31,11 @@ def _funnel_kwargs_completion_forwards(monkeypatch, wif_kwargs: dict[str, object
from unittest.mock import MagicMock
import litellm
import litellm.main as litellm_main
import litellm.native.chat as native_chat
spy = MagicMock(wraps=litellm_main.get_litellm_params)
spy = MagicMock(wraps=native_chat.get_litellm_params)
monkeypatch.setattr( # test-quality-ok: completion() has no injection seam for its kwargs funnel
litellm_main, "get_litellm_params", spy
native_chat, "get_litellm_params", spy
)
litellm.completion(
model="anthropic/claude-sonnet-5",

View file

@ -66,6 +66,14 @@ def test_translate_openai_response_to_anthropic_empty_choices() -> None:
assert result["usage"]["input_tokens"] == 10
def test_translate_openai_response_to_anthropic_without_usage_reports_zero_tokens() -> None:
response: Final = ModelResponse(choices=[Choices(message=Message(content="x"), finish_reason="stop")])
result: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response)
assert (result["usage"]["input_tokens"], result["usage"]["output_tokens"]) == (0, 0)
@pytest.mark.parametrize("text,count,expected_stop", [
("", 1, "compaction"), (None, 1, "compaction"), ("Answer", 1, "max_tokens"),
(" ", 1, "max_tokens"), ("", 2, "max_tokens"), ("", 0, "max_tokens"),

View file

@ -0,0 +1,380 @@
import asyncio
import datetime
from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
from types import MappingProxyType
from typing import Final, TypeAlias, cast
import httpx
import pytest
import respx
from pydantic import TypeAdapter
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.custom_prompt_management import CustomPromptManagement
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.anthropic.messages.adapters.handler import LiteLLMMessagesToCompletionTransformationHandler
from litellm.llms.anthropic.messages.adapters.streaming_iterator import AnthropicSSEStream
from litellm.types.llms.openai import AllMessageValues
from litellm.types.prompts.init_prompts import PromptSpec
from litellm.types.utils import StandardCallbackDynamicParams
API_BASE: Final = "http://chat-upstream.test/v1"
CHAT_RESPONSE: Final = {
"id": "chatcmpl-bridge",
"object": "chat.completion",
"created": 1,
"model": "served-model",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "from upstream"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
}
class _DeploymentHookCounter(CustomLogger):
pre_call: int = 0
post_call: int = 0
async def async_pre_call_deployment_hook(self, kwargs: Mapping[str, object], call_type: object) -> None:
self.pre_call += 1
async def async_post_call_success_deployment_hook(
self, request_data: Mapping[str, object], response: object, call_type: object
) -> None:
self.post_call += 1
_PromptResult: TypeAlias = tuple[str, list[AllMessageValues], dict[str, object]]
class _TemplatePromptManager(CustomPromptManagement):
async def async_get_chat_completion_prompt(
self,
model: str,
messages: Sequence[AllMessageValues],
non_default_params: Mapping[str, object],
prompt_id: str | None,
prompt_variables: Mapping[str, object] | None,
dynamic_callback_params: StandardCallbackDynamicParams,
litellm_logging_obj: Logging,
prompt_spec: PromptSpec | None = None,
tools: Sequence[Mapping[str, object]] | None = None,
prompt_label: str | None = None,
prompt_version: int | None = None,
ignore_prompt_manager_model: bool | None = False,
ignore_prompt_manager_optional_params: bool | None = False,
) -> _PromptResult:
return model, [{"role": "user", "content": "from the template"}], {**non_default_params}
@pytest.fixture
def upstream(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch) -> respx.Route:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
return respx_mock.post(f"{API_BASE}/chat/completions").mock(return_value=httpx.Response(200, json=CHAT_RESPONSE))
async def _ask(extra: Mapping[str, object] = MappingProxyType({})) -> object:
return await litellm.anthropic.messages.acreate(
model="hosted_vllm/served-model",
api_base=API_BASE,
api_key="sk-chat",
max_tokens=10,
messages=[{"role": "user", "content": "hi"}],
**extra,
)
@pytest.mark.asyncio
async def test_bridge_runs_deployment_hooks_once_per_request(upstream: respx.Route, monkeypatch: pytest.MonkeyPatch):
counter: Final = _DeploymentHookCounter()
monkeypatch.setattr(litellm, "callbacks", [counter])
response: Final = await _ask()
assert isinstance(response, dict)
assert response["content"] == [{"type": "text", "text": "from upstream"}]
assert upstream.call_count == 1
assert (counter.pre_call, counter.post_call) == (1, 1)
@pytest.mark.asyncio
async def test_bridge_streams_the_upstream_reply(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
chunk: Final = '{"id":"c","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{"role":"assistant","content":"streamed"},"finish_reason":null}]}'
done: Final = '{"id":"c","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}'
respx_mock.post(f"{API_BASE}/chat/completions").mock(
return_value=httpx.Response(
200,
content=f"data: {chunk}\n\ndata: {done}\n\ndata: [DONE]\n\n".encode(),
headers={"content-type": "text/event-stream"},
)
)
stream: Final = await _ask({"stream": True})
body: Final = b"".join([event async for event in stream]).decode()
assert "event: message_start" in body
assert '"text":"streamed"' in body.replace(" ", "")
@pytest.mark.asyncio
async def test_bridge_applies_the_prompt_template_named_by_prompt_id(
upstream: respx.Route, monkeypatch: pytest.MonkeyPatch
):
monkeypatch.setattr(litellm, "callbacks", [_TemplatePromptManager()])
await _ask({"prompt_id": "bridge_template"})
assert "from the template" in upstream.calls.last.request.content.decode()
@pytest.mark.asyncio
async def test_bridge_maps_an_upstream_429_to_rate_limit_error(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
respx_mock.post(f"{API_BASE}/chat/completions").mock(
return_value=httpx.Response(429, json={"error": {"message": "slow down", "type": "rate_limit"}})
)
with pytest.raises(litellm.RateLimitError):
await _ask()
def _sync_bridge(logging_obj: Logging, *, stream: bool) -> object:
return LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler(
max_tokens=10,
messages=[{"role": "user", "content": "hi"}],
model="hosted_vllm/served-model",
stream=stream,
api_base=API_BASE,
api_key="sk-chat",
litellm_logging_obj=logging_obj,
)
def test_sync_bridge_answers_from_one_upstream_call(upstream: respx.Route):
logging_obj: Final = Logging(
model="hosted_vllm/served-model",
messages=[{"role": "user", "content": "hi"}],
stream=False,
call_type="anthropic_messages",
start_time=datetime.datetime.now(datetime.timezone.utc),
litellm_call_id="6825beef-0000-4000-8000-000000000005",
function_id="1234",
)
response: Final = _sync_bridge(logging_obj, stream=False)
assert isinstance(response, dict)
assert response["content"] == [{"type": "text", "text": "from upstream"}]
assert upstream.call_count == 1
@pytest.mark.asyncio
async def test_bridge_runs_prompt_management_triggered_without_prompt_id(
upstream: respx.Route, monkeypatch: pytest.MonkeyPatch
):
monkeypatch.setattr(litellm, "callbacks", [_TemplatePromptManager()])
await _ask({"vector_store_ids": ["vs_bridge"]})
assert "from the template" in upstream.calls.last.request.content.decode()
@pytest.mark.asyncio
async def test_bridge_keeps_the_litellm_system_prompt(upstream: respx.Route):
await _ask({"litellm_system_prompt": "answer like a pirate"})
assert "answer like a pirate" in upstream.calls.last.request.content.decode()
@pytest.mark.asyncio
async def test_bridge_goes_through_a_wrapped_acompletion(upstream: respx.Route, monkeypatch: pytest.MonkeyPatch):
unwrapped: Final = litellm.acompletion
seen: Final[list[object]] = [] # mutable-ok: records the calls the wrapper sees
async def traced_acompletion(*args: object, **kwargs: object) -> object:
seen.append(kwargs.get("model"))
return await unwrapped(*args, **kwargs)
monkeypatch.setattr(litellm, "acompletion", traced_acompletion)
response: Final = await _ask()
assert isinstance(response, dict)
assert response["content"] == [{"type": "text", "text": "from upstream"}]
assert seen == ["hosted_vllm/served-model"]
assert upstream.call_count == 1
class _SuccessRowRecorder(CustomLogger):
call_types: tuple[str, ...] = ()
async def async_log_success_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
self.call_types = (*self.call_types, str(kwargs.get("call_type")))
def log_success_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
self.call_types = (*self.call_types, str(kwargs.get("call_type")))
_PARKED_COMPLETION: Final = TypeAdapter(tuple[object])
async def _release(parked: Coroutine[object, object, object]) -> None:
await parked
async def _settled_rows(recorder: _SuccessRowRecorder) -> tuple[str, ...]:
for _ in range(200):
if recorder.call_types:
break
await asyncio.sleep(0.01)
await asyncio.sleep(0.05)
return recorder.call_types
STREAM_CHUNK: Final = '{"id":"c","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{"role":"assistant","content":"streamed"},"finish_reason":null}]}'
STREAM_DONE: Final = '{"id":"c","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}'
@pytest.fixture
def stream_recorder(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch) -> _SuccessRowRecorder:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
respx_mock.post(f"{API_BASE}/chat/completions").mock(
return_value=httpx.Response(
200,
content=f"data: {STREAM_CHUNK}\n\ndata: {STREAM_DONE}\n\ndata: [DONE]\n\n".encode(),
headers={"content-type": "text/event-stream"},
)
)
recorder: Final = _SuccessRowRecorder()
monkeypatch.setattr(litellm, "callbacks", [recorder])
return recorder
async def _read_through_message_stop(stream: AsyncIterator[bytes]) -> None:
async for frame in stream:
if frame.startswith(b"event: message_stop"):
return
@pytest.mark.asyncio
async def test_bridged_stream_logs_one_row_when_the_client_stream_ends(stream_recorder: _SuccessRowRecorder):
stream: Final = await _ask({"stream": True})
assert isinstance(stream, AnthropicSSEStream)
await _read_through_message_stop(stream)
rows_while_the_client_stream_is_open: Final = await _settled_rows(stream_recorder)
remaining: Final[list[bytes]] = [frame async for frame in stream]
assert rows_while_the_client_stream_is_open == ()
assert remaining == []
assert await _settled_rows(stream_recorder) == ("anthropic_messages",)
@pytest.mark.asyncio
async def test_bridged_stream_still_logs_when_the_client_leaves_after_message_stop(
stream_recorder: _SuccessRowRecorder,
):
stream: Final = await _ask({"stream": True})
assert isinstance(stream, AnthropicSSEStream)
await _read_through_message_stop(stream)
await stream.aclose()
assert await _settled_rows(stream_recorder) == ("anthropic_messages",)
@pytest.mark.asyncio
async def test_bridged_stream_holds_its_row_for_the_proxy_end_of_stream_guardrails(
stream_recorder: _SuccessRowRecorder, monkeypatch: pytest.MonkeyPatch
):
monkeypatch.setattr(litellm, "_async_success_callback", [stream_recorder])
logging_obj: Final = Logging(
model="hosted_vllm/served-model",
messages=[{"role": "user", "content": "hi"}],
stream=True,
call_type="anthropic_messages",
start_time=datetime.datetime.now(datetime.timezone.utc),
litellm_call_id="6825beef-0000-4000-8000-000000000007",
function_id="1234",
)
async def release(*held: object) -> None:
return None
setattr(logging_obj, "_on_deferred_stream_complete", release) # noqa: B010 # the proxy arms this slot dynamically
stream: Final = await LiteLLMMessagesToCompletionTransformationHandler.async_anthropic_messages_handler(
max_tokens=10,
messages=[{"role": "user", "content": "hi"}],
model="hosted_vllm/served-model",
stream=True,
api_base=API_BASE,
api_key="sk-chat",
litellm_logging_obj=logging_obj,
)
assert isinstance(stream, AnthropicSSEStream)
frames: Final = [frame async for frame in stream]
rows_before_release: Final = await _settled_rows(stream_recorder)
held: Final[object] = getattr(logging_obj, "_deferred_stream_complete_args", None)
assert frames[-1].startswith(b"event: message_stop")
assert rows_before_release == ()
(parked,) = _PARKED_COMPLETION.validate_python(held)
assert asyncio.iscoroutine(parked)
await _release(parked)
assert stream_recorder.call_types == ("anthropic_messages",)
def test_sync_bridged_stream_logs_one_row(stream_recorder: _SuccessRowRecorder, monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(litellm, "success_callback", [stream_recorder])
logging_obj: Final = Logging(
model="hosted_vllm/served-model",
messages=[{"role": "user", "content": "hi"}],
stream=True,
call_type="anthropic_messages",
start_time=datetime.datetime.now(datetime.timezone.utc),
litellm_call_id="6825beef-0000-4000-8000-000000000006",
function_id="1234",
)
stream: Final = _sync_bridge(logging_obj, stream=True)
assert isinstance(stream, Iterator)
frames: Final = list(cast(Iterator[bytes], stream)) # cast-ok: the sync bridge streams SSE frames as bytes
assert frames[-1].startswith(b"event: message_stop")
assert asyncio.run(_settled_rows(stream_recorder)) == ("anthropic_messages",)
@pytest.mark.asyncio
async def test_bridged_empty_stream_still_logs_one_row(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
respx_mock.post(f"{API_BASE}/chat/completions").mock(
return_value=httpx.Response(200, content=b"data: [DONE]\n\n", headers={"content-type": "text/event-stream"})
)
recorder: Final = _SuccessRowRecorder()
monkeypatch.setattr(litellm, "callbacks", [recorder])
# deepinfra sends no stream_options, so the empty stream assembles no response to log
stream: Final = await litellm.anthropic.messages.acreate(
model="deepinfra/meta-llama/Meta-Llama-3.1-8B-Instruct",
api_base=API_BASE,
api_key="sk-chat",
max_tokens=10,
messages=[{"role": "user", "content": "hi"}],
stream=True,
)
assert isinstance(stream, AnthropicSSEStream)
frames: Final = [frame async for frame in stream]
assert frames[-1].startswith(b"event: message_stop")
assert await _settled_rows(recorder) == ("anthropic_messages",)

View file

@ -19,9 +19,9 @@ import json
import os
import sys
import threading
from collections.abc import Callable, Mapping, Sequence
from collections.abc import AsyncIterator, Callable, Iterator, Mapping, Sequence
from datetime import datetime
from typing import Final, Iterator, List, Optional
from typing import Final, List, Optional
from unittest.mock import MagicMock
import httpx
@ -40,11 +40,12 @@ from litellm.llms.anthropic.messages.adapters.streaming_iterator import (
AnthropicStreamWrapper,
_mid_stream_error_sse_event,
)
from litellm.llms.anthropic.messages.held_stream_completion import HeldStreamCompletion
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.types.utils import Delta, ModelResponseStream, StandardLoggingPayload, StreamingChoices, Usage
def _make_chunk(delta: Delta, finish_reason: Optional[str] = None) -> MagicMock:
def _make_chunk(delta: Delta, finish_reason: str | None = None) -> MagicMock:
chunk = MagicMock()
chunk.choices = [
StreamingChoices(finish_reason=finish_reason, index=0, delta=delta, logprobs=None)
@ -78,7 +79,7 @@ def _parse_sse(raw: bytes) -> tuple[str, dict]:
return event_line.removeprefix("event: "), json.loads(data_line.removeprefix("data: "))
async def _drain_sse(wrapper: AnthropicStreamWrapper) -> List[bytes]:
async def _drain_sse(wrapper: AnthropicStreamWrapper) -> list[bytes]:
return [event async for event in wrapper.async_anthropic_sse_wrapper()]
@ -385,3 +386,51 @@ async def test_mid_stream_error_emits_error_event_without_logging_obj():
events = await _drain_sse(_failing_wrapper(None))
assert _parse_sse(events[-1])[0] == "error"
class _AsyncSuccessRecorder(CustomLogger):
calls: tuple[object, ...] = ()
async def async_log_success_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
self.calls = (*self.calls, response_obj)
class _CompletesThenRaises:
def __init__(self, exc: Exception, held: HeldStreamCompletion) -> None:
self._exc: Final = exc
self._held: Final = held
def __iter__(self) -> Iterator[ModelResponseStream]:
return iter(())
def __aiter__(self) -> AsyncIterator[ModelResponseStream]:
return self._chunks()
async def _chunks(self) -> AsyncIterator[ModelResponseStream]:
yield ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content="partial"))])
self._held.take(litellm.ModelResponse())
raise self._exc
@pytest.mark.asyncio
async def test_a_stream_that_fails_after_the_executor_completed_logs_no_success_row():
success_recorder: Final = _AsyncSuccessRecorder()
async_recorder: Final = _AsyncFailureRecorder()
logging_obj: Final = _make_logging_obj("fails-after-complete", async_recorder, _SyncFailureRecorder())
logging_obj.dynamic_async_success_callbacks = [success_recorder]
held: Final = HeldStreamCompletion(logging_obj)
wrapper: Final = AnthropicStreamWrapper(
completion_stream=_CompletesThenRaises(_bedrock_drop(), held),
model="bedrock-converse-sonnet-4-6",
litellm_logging_obj=logging_obj,
held_completion=held,
)
events: Final = await _drain_sse(wrapper)
await asyncio.sleep(0.2)
assert _parse_sse(events[-1])[0] == "error"
assert [str(exc) for exc in async_recorder.exceptions] == ["ConverseStream ended without messageStop"]
assert success_recorder.calls == ()

View file

@ -103,15 +103,18 @@ async def _mock_streaming_chunks():
)
_PLANNED_CHAT = "litellm.llms.anthropic.messages.adapters.handler.aexecute_planned_chat"
@pytest.mark.asyncio
async def test_polyfill_round_trip_non_streaming():
captured = {}
async def fake_acompletion(**kwargs):
captured.update(kwargs)
async def fake_chat(plan):
captured.update({**plan.kwargs, **plan.named})
return _mock_completion_response()
with patch("litellm.acompletion", side_effect=fake_acompletion):
with patch(_PLANNED_CHAT, side_effect=fake_chat):
response = await litellm.anthropic.messages.acreate(
model=MODEL,
messages=_make_history(n_pairs=5),
@ -173,11 +176,11 @@ async def test_polyfill_round_trip_non_streaming():
async def test_polyfill_trigger_not_met_passes_through_unchanged():
captured = {}
async def fake_acompletion(**kwargs):
captured.update(kwargs)
async def fake_chat(plan):
captured.update({**plan.kwargs, **plan.named})
return _mock_completion_response()
with patch("litellm.acompletion", side_effect=fake_acompletion):
with patch(_PLANNED_CHAT, side_effect=fake_chat):
response = await litellm.anthropic.messages.acreate(
model=MODEL,
messages=_make_history(n_pairs=2),
@ -209,10 +212,10 @@ async def test_polyfill_trigger_not_met_passes_through_unchanged():
@pytest.mark.asyncio
async def test_polyfill_streaming_attaches_to_message_delta():
async def fake_acompletion(**kwargs):
async def fake_chat(plan):
return _mock_streaming_chunks()
with patch("litellm.acompletion", side_effect=fake_acompletion):
with patch(_PLANNED_CHAT, side_effect=fake_chat):
response = await litellm.anthropic.messages.acreate(
model=MODEL,
messages=_make_history(n_pairs=5),

View file

@ -2,20 +2,20 @@ import asyncio
import copy
import json
import os
import unittest.mock
import uuid
from dataclasses import dataclass
from collections.abc import Mapping
from typing import Any, Dict, Final, List
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import responses
import respx
import pytest
import respx
from fastapi.testclient import TestClient
from pydantic import TypeAdapter, ValidationError
from unittest.mock import AsyncMock, MagicMock, patch
import litellm
from litellm.anthropic_interface import messages
from litellm.integrations.custom_logger import CustomLogger
@ -30,7 +30,6 @@ from litellm.types.utils import (
StreamingChoices,
)
from litellm.utils import _invalidate_model_cost_lowercase_map
import unittest.mock
@pytest.mark.asyncio
@ -340,8 +339,10 @@ async def test_bedrock_converse_budget_tokens_preserved():
usage={"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
)
with patch("litellm.acompletion", new_callable=AsyncMock) as mock_acompletion:
mock_acompletion.return_value = mock_response
with patch(
"litellm.llms.anthropic.messages.adapters.handler.aexecute_planned_chat", new_callable=AsyncMock
) as mock_chat:
mock_chat.return_value = mock_response
try:
await messages.acreate(
@ -353,13 +354,9 @@ async def test_bedrock_converse_budget_tokens_preserved():
except (ValueError, TypeError, AttributeError):
pass # Expected due to response format conversion
mock_acompletion.assert_called_once()
mock_chat.assert_called_once()
call_kwargs = mock_acompletion.call_args.kwargs
print("acompletion call kwargs: ", json.dumps(call_kwargs, indent=4, default=str))
# Verify thinking parameter is passed through with budget_tokens preserved
thinking_param = call_kwargs.get("thinking")
thinking_param = mock_chat.call_args.args[0].named.get("thinking")
assert thinking_param is not None, "thinking parameter should be passed to acompletion"
assert thinking_param.get("type") == "enabled", "thinking.type should be 'enabled'"
assert thinking_param.get("budget_tokens") == 1024, (
@ -1241,7 +1238,10 @@ async def test_messages_streaming_reports_provider_local_model(requested_model,
async def __anext__(self):
raise StopAsyncIteration
with patch("litellm.acompletion", new=AsyncMock(return_value=_EmptyStream())):
with patch(
"litellm.llms.anthropic.messages.adapters.handler.aexecute_planned_chat",
new=AsyncMock(return_value=_EmptyStream()),
):
stream = await litellm.anthropic.messages.acreate(
max_tokens=100,
messages=[{"role": "user", "content": "ping"}],
@ -2455,3 +2455,81 @@ async def test_anthropic_messages_strict_tools_rejection_is_sent_again_without_s
assert first["tools"] == [tool]
assert second == {**first, "tools": [{"name": "lookup_record", "input_schema": {"type": "object"}}]}
assert _JSON_BODY.validate_python(response)["id"] == "msg_strict_retry"
_REASONING_RESPONSE: Final = {
"id": "resp_thinking",
"object": "response",
"created_at": 1,
"status": "completed",
"model": "gpt-5.5",
"output": [
{"type": "reasoning", "id": "rs_1", "summary": [{"type": "summary_text", "text": "worked it out"}]},
{
"type": "message",
"id": "msg_1",
"role": "assistant",
"status": "completed",
"content": [{"type": "output_text", "text": "the answer", "annotations": []}],
},
],
"parallel_tool_calls": True,
"tool_choice": "auto",
"tools": [],
"usage": {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7},
}
class _DeploymentHookCounter(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.calls = 0
async def async_pre_call_deployment_hook(self, kwargs: Mapping[str, object], call_type: object) -> None:
self.calls += 1
@pytest.mark.asyncio
async def test_forced_chat_sends_openai_thinking_to_responses_through_the_chat_bridge(
monkeypatch: pytest.MonkeyPatch,
):
counter: Final = _DeploymentHookCounter()
monkeypatch.setattr(litellm, "callbacks", [counter])
monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", True)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
with respx.mock(assert_all_called=False) as upstream:
responses_route: Final = upstream.post("http://openai.test/v1/responses").mock(
return_value=httpx.Response(200, json=_REASONING_RESPONSE)
)
chat_route: Final = upstream.post("http://openai.test/v1/chat/completions")
response: Final = await litellm.anthropic.messages.acreate(
model="openai/gpt-5.5",
api_base="http://openai.test/v1",
api_key="sk-test",
max_tokens=1024,
messages=[
{"role": "user", "content": "weather?"},
{
"role": "assistant",
"content": [
{"type": "text", "text": "let me check"},
{"type": "tool_use", "id": "toolu_1", "name": "get_w", "input": {"c": "a"}},
],
},
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "sunny"}]},
],
tools=[{"name": "get_w", "description": "d", "input_schema": {"type": "object", "properties": {}}}],
thinking={"type": "enabled", "budget_tokens": 1024},
)
assert isinstance(response, dict)
assert [block["type"] for block in response["content"]] == ["thinking", "text"]
assert response["content"][1]["text"] == "the answer"
assert (responses_route.call_count, chat_route.call_count) == (1, 0)
assert counter.calls == 1
sent_input: Final = json.loads(responses_route.calls[0].request.content)["input"]
assert [item["type"] for item in sent_input] == ["message", "message", "function_call", "function_call_output"]
assert sent_input[1]["role"] == "assistant"
assert sent_input[3]["output"] == [{"type": "input_text", "text": "sunny"}]

View file

@ -32,13 +32,7 @@ def _call_handler_and_capture_optional_params(thinking=None, **extra_kwargs):
with patch(
"litellm.llms.anthropic.messages.native.handler."
"base_llm_http_handler"
) as mock_handler, patch(
"litellm.llms.anthropic.messages.native.handler."
"ProviderConfigManager"
) as mock_pcm:
# Make get_provider_anthropic_messages_config return a non-None config
# so the handler takes the native Anthropic path
mock_pcm.get_provider_anthropic_messages_config.return_value = MagicMock()
) as mock_handler:
mock_handler.anthropic_messages_handler.return_value = MagicMock()
kwargs = dict(extra_kwargs)

View file

@ -1,30 +1,40 @@
import asyncio
import datetime
import json
import os
import sys
from collections.abc import AsyncIterator, Mapping
from collections.abc import AsyncIterator, Callable, Coroutine, Mapping
from itertools import chain
from typing import Final, cast
from unittest.mock import AsyncMock, patch
import httpx
import pytest
import respx
from pydantic import TypeAdapter
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../..")))
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.custom_prompt_management import CustomPromptManagement
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.prompt_templates.common_utils import (
TOOL_RESULT_IMAGE_BOUNDARY,
TOOL_RESULT_IMAGE_PLACEHOLDER,
)
from litellm.llms.anthropic.messages.responses_adapters.handler import (
LiteLLMMessagesToResponsesAPIHandler,
_build_responses_kwargs,
_build_responses_request,
)
from litellm.llms.anthropic.messages.responses_adapters.streaming_iterator import AnthropicResponsesSSEByteStream
from litellm.router import Router
from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS, converted_stream_requested
from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS
from litellm.types.llms.openai import AllMessageValues
from litellm.types.prompts.init_prompts import PromptSpec
from litellm.types.utils import StandardCallbackDynamicParams
MESSAGES = [{"role": "user", "content": "hello"}]
MESSAGES: Final[list[dict[str, object]]] = [{"role": "user", "content": "hello"}]
CLAUDE_CODE_USER_ID = json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": "session-abc"})
RESPONSES_SSE_BODY = (
@ -40,20 +50,8 @@ RESPONSES_SSE_BODY = (
)
def test_build_responses_kwargs_never_asks_the_nested_call_to_replay_a_stream():
responses_kwargs: Final = _build_responses_kwargs(
max_tokens=1024,
messages=MESSAGES,
model="azure/gpt-5.6-luna",
extra_kwargs={"custom_llm_provider": "azure", **{key: True for key in CONVERTED_STREAM_KEYS}},
)
assert not converted_stream_requested(responses_kwargs)
assert responses_kwargs["custom_llm_provider"] == "azure"
def test_build_responses_kwargs_derives_prompt_cache_key_from_claude_code_session_id():
responses_kwargs = _build_responses_kwargs(
def test_build_responses_request_derives_prompt_cache_key_from_claude_code_session_id():
responses_kwargs = _build_responses_request(
max_tokens=1024,
messages=MESSAGES,
model="openai/gpt-5.6-luna",
@ -64,8 +62,8 @@ def test_build_responses_kwargs_derives_prompt_cache_key_from_claude_code_sessio
assert responses_kwargs["prompt_cache_key"] == "session-abc"
def test_build_responses_kwargs_sets_no_prompt_cache_key_for_plain_user_id():
responses_kwargs = _build_responses_kwargs(
def test_build_responses_request_sets_no_prompt_cache_key_for_plain_user_id():
responses_kwargs = _build_responses_request(
max_tokens=1024,
messages=MESSAGES,
model="openai/gpt-5.6-luna",
@ -76,8 +74,8 @@ def test_build_responses_kwargs_sets_no_prompt_cache_key_for_plain_user_id():
assert "prompt_cache_key" not in responses_kwargs
def test_build_responses_kwargs_prefers_explicit_prompt_cache_key_over_derived():
responses_kwargs = _build_responses_kwargs(
def test_build_responses_request_prefers_explicit_prompt_cache_key_over_derived():
responses_kwargs = _build_responses_request(
max_tokens=1024,
messages=MESSAGES,
model="openai/gpt-5.6-luna",
@ -88,8 +86,8 @@ def test_build_responses_kwargs_prefers_explicit_prompt_cache_key_over_derived()
assert responses_kwargs["prompt_cache_key"] == "explicit-key"
def test_build_responses_kwargs_asks_openai_for_encrypted_reasoning_without_thinking():
responses_kwargs = _build_responses_kwargs(
def test_build_responses_request_asks_openai_for_encrypted_reasoning_without_thinking():
responses_kwargs = _build_responses_request(
max_tokens=1024,
messages=MESSAGES,
model="openai/gpt-5.6-luna",
@ -99,8 +97,8 @@ def test_build_responses_kwargs_asks_openai_for_encrypted_reasoning_without_thin
assert "reasoning" not in responses_kwargs
def test_build_responses_kwargs_skips_include_for_a_responses_provider_that_rejects_it():
responses_kwargs = _build_responses_kwargs(
def test_build_responses_request_skips_include_for_a_responses_provider_that_rejects_it():
responses_kwargs = _build_responses_request(
max_tokens=1024,
messages=MESSAGES,
model="perplexity/sonar",
@ -118,11 +116,11 @@ def test_build_responses_kwargs_skips_include_for_a_responses_provider_that_reje
("openai/gpt-5.6-luna", "openai", "detailed"),
],
)
def test_build_responses_kwargs_auto_summary_asks_each_target_for_a_level_it_accepts(
def test_build_responses_request_auto_summary_asks_each_target_for_a_level_it_accepts(
monkeypatch: pytest.MonkeyPatch, model: str, provider: str, summary: str
) -> None:
monkeypatch.setenv("LITELLM_REASONING_AUTO_SUMMARY", "true")
responses_kwargs: Final = _build_responses_kwargs(
responses_kwargs: Final = _build_responses_request(
max_tokens=1024,
messages=MESSAGES,
model=model,
@ -132,8 +130,8 @@ def test_build_responses_kwargs_auto_summary_asks_each_target_for_a_level_it_acc
assert cast(Mapping[str, object], responses_kwargs["reasoning"])["summary"] == summary
def test_build_responses_kwargs_keeps_the_deployment_include_next_to_encrypted_reasoning():
responses_kwargs = _build_responses_kwargs(
def test_build_responses_request_keeps_the_deployment_include_next_to_encrypted_reasoning():
responses_kwargs = _build_responses_request(
max_tokens=1024,
messages=MESSAGES,
model="openai/gpt-5.6-luna",
@ -142,8 +140,8 @@ def test_build_responses_kwargs_keeps_the_deployment_include_next_to_encrypted_r
assert responses_kwargs["include"] == ["reasoning.encrypted_content", "file_search_call.results"]
def test_build_responses_kwargs_without_metadata_sets_no_prompt_cache_key():
responses_kwargs = _build_responses_kwargs(
def test_build_responses_request_without_metadata_sets_no_prompt_cache_key():
responses_kwargs = _build_responses_request(
max_tokens=1024,
messages=MESSAGES,
model="openai/gpt-5.6-luna",
@ -194,8 +192,6 @@ async def test_streaming_hands_the_logging_object_the_message_id_the_caller_is_s
The bridge mints the ``msg_`` id itself, and it is the only request id a streaming
/v1/messages caller ever sees, so the spend row has to be keyed on that same value.
"""
from litellm.litellm_core_utils.litellm_logging import Logging
monkeypatch.setenv("OPENAI_API_KEY", "sk-lit6825-test")
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
litellm.in_memory_llm_clients_cache.flush_cache()
@ -425,7 +421,7 @@ def _request_with_client_cache_marks(cache_control: dict[str, str] | None) -> di
def test_client_cache_control_reaches_gpt_5_6_as_a_prompt_cache_breakpoint_on_the_same_block(
extra_kwargs: dict[str, object],
) -> None:
responses_kwargs: Final = _build_responses_kwargs(
responses_kwargs: Final = _build_responses_request(
model="openai/gpt-5.6",
extra_kwargs=extra_kwargs,
**_request_with_client_cache_marks({**EPHEMERAL, "ttl": "1h"}),
@ -453,7 +449,7 @@ def test_client_cache_control_reaches_gpt_5_6_as_a_prompt_cache_breakpoint_on_th
def test_client_cache_control_on_a_screenshot_inside_a_tool_result_reaches_gpt_5_6_on_that_screenshot() -> None:
responses_kwargs: Final = _build_responses_kwargs(
responses_kwargs: Final = _build_responses_request(
model="openai/gpt-5.6",
extra_kwargs=OPENAI_API,
max_tokens=1024,
@ -532,7 +528,7 @@ RAW_QUESTION: Final = {"type": "input_text", "text": "q", "prompt_cache_breakpoi
def test_raw_prompt_cache_breakpoints_reach_a_deployment_unless_drop_params_drops_them_for_a_model_that_takes_none(
model: str, extra_kwargs: dict[str, object], log_output: object, question: dict[str, object]
) -> None:
responses_kwargs: Final = _build_responses_kwargs(
responses_kwargs: Final = _build_responses_request(
model=model,
extra_kwargs=extra_kwargs,
max_tokens=1024,
@ -558,7 +554,7 @@ def test_raw_prompt_cache_breakpoints_reach_a_deployment_unless_drop_params_drop
def _gpt_5_6_input_for_a_cached_block(drop_params: bool, raw_mark: Mapping[str, object]) -> object:
responses_kwargs: Final = _build_responses_kwargs(
responses_kwargs: Final = _build_responses_request(
model="openai/gpt-5.6",
extra_kwargs={**OPENAI_API, "drop_params": drop_params},
max_tokens=1024,
@ -589,10 +585,10 @@ def test_a_malformed_raw_marker_beside_cache_control_gives_way_to_it_only_under_
def test_client_cache_control_changes_nothing_for_a_deployment_that_does_not_read_prompt_cache_breakpoint(
model: str, extra_kwargs: dict[str, object]
) -> None:
marked: Final = _build_responses_kwargs(
marked: Final = _build_responses_request(
model=model, extra_kwargs=extra_kwargs, **_request_with_client_cache_marks(EPHEMERAL)
)
unmarked: Final = _build_responses_kwargs(
unmarked: Final = _build_responses_request(
model=model, extra_kwargs=extra_kwargs, **_request_with_client_cache_marks(None)
)
@ -623,7 +619,7 @@ def _tool_result_turn_with(text_block: Mapping[str, object]) -> list[dict[str, o
@pytest.mark.parametrize("text_block", MALFORMED_TEXT_BLOCKS)
def test_malformed_tool_result_text_answers_400_naming_the_block(text_block: Mapping[str, object]) -> None:
with pytest.raises(litellm.BadRequestError) as rejected:
_build_responses_kwargs(
_build_responses_request(
model="openai/gpt-5.6",
extra_kwargs=OPENAI_API,
max_tokens=1024,
@ -650,7 +646,7 @@ def test_many_malformed_tool_result_texts_answer_one_bounded_error() -> None:
]
with pytest.raises(litellm.BadRequestError) as rejected:
_build_responses_kwargs(model="openai/gpt-5.6", extra_kwargs=OPENAI_API, max_tokens=1024, messages=messages)
_build_responses_request(model="openai/gpt-5.6", extra_kwargs=OPENAI_API, max_tokens=1024, messages=messages)
assert rejected.value.status_code == 400
assert f"tool_result {tool_use_id} content.0.text: Input should be a valid string" in rejected.value.message
@ -659,7 +655,7 @@ def test_many_malformed_tool_result_texts_answer_one_bounded_error() -> None:
@pytest.mark.parametrize("text_block", MALFORMED_TEXT_BLOCKS)
def test_drop_params_drops_a_malformed_tool_result_text(text_block: Mapping[str, object]) -> None:
responses_kwargs: Final = _build_responses_kwargs(
responses_kwargs: Final = _build_responses_request(
model="openai/gpt-5.6",
extra_kwargs={**OPENAI_API, "drop_params": True},
max_tokens=1024,
@ -672,7 +668,7 @@ def test_drop_params_drops_a_malformed_tool_result_text(text_block: Mapping[str,
def test_tool_result_text_block_without_text_still_reaches_the_model_as_empty_text() -> None:
responses_kwargs: Final = _build_responses_kwargs(
responses_kwargs: Final = _build_responses_request(
model="openai/gpt-5.6",
extra_kwargs=OPENAI_API,
max_tokens=1024,
@ -1084,3 +1080,344 @@ async def test_the_sync_path_reports_its_edits_on_the_streamed_message_delta(
]
message_delta: Final = next(event for event in events if event["type"] == "message_delta")
assert message_delta["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2
SEARCH_MODEL: Final = "gpt-bridge-search-cost"
SEARCH_RATES: Final = {"input_cost_per_token": 2e-06, "output_cost_per_token": 1e-05}
SEARCH_COST_PER_QUERY: Final = 0.03
SEARCH_RESPONSE: Final = {
**REASONING_RESPONSE,
"id": "resp_bridge_search",
"model": SEARCH_MODEL,
"output": [
*(
{
"type": "web_search_call",
"id": f"ws_{i}",
"status": "completed",
"action": {"type": "search", "query": "q"},
}
for i in range(3)
),
REASONING_RESPONSE["output"][1],
],
"usage": {"input_tokens": 100, "output_tokens": 20, "total_tokens": 120},
}
@pytest.mark.asyncio
async def test_served_bridge_response_is_billed_for_each_web_search_call_the_provider_made(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
monkeypatch.setenv("OPENAI_API_KEY", "sk-bridge-search-test")
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
monkeypatch.setitem(
litellm.model_cost,
SEARCH_MODEL,
{
"litellm_provider": "openai",
"mode": "responses",
"search_context_cost_per_query": {
f"search_context_size_{size}": SEARCH_COST_PER_QUERY for size in ("low", "medium", "high")
},
**SEARCH_RATES,
},
)
litellm.in_memory_llm_clients_cache.flush_cache()
respx_mock.post("https://api.openai.com/v1/responses").respond(status_code=200, json=SEARCH_RESPONSE)
logging_obj: Final = Logging(
model=SEARCH_MODEL,
messages=MESSAGES,
stream=False,
call_type="anthropic_messages",
start_time=datetime.datetime.now(datetime.timezone.utc),
litellm_call_id="bridge-search-0000-4000-8000-000000000001",
function_id="1234",
)
served: Final = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
max_tokens=412,
messages=MESSAGES,
model=f"openai/{SEARCH_MODEL}",
custom_llm_provider="openai",
tools=[{"type": "web_search_20250305", "name": "web_search"}],
litellm_logging_obj=logging_obj,
)
assert isinstance(served, dict)
cost: Final = ProxyBaseLLMRequestProcessing._response_cost_from_logging_obj(
response=dict(served), logging_obj=logging_obj
)
expected_tokens: Final = 100 * SEARCH_RATES["input_cost_per_token"] + 20 * SEARCH_RATES["output_cost_per_token"]
assert cost == pytest.approx(expected_tokens + 3 * SEARCH_COST_PER_QUERY)
assert logging_obj.cost_breakdown is not None
assert logging_obj.cost_breakdown["tool_usage_cost"] == pytest.approx(3 * SEARCH_COST_PER_QUERY)
def _messages_logging_obj(stream: bool) -> Logging:
return Logging(
model="gpt-5.6-luna",
messages=MESSAGES,
stream=stream,
call_type="anthropic_messages",
start_time=datetime.datetime.now(datetime.timezone.utc),
litellm_call_id="6825beef-0000-4000-8000-000000000004",
function_id="1234",
)
def _untyped_bridge() -> Callable[..., Coroutine[object, object, object]]:
return LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler
async def _logged_bridge_call(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, upstream_response: httpx.Response, **extra: object
) -> tuple[object, respx.Route]:
monkeypatch.setenv("OPENAI_API_KEY", "sk-native-test")
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
litellm.in_memory_llm_clients_cache.flush_cache()
upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=upstream_response)
response: Final = await _untyped_bridge()(
max_tokens=1024,
messages=MESSAGES,
model="openai/gpt-5.6-luna",
custom_llm_provider="openai",
litellm_logging_obj=_messages_logging_obj(stream=False),
**extra,
)
return response, upstream
@pytest.mark.asyncio
async def test_bridge_answers_a_non_stream_request_with_a_message_even_when_the_caller_asked_for_stream_replay(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
response, upstream = await _logged_bridge_call(
respx_mock,
monkeypatch,
httpx.Response(200, json=TOOL_CALL_RESPONSE),
**{key: True for key in CONVERTED_STREAM_KEYS},
)
assert isinstance(response, dict) and response["type"] == "message"
assert json.loads(upstream.calls.last.request.content).get("stream") is not True
@pytest.mark.asyncio
async def test_bridge_sends_the_deployment_prompt_cache_options_upstream(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
_, upstream = await _logged_bridge_call(
respx_mock,
monkeypatch,
httpx.Response(200, json=TOOL_CALL_RESPONSE),
prompt_cache_options={"mode": "explicit"},
)
assert json.loads(upstream.calls.last.request.content)["prompt_cache_options"] == {"mode": "explicit"}
@pytest.mark.asyncio
async def test_bridge_raises_the_litellm_rate_limit_error_for_an_upstream_429(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
with pytest.raises(litellm.RateLimitError):
await _logged_bridge_call(
respx_mock,
monkeypatch,
httpx.Response(429, json={"error": {"message": "slow down", "type": "rate_limit_error"}}),
)
class _TemplatePromptManager(CustomPromptManagement):
async def async_get_chat_completion_prompt(
self,
model: str,
messages: list[AllMessageValues],
non_default_params: dict[str, object],
prompt_id: str | None,
prompt_variables: dict[str, object] | None,
dynamic_callback_params: StandardCallbackDynamicParams,
litellm_logging_obj: Logging,
prompt_spec: PromptSpec | None = None,
tools: list[dict[str, object]] | None = None,
prompt_label: str | None = None,
prompt_version: int | None = None,
ignore_prompt_manager_model: bool | None = False,
ignore_prompt_manager_optional_params: bool | None = False,
) -> tuple[str, list[AllMessageValues], dict[str, object]]:
return model, [{"role": "user", "content": "from the template"}], non_default_params
@pytest.mark.asyncio
async def test_bridge_applies_the_prompt_template_named_by_prompt_id(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
monkeypatch.setattr(litellm, "callbacks", [_TemplatePromptManager()])
_, upstream = await _logged_bridge_call(
respx_mock, monkeypatch, httpx.Response(200, json=TOOL_CALL_RESPONSE), prompt_id="bridge_template"
)
assert "from the template" in upstream.calls.last.request.content.decode()
def _logged_sync_bridge_call(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, upstream_response: httpx.Response
) -> tuple[object, respx.Route]:
monkeypatch.setenv("OPENAI_API_KEY", "sk-native-test")
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
litellm.in_memory_llm_clients_cache.flush_cache()
upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=upstream_response)
response: Final = LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
max_tokens=1024,
messages=MESSAGES,
model="openai/gpt-5.6-luna",
custom_llm_provider="openai",
litellm_logging_obj=_messages_logging_obj(stream=False),
)
return response, upstream
def test_sync_bridge_answers_with_a_message_from_one_upstream_call(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
response, upstream = _logged_sync_bridge_call(respx_mock, monkeypatch, httpx.Response(200, json=TOOL_CALL_RESPONSE))
assert isinstance(response, dict) and response["type"] == "message"
assert [block["id"] for block in response["content"] if block["type"] == "tool_use"] == ["call_curl"]
assert upstream.call_count == 1
def test_sync_bridge_raises_the_litellm_rate_limit_error_for_an_upstream_429(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
with pytest.raises(litellm.RateLimitError):
_logged_sync_bridge_call(
respx_mock,
monkeypatch,
httpx.Response(429, json={"error": {"message": "slow down", "type": "rate_limit_error"}}),
)
class _SuccessRowRecorder(CustomLogger):
call_types: tuple[str, ...] = ()
async def async_log_success_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
self.call_types = (*self.call_types, str(kwargs.get("call_type")))
_PARKED_COMPLETION: Final = TypeAdapter(tuple[object])
async def _release(parked: Coroutine[object, object, object]) -> None:
await parked
async def _settled_rows(recorder: _SuccessRowRecorder) -> tuple[str, ...]:
for _ in range(200):
if recorder.call_types:
break
await asyncio.sleep(0.01)
await asyncio.sleep(0.05)
return recorder.call_types
async def _bridged_stream(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, logging_obj: Logging
) -> tuple[AnthropicResponsesSSEByteStream, _SuccessRowRecorder]:
monkeypatch.setenv("OPENAI_API_KEY", "sk-native-test")
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
litellm.in_memory_llm_clients_cache.flush_cache()
recorder: Final = _SuccessRowRecorder()
monkeypatch.setattr(litellm, "_async_success_callback", [recorder])
respx_mock.post("https://api.openai.com/v1/responses").mock(
return_value=httpx.Response(200, headers={"Content-Type": "text/event-stream"}, content=RESPONSES_SSE_BODY)
)
stream: Final = await _untyped_bridge()(
max_tokens=1024,
messages=MESSAGES,
model="openai/gpt-5.6-luna",
stream=True,
custom_llm_provider="openai",
litellm_logging_obj=logging_obj,
)
assert isinstance(stream, AnthropicResponsesSSEByteStream)
return stream, recorder
@pytest.mark.asyncio
async def test_bridged_stream_opens_with_the_cost_breakdown_headers(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
from litellm.proxy.common_request_processing import cost_headers
logging_obj: Final = _messages_logging_obj(stream=True)
stream, _ = await _bridged_stream(respx_mock, monkeypatch, logging_obj)
headers: Final = cost_headers(None, logging_obj)
await stream.aclose()
assert headers["x-litellm-response-cost-output"] == "0.0"
assert headers["x-litellm-response-cost-discount-amount"] == "0.0"
assert headers["x-litellm-response-cost-margin-amount"] == "0.0"
async def _read_through_message_stop(stream: AsyncIterator[bytes]) -> None:
async for frame in stream:
if frame.startswith(b"event: message_stop"):
return
@pytest.mark.asyncio
async def test_bridged_stream_logs_one_row_when_the_client_stream_ends(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
stream, recorder = await _bridged_stream(respx_mock, monkeypatch, _messages_logging_obj(stream=True))
await _read_through_message_stop(stream)
rows_while_the_client_stream_is_open: Final = await _settled_rows(recorder)
remaining: Final[list[bytes]] = [frame async for frame in stream]
assert rows_while_the_client_stream_is_open == ()
assert remaining == []
assert await _settled_rows(recorder) == ("anthropic_messages",)
@pytest.mark.asyncio
async def test_bridged_stream_still_logs_when_the_client_leaves_after_message_stop(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
stream, recorder = await _bridged_stream(respx_mock, monkeypatch, _messages_logging_obj(stream=True))
await _read_through_message_stop(stream)
await stream.aclose()
assert await _settled_rows(recorder) == ("anthropic_messages",)
@pytest.mark.asyncio
async def test_bridged_stream_holds_its_row_for_the_proxy_end_of_stream_guardrails(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
logging_obj: Final = _messages_logging_obj(stream=True)
async def release(*held: object) -> None:
return None
setattr(logging_obj, "_on_deferred_stream_complete", release) # noqa: B010 # the proxy arms this slot dynamically
stream, recorder = await _bridged_stream(respx_mock, monkeypatch, logging_obj)
frames: Final = [frame async for frame in stream]
rows_before_release: Final = await _settled_rows(recorder)
held: Final[object] = getattr(logging_obj, "_deferred_stream_complete_args", None)
assert frames[-1].startswith(b"event: message_stop")
assert rows_before_release == ()
(parked,) = _PARKED_COMPLETION.validate_python(held)
assert asyncio.iscoroutine(parked)
await _release(parked)
assert recorder.call_types == ("anthropic_messages",)

View file

@ -8,7 +8,7 @@ import datetime
import json
import os
import sys
from collections.abc import AsyncIterator, Awaitable, Callable
from collections.abc import AsyncIterator, Awaitable, Callable, Mapping
from types import SimpleNamespace
import anyio
@ -18,10 +18,12 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.
import litellm
from litellm.exceptions import MidStreamFallbackError
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
from litellm.litellm_core_utils.prompt_templates.common_utils import (
encrypted_reasoning_signature,
)
from litellm.llms.anthropic.messages.held_stream_completion import HeldStreamCompletion
from litellm.llms.anthropic.messages.native.streaming_iterator import aclose_superseded_attempt
from litellm.llms.anthropic.messages.native.utils import INCOMPLETE_STREAM_ERROR_MESSAGE
from litellm.llms.anthropic.messages.responses_adapters.streaming_iterator import (
@ -818,3 +820,56 @@ class TestUpstreamFailureReachesTheProxyFailureHook:
asyncio.run(_disconnect_mid_hook())
assert len(finished) == 1
class _AsyncSuccessRecorder(CustomLogger):
calls: tuple[object, ...] = ()
async def async_log_success_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
self.calls = (*self.calls, response_obj)
def test_a_stream_that_fails_after_the_executor_completed_logs_no_success_row():
recorder = _AsyncSuccessRecorder()
logging_obj = LitellmLogging(
model="m",
messages=[{"role": "user", "content": "hi"}],
stream=True,
call_type="anthropic_messages",
start_time=datetime.datetime.now(),
litellm_call_id="fails-after-complete",
function_id="fails-after-complete",
dynamic_async_success_callbacks=[recorder],
)
held = HeldStreamCompletion(logging_obj)
completed = ResponsesAPIResponse(
id="resp_1",
created_at=1,
status="completed",
model="m",
output=[],
parallel_tool_calls=True,
tool_choice="auto",
tools=[],
)
async def _gen() -> AsyncIterator[dict[str, object]]:
yield {"type": "response.created"}
held.take(completed)
yield {"type": "response.completed", "response": completed}
raise ConnectionResetError("Response payload is not completed")
async def _run() -> list[bytes]:
wrapper = AnthropicResponsesStreamWrapper(
responses_stream=_gen(), model="m", litellm_logging_obj=logging_obj, held_completion=held
)
frames: list[bytes] = [frame async for frame in wrapper.async_anthropic_sse_wrapper()]
await asyncio.sleep(0.2)
return frames
frames = asyncio.run(_run())
assert frames[-1].startswith(b"event: error")
assert recorder.calls == ()

View file

@ -2411,16 +2411,3 @@ class TestPromptCacheBreakpointToResponses:
self.MARKED_SCREENSHOT_PART,
]
assert log_output == self.LOG_OUTPUT
def test_prompt_cache_options_forwarded_to_responses_kwargs(self):
from litellm.llms.anthropic.messages.responses_adapters.handler import (
_build_responses_kwargs,
)
kwargs = _build_responses_kwargs(
max_tokens=16,
messages=[{"role": "user", "content": "hi"}],
model="openai/gpt-5.6",
extra_kwargs={"prompt_cache_options": {"mode": "explicit"}},
)
assert kwargs["prompt_cache_options"] == {"mode": "explicit"}

View file

@ -172,7 +172,7 @@ def test_azure_o_series_routing():
assert "stream" not in mock_create.call_args.kwargs["body"]
@patch("litellm.main.azure_o1_chat_completions._get_openai_client")
@patch("litellm.native.chat_providers.handlers.azure_o1_chat_completions._get_openai_client")
def test_openai_o_series_max_retries_0(mock_get_openai_client):
import litellm

View file

@ -583,7 +583,9 @@ def test_azure_openai_responses_bridge():
litellm.turn_on_debug()
with patch.object(litellm, "responses") as mock_responses:
from litellm.responses import bridge_executor
with patch.object(bridge_executor, "native_responses") as mock_responses:
try:
response = completion(
model="azure/responses/test-azure-computer-use-preview",
@ -597,10 +599,10 @@ def test_azure_openai_responses_bridge():
mock_responses.assert_called_once()
assert (
mock_responses.call_args.kwargs["model"]
mock_responses.call_args.args[0]["model"]
== "azure/test-azure-computer-use-preview"
)
assert mock_responses.call_args.kwargs["custom_llm_provider"] == "azure"
assert mock_responses.call_args.args[0]["custom_llm_provider"] == "azure"
def test_azure_with_content_safety_error():

View file

@ -735,7 +735,7 @@ async def test_ensure_initialize_azure_sdk_client_always_used_azure_text(call_ty
# Get appropriate input for this call type
input_kwarg = test_inputs.get(call_type.value, {})
patch_target = "litellm.main.azure_text_completions.initialize_azure_sdk_client"
patch_target = "litellm.native.chat_providers.handlers.azure_text_completions.initialize_azure_sdk_client"
# Mock the initialize_azure_sdk_client function
with patch(patch_target) as mock_init_azure:

View file

@ -21,7 +21,7 @@ def test_main_azure_ai_claude_completion_passes_timeout_to_azure_anthropic_handl
captured.update(kwargs)
return ModelResponse()
with patch("litellm.main.azure_anthropic_chat_completions") as mock_azure_anthropic:
with patch("litellm.native.chat_providers.azure.azure_anthropic_chat_completions") as mock_azure_anthropic:
mock_azure_anthropic.completion = MagicMock(
side_effect=fake_azure_anthropic_completion
)

View file

@ -686,7 +686,7 @@ class TestGDCGeminiConfig:
class TestCompleteGDC:
@patch("litellm.main.base_llm_http_handler.completion")
def test_complete_gdc_resolves_key_and_base(self, mock_completion, monkeypatch):
from litellm.main import gdc_transformation
from litellm.native.chat_providers.handlers import gdc_transformation
mock_completion.return_value = MagicMock()
monkeypatch.setattr(litellm, "gdc_key", "resolved-key", raising=False)

View file

@ -28,7 +28,7 @@ def test_gemini_completion_no_api_key():
del os.environ[key]
# Test without mock_response to ensure actual API key validation
with pytest.raises(Exception, match='in _complete_vertex_ai_beta') as exc_info:
with pytest.raises(Exception, match='in complete_vertex_ai_beta') as exc_info:
completion(
model="gemini/gemini-1.5-flash",
messages=[{"role": "user", "content": "Test message"}],
@ -60,7 +60,7 @@ def test_gemini_completion_no_api_key_with_mock():
with patch("litellm.get_secret") as mock_get_secret:
mock_get_secret.return_value = None
with pytest.raises(Exception, match='in _complete_vertex_ai_beta') as exc_info:
with pytest.raises(Exception, match='in complete_vertex_ai_beta') as exc_info:
completion(
model="gemini/gemini-1.5-flash",
messages=[{"role": "user", "content": "Test message"}],

View file

@ -668,7 +668,7 @@ def test_e2e_completion_per_user_credential(per_user_credential):
def test_e2e_completion_per_user_session_token_wins_over_caller_authorization(per_user_credential):
"""The session token reaches the wire even when the caller passes their own
Authorization in extra_headers (chat dispatches through _complete_custom_openai)."""
Authorization in extra_headers (chat dispatches through complete_custom_openai)."""
with (
github_http(_exchange_responder()),
_no_authenticator(),

View file

@ -1934,7 +1934,9 @@ def test_responses_gpt54_with_xhigh_reasoning():
Ensure chat->responses bridge sends the correct request payload for
openai/responses/gpt-5.4 with reasoning_effort="xhigh".
"""
with patch("litellm.responses") as mock_responses:
from litellm.responses import bridge_executor
with patch.object(bridge_executor, "native_responses") as mock_responses:
mock_responses.side_effect = RuntimeError("stop_after_request_build")
with pytest.raises(litellm.APIConnectionError):
@ -1946,7 +1948,7 @@ def test_responses_gpt54_with_xhigh_reasoning():
)
mock_responses.assert_called_once()
request_body = mock_responses.call_args.kwargs
request_body = mock_responses.call_args.args[0]
assert request_body["model"] == "openai/gpt-5.4"

View file

@ -8515,8 +8515,11 @@ async def test_claude_tool_use_with_gemini():
return chunk
raise StopAsyncIteration
with patch("litellm.acompletion", new_callable=AsyncMock) as mock_acompletion:
mock_acompletion.return_value = MockAsyncStream()
with patch(
"litellm.llms.anthropic.messages.adapters.handler.aexecute_planned_chat",
new_callable=AsyncMock,
) as mock_chat:
mock_chat.return_value = MockAsyncStream()
response = await litellm.anthropic.messages.acreate(
messages=[

View file

@ -1545,9 +1545,9 @@ def test_system_prompt_only_adds_blank_user_message():
@pytest.fixture
def _cached_vertex_credentials(monkeypatch: pytest.MonkeyPatch) -> str:
from litellm.main import (
from litellm.main import vertex_embedding
from litellm.native.chat_providers.handlers import (
vertex_chat_completion,
vertex_embedding,
vertex_model_garden_chat_completion,
vertex_partner_models_chat_completion,
)

View file

@ -15,6 +15,7 @@ import pytest
import litellm
from litellm._uuid import uuid
from litellm.native.chat_providers.handlers import vertex_partner_models_chat_completion
from litellm.litellm_core_utils.prompt_templates.factory import (
anthropic_messages_pt,
convert_to_anthropic_tool_result,
@ -406,7 +407,7 @@ async def test_vertex_ai_anthropic_async_completion_inlines_https_images_off_the
with (
patch.dict(sys.modules, {"vertexai": vertexai}),
patch.object( # test-quality-ok: litellm.acompletion has no seam for Vertex token minting
litellm.main.vertex_partner_models_chat_completion,
vertex_partner_models_chat_completion,
"_ensure_access_token",
return_value=("token", "test-project"),
),

View file

@ -585,7 +585,7 @@ async def test_vertex_ai_anthropic_prompt_caching_does_not_add_beta_header(
monkeypatch: pytest.MonkeyPatch,
sync_mode: bool,
) -> None:
from litellm.main import vertex_partner_models_chat_completion
from litellm.native.chat_providers.handlers import vertex_partner_models_chat_completion
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setitem(

View file

@ -1320,7 +1320,7 @@ def _gemma_cached_access_token():
"""Serve a fake token from the handler's credential cache so no auth round-trip runs."""
from types import SimpleNamespace
from litellm.main import vertex_gemma_chat_completion
from litellm.native.chat_providers.handlers import vertex_gemma_chat_completion
cache = vertex_gemma_chat_completion._credentials_project_mapping
key = (_FAKE_GEMMA_CREDENTIALS, "test")

View file

View file

@ -0,0 +1,166 @@
import json
from datetime import datetime
from typing import Final
import httpx
import pytest
import respx
import litellm
from litellm.exceptions import JSONSchemaValidationError
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.native.chat_plan import plan_chat
API_BASE: Final = "https://bridge-parity.invalid/v1"
SCHEMA: Final = {
"type": "object",
"properties": {"b": {"type": "integer"}},
"required": ["b"],
"additionalProperties": False,
}
OFF_SCHEMA_TEXT: Final = '{"b": 2, "extra": true}'
_CHAT_REPLY: Final = {
"id": "chatcmpl-1",
"object": "chat.completion",
"created": 1,
"model": "m",
"choices": [{"index": 0, "message": {"role": "assistant", "content": OFF_SCHEMA_TEXT}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
}
def _router(litellm_params: dict[str, object], model_info: dict[str, object] | None = None) -> litellm.Router:
return litellm.Router(
model_list=[
{
"model_name": "m",
"litellm_params": {"api_base": API_BASE, "api_key": "k", **litellm_params},
**({"model_info": model_info} if model_info is not None else {}),
}
],
num_retries=0,
)
@pytest.fixture(autouse=True)
def _offline(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
@pytest.mark.parametrize(
("litellm_params", "model_info"),
[
({"model": "fireworks_ai/accounts/fireworks/models/llama-v3p1-8b-instruct"}, None),
({"model": "openai/gpt-4.1", "use_chat_completions_api": True}, None),
],
)
@pytest.mark.asyncio
async def test_a_messages_request_bridged_to_chat_validates_its_json_schema(
litellm_params: dict[str, object],
model_info: dict[str, object] | None,
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(litellm, "enable_json_schema_validation", True)
respx_mock.post(f"{API_BASE}/chat/completions").mock(return_value=httpx.Response(200, json=_CHAT_REPLY))
with pytest.raises(JSONSchemaValidationError):
await _router(litellm_params, model_info).aanthropic_messages(
model="m",
max_tokens=32,
messages=[{"role": "user", "content": "hi"}],
output_format={"type": "json_schema", "schema": SCHEMA},
)
@pytest.mark.asyncio
async def test_a_responses_request_bridged_to_chat_validates_its_json_schema(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
monkeypatch.setattr(litellm, "enable_json_schema_validation", True)
respx_mock.post(f"{API_BASE}/chat/completions").mock(return_value=httpx.Response(200, json=_CHAT_REPLY))
with pytest.raises(JSONSchemaValidationError):
await _router({"model": "deepseek/deepseek-reasoner"}, {"mode": "responses"}).aresponses(
model="m",
input="hi",
text={"format": {"type": "json_schema", "name": "x", "strict": True, "schema": SCHEMA}},
)
@pytest.mark.asyncio
async def test_a_provider_failure_on_the_messages_to_chat_bridge_is_mapped(respx_mock: respx.MockRouter):
respx_mock.post(f"{API_BASE}/chat/completions").mock(return_value=httpx.Response(200, content=b"not json"))
with pytest.raises(litellm.exceptions.APIConnectionError):
await _router({"model": "groq/llama-3.1-8b-instant"}, {"mode": "responses"}).aanthropic_messages(
model="m", max_tokens=32, messages=[{"role": "user", "content": "hi"}]
)
@pytest.mark.asyncio
async def test_a_responses_request_on_a_chat_api_deployment_stays_on_chat_despite_mode_responses(
respx_mock: respx.MockRouter,
):
chat: Final = respx_mock.post(f"{API_BASE}/chat/completions").mock(
return_value=httpx.Response(200, json=_CHAT_REPLY)
)
await _router({"model": "openai/gpt-4.1", "use_chat_completions_api": True}, {"mode": "responses"}).aresponses(
model="m", input="hi"
)
assert chat.call_count == 1
@pytest.mark.asyncio
async def test_the_skip_bridge_flag_is_never_sent_to_the_provider(respx_mock: respx.MockRouter):
chat: Final = respx_mock.post(f"{API_BASE}/chat/completions").mock(
return_value=httpx.Response(200, json=_CHAT_REPLY)
)
await _router({"model": "openai/gpt-4.1"}).acompletion(
model="m", messages=[{"role": "user", "content": "hi"}], _skip_responses_api_bridge=True
)
assert "_skip_responses_api_bridge" not in json.loads(chat.calls.last.request.content)
@pytest.mark.asyncio
async def test_the_skip_bridge_flag_keeps_a_mode_responses_deployment_on_chat(respx_mock: respx.MockRouter):
chat: Final = respx_mock.post(f"{API_BASE}/chat/completions").mock(
return_value=httpx.Response(200, json=_CHAT_REPLY)
)
await _router({"model": "openai/gpt-4.1"}, {"mode": "responses"}).acompletion(
model="m", messages=[{"role": "user", "content": "hi"}], _skip_responses_api_bridge=True
)
assert chat.call_count == 1
def _bridged_chat_logging() -> Logging:
return Logging(
model="gpt-4.1",
messages=[{"role": "user", "content": "hi"}],
stream=False,
call_type="aresponses",
start_time=datetime.now(),
litellm_call_id="planned-chat-cache",
function_id="planned-chat-cache",
)
@pytest.mark.parametrize(("response_cache", "planned"), [(None, True), (litellm.Cache(type="local"), False)])
def test_a_bridged_chat_call_goes_through_acompletion_when_the_response_cache_is_on(
response_cache: litellm.Cache | None, planned: bool, monkeypatch: pytest.MonkeyPatch
):
monkeypatch.setattr(litellm, "cache", response_cache)
plan: Final = plan_chat(
{"model": "openai/gpt-4.1", "messages": [{"role": "user", "content": "hi"}], "api_key": "k"},
_bridged_chat_logging(),
)
assert (plan is not None) is planned

View file

@ -0,0 +1,288 @@
import asyncio
import json
from collections.abc import AsyncIterable, Callable, Coroutine, Mapping, Sequence
from typing import Final, cast
import httpx
import pytest
import respx
from pydantic import TypeAdapter
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.llms.openai_like.messages.transformation import OpenAILikeAnthropicMessagesConfig
from litellm.messages.dispatch import anthropic_messages
from litellm.native.route_plan import (
NativeChat,
NativeMessages,
NativeResponses,
RoutePlan,
plan_chat_route,
plan_messages_route,
plan_responses_route,
)
MESSAGES_PASSTHROUGH: dict[str, object] = {"supported_endpoints": ["/v1/messages"]}
def _plan(
model: str,
provider: str,
*,
model_info: object = None,
compaction_requested: bool = False,
force_chat_completions: bool = False,
) -> RoutePlan:
return plan_messages_route(
requested_model=f"{provider}/{model}",
resolved_model=model,
custom_llm_provider=provider,
model_info=model_info,
api_base=None,
compaction_requested=compaction_requested,
force_chat_completions=force_chat_completions,
)
def test_a_provider_that_speaks_messages_gets_its_own_messages_config():
plan = _plan("claude-sonnet-4-5", "anthropic")
assert isinstance(plan, NativeMessages)
assert isinstance(plan.config, AnthropicMessagesConfig)
@pytest.mark.parametrize(
("model", "provider", "kwargs", "expected"),
[
("gpt-5.5", "openai", {}, NativeResponses()),
("gpt-5.5", "openai", {"force_chat_completions": True}, NativeChat()),
("gpt-5.5", "openai", {"compaction_requested": True}, NativeChat()),
("my-model", "hosted_vllm", {}, NativeChat()),
],
)
def test_messages_without_a_native_config_bridge_to_responses_only_for_openai(
model: str, provider: str, kwargs: dict[str, bool], expected: RoutePlan
):
assert _plan(model, provider, **kwargs) == expected
_MODEL_COST: Final = TypeAdapter(dict[str, object])
def _isolate_model_cost(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "model_cost", _MODEL_COST.validate_python(litellm.model_cost))
@pytest.mark.parametrize("cache_control_ttl", [True, False])
def test_a_deployment_that_opts_into_messages_passthrough_skips_translation(cache_control_ttl: bool):
plan = _plan("my-model", "hosted_vllm", model_info={**MESSAGES_PASSTHROUGH, "cache_control_ttl": cache_control_ttl})
assert isinstance(plan, NativeMessages)
assert isinstance(plan.config, OpenAILikeAnthropicMessagesConfig)
assert plan.config.supports_cache_control_ttl() is cache_control_ttl
@pytest.mark.parametrize(
("has_config", "use_chat_completions_api", "expected"),
[
(True, False, NativeResponses()),
(True, True, NativeChat()),
(False, False, NativeChat()),
(False, True, NativeChat()),
],
)
def test_responses_reach_chat_without_a_provider_config_or_when_asked(
has_config: bool, use_chat_completions_api: bool, expected: RoutePlan
):
config = OpenAIResponsesAPIConfig() if has_config else None
assert plan_responses_route(config, use_chat_completions_api) == expected
@pytest.mark.parametrize(
("has_config", "expected"),
[
(True, NativeResponses()),
(False, NativeChat()),
],
)
def test_chat_bridges_to_responses_only_when_the_provider_serves_it(has_config: bool, expected: RoutePlan):
assert plan_chat_route(OpenAIResponsesAPIConfig() if has_config else None) == expected
CHAT_BASE: Final = "http://chat-upstream.test/v1"
RESPONSES_BASE: Final = "http://responses-upstream.test/v1"
CHAT_MODEL: Final = "hosted_vllm/served-model"
RESPONSES_ONLY_MODEL: Final = "openai/bridge-once-model"
USER_TURN: Final = ({"role": "user", "content": "hi"},)
_CHAT_REPLY: Final[Mapping[str, object]] = {
"id": "chatcmpl-once",
"object": "chat.completion",
"created": 1,
"model": "served-model",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "once"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
}
_CHAT_CHUNKS: Final[tuple[Mapping[str, object], ...]] = (
{
"id": "chatcmpl-once",
"object": "chat.completion.chunk",
"created": 1,
"model": "served-model",
"choices": [{"index": 0, "delta": {"role": "assistant", "content": "once"}, "finish_reason": None}],
},
{
"id": "chatcmpl-once",
"object": "chat.completion.chunk",
"created": 1,
"model": "served-model",
"choices": [{"index": 0, "delta": dict[str, object](), "finish_reason": "stop"}],
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
},
)
_OUTPUT_MESSAGE: Final[Mapping[str, object]] = {
"type": "message",
"id": "msg_once",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "once", "annotations": list[object]()}],
}
_RESPONSES_REPLY: Final[Mapping[str, object]] = {
"id": "resp_once",
"object": "response",
"created_at": 1,
"model": "bridge-once-model",
"status": "completed",
"output": [_OUTPUT_MESSAGE],
"parallel_tool_calls": True,
"tool_choice": "auto",
"tools": list[object](),
"usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
}
_RESPONSES_EVENTS: Final[tuple[Mapping[str, object], ...]] = (
{"type": "response.created", "sequence_number": 0, "response": {**_RESPONSES_REPLY, "status": "in_progress"}},
{"type": "response.output_item.added", "sequence_number": 1, "output_index": 0, "item": _OUTPUT_MESSAGE},
{
"type": "response.output_text.delta",
"sequence_number": 2,
"item_id": "msg_once",
"output_index": 0,
"content_index": 0,
"delta": "once",
},
{"type": "response.output_item.done", "sequence_number": 3, "output_index": 0, "item": _OUTPUT_MESSAGE},
{"type": "response.completed", "sequence_number": 4, "response": _RESPONSES_REPLY},
)
def _sse(events: Sequence[Mapping[str, object]], *, done: bool) -> bytes:
frames: Final = "".join(f"data: {json.dumps(event)}\n\n" for event in events)
return (frames + ("data: [DONE]\n\n" if done else "")).encode()
class _CallCounter(CustomLogger):
pre_call: int = 0
success: int = 0
async def async_pre_call_deployment_hook(self, kwargs: Mapping[str, object], call_type: object) -> None:
self.pre_call += 1
async def async_log_success_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
self.success += 1
async def _messages(model: str, api_base: str, stream: bool) -> object:
return await anthropic_messages(
model=model, api_base=api_base, api_key="sk-x", max_tokens=10, messages=list(USER_TURN), stream=stream
)
async def _messages_to_chat(stream: bool) -> object:
return await _messages(CHAT_MODEL, CHAT_BASE, stream)
async def _messages_to_responses(stream: bool) -> object:
return await _messages("openai/gpt-5.6-luna", RESPONSES_BASE, stream)
async def _chat_to_responses(stream: bool) -> object:
return await litellm.acompletion(
model=RESPONSES_ONLY_MODEL, api_base=RESPONSES_BASE, api_key="sk-x", messages=list(USER_TURN), stream=stream
)
async def _responses_to_chat(stream: bool) -> object:
return await litellm.aresponses(
model="deepseek/served-model", api_base=CHAT_BASE, api_key="sk-x", input="hi", stream=stream
)
async def _drained(reply: object) -> None:
if isinstance(reply, AsyncIterable):
async for _ in cast(AsyncIterable[object], reply): # cast-ok: every bridged stream is drained the same way
pass
async def _settled(counter: _CallCounter) -> _CallCounter:
for _ in range(200):
if counter.success:
break
await asyncio.sleep(0.01)
await asyncio.sleep(0.1)
return counter
_BRIDGES: Final = (
pytest.param(_messages_to_chat, "chat", id="messages_to_chat"),
pytest.param(_messages_to_responses, "responses", id="messages_to_responses"),
pytest.param(_chat_to_responses, "responses", id="chat_to_responses"),
pytest.param(_responses_to_chat, "chat", id="responses_to_chat"),
)
@pytest.mark.parametrize("stream", [False, True])
@pytest.mark.parametrize(("bridge", "upstream_api"), _BRIDGES)
@pytest.mark.asyncio
async def test_a_bridged_request_runs_hooks_logs_and_calls_the_provider_once(
bridge: Callable[[bool], Coroutine[object, object, object]],
upstream_api: str,
stream: bool,
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
):
counter: Final = _CallCounter()
_isolate_model_cost(monkeypatch)
monkeypatch.setattr(litellm, "callbacks", [counter])
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
litellm.register_model(
{RESPONSES_ONLY_MODEL: {"litellm_provider": "openai", "mode": "responses", "max_tokens": 100}}
)
upstream: Final = (
respx_mock.post(f"{CHAT_BASE}/chat/completions").mock(
return_value=httpx.Response(
200, headers={"content-type": "text/event-stream"}, content=_sse(_CHAT_CHUNKS, done=True)
)
if stream
else httpx.Response(200, json=_CHAT_REPLY)
)
if upstream_api == "chat"
else respx_mock.post(f"{RESPONSES_BASE}/responses").mock(
return_value=httpx.Response(
200, headers={"content-type": "text/event-stream"}, content=_sse(_RESPONSES_EVENTS, done=False)
)
if stream
else httpx.Response(200, json=_RESPONSES_REPLY)
)
)
await _drained(await bridge(stream))
settled: Final = await _settled(counter)
assert upstream.call_count == 1
assert (settled.pre_call, settled.success) == (1, 1)

View file

@ -20,7 +20,7 @@ class TestGetFileContentsFromS3:
"""Test suite for S3 config loading functionality."""
@patch("boto3.client")
@patch("litellm.main.bedrock_converse_chat_completion")
@patch("litellm.native.chat_providers.handlers.bedrock_converse_chat_completion")
@patch("yaml.safe_load")
def test_get_file_contents_from_s3_no_temp_file_creation(
self, mock_yaml_load, mock_bedrock, mock_boto3_client

View file

@ -19,6 +19,7 @@ import logging
from collections.abc import Callable, Coroutine, Mapping
from datetime import datetime
from threading import Event
from functools import partial
from typing import Any, Final, cast
from unittest.mock import AsyncMock, MagicMock, patch
@ -28,7 +29,6 @@ import respx
import litellm
from litellm.caching.caching import DualCache
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.llm_cost_calc.finalization import finalize_cost
from litellm.types.utils import StandardLoggingPayload
@ -59,6 +59,7 @@ def _attach_mock_success_dispatch(mock_logging_obj, async_success_fn):
mock_logging_obj.dispatch_success_handlers = dispatch_success_handlers
mock_logging_obj.async_success_handler = async_success_fn
mock_logging_obj.hold_stream_completion = partial(Logging.hold_stream_completion, mock_logging_obj)
async def _wait_until(condition: Callable[[], bool]) -> None:
@ -593,6 +594,7 @@ class TestDeferredStreamingClosure:
response args on logging_obj instead of calling the closure directly."""
mock_logging_obj = MagicMock()
mock_logging_obj._on_deferred_stream_complete = MagicMock()
mock_logging_obj.hold_stream_completion = partial(Logging.hold_stream_completion, mock_logging_obj)
resp = await litellm.acompletion(
model="gpt-3.5-turbo",
@ -1388,6 +1390,7 @@ class TestResponsesIteratorDeferredLogging:
logging_obj = MagicMock()
logging_obj.dispatch_success_handlers = dispatch_success_handlers
logging_obj.hold_stream_completion = partial(Logging.hold_stream_completion, logging_obj)
return logging_obj, recorded
@pytest.mark.asyncio

View file

@ -1,3 +1,4 @@
import datetime
import json
from collections.abc import Mapping, Sequence
from itertools import chain
@ -13,6 +14,7 @@ from pydantic import BaseModel
import litellm
from litellm import CustomLLM
from litellm.constants import REDACTED_BY_LITELLM
from litellm.integrations.custom_logger import CustomLogger
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.responses.litellm_completion_transformation.handler import (
LiteLLMCompletionTransformationHandler,
@ -92,8 +94,8 @@ async def test_aresponses_forwards_timeout_to_acompletion():
and similar providers, with calls falling back to the provider SDK default
(~600s for Anthropic).
"""
with patch("litellm.acompletion", new_callable=AsyncMock) as mock_acompletion:
mock_acompletion.return_value = ModelResponse(
with patch("litellm.native.chat_plan.aexecute_planned_chat", new_callable=AsyncMock) as mock_chat:
mock_chat.return_value = ModelResponse(
id="id",
created=0,
model="anthropic/claude-sonnet-4-5",
@ -108,8 +110,8 @@ async def test_aresponses_forwards_timeout_to_acompletion():
api_key="sk-ant-fake",
)
assert mock_acompletion.call_count == 1
forwarded_timeout = mock_acompletion.call_args.kwargs.get("timeout")
assert mock_chat.call_count == 1
forwarded_timeout = mock_chat.call_args.args[0].named.get("timeout")
assert forwarded_timeout == 42, (
f"timeout was not forwarded to acompletion (got {forwarded_timeout!r}); "
"this means Router(timeout=N) silently fails for providers on the "
@ -1569,3 +1571,151 @@ async def test_previous_response_id_keeps_the_tools_an_earlier_codex_tool_search
[("tool_result", "toolu_standup"), ("text", None)],
]
assert prisma_client.db.queries == 1
class _DeploymentHookCounter(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.calls = 0
async def async_pre_call_deployment_hook(self, kwargs: Mapping[str, object], call_type: object) -> None:
self.calls += 1
@pytest.mark.asyncio
async def test_bridged_responses_call_runs_deployment_hooks_once(monkeypatch: pytest.MonkeyPatch):
counter: Final = _DeploymentHookCounter()
monkeypatch.setattr(litellm, "callbacks", [counter])
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
with respx.mock(assert_all_called=False) as upstream:
messages_route: Final = upstream.post("http://anthropic.test/v1/messages").mock(
return_value=httpx.Response(
200,
json={
"id": "msg_bridge",
"type": "message",
"role": "assistant",
"model": "claude-bridge",
"content": [{"type": "text", "text": "from upstream"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 10, "output_tokens": 5},
},
)
)
response: Final = await litellm.aresponses(
model="anthropic/claude-bridge", input="hi", api_base="http://anthropic.test", api_key="sk-test"
)
assert response.output[0].content[0].text == "from upstream"
assert messages_route.call_count == 1
assert counter.calls == 1
@pytest.mark.asyncio
async def test_bridged_responses_call_is_billed_like_the_chat_call_for_one_hour_cache_writes_and_web_searches(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
usage: Final = {
"input_tokens": 100,
"output_tokens": 50,
"cache_creation_input_tokens": 2000,
"cache_read_input_tokens": 3000,
"cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2000},
"server_tool_use": {"web_search_requests": 2},
}
upstream_message: Final = {
"id": "msg_bridge_cost",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-5",
"content": [{"type": "text", "text": "searched"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": usage,
}
credentials: Final = {"api_base": "http://anthropic.test", "api_key": "sk-test"}
with respx.mock(assert_all_called=False) as upstream:
upstream.post("http://anthropic.test/v1/messages").mock(return_value=httpx.Response(200, json=upstream_message))
chat: Final = await litellm.acompletion(
model="anthropic/claude-sonnet-4-5", messages=[{"role": "user", "content": "hi"}], **credentials
)
bridged: Final = await litellm.aresponses(model="anthropic/claude-sonnet-4-5", input="hi", **credentials)
chat_cost: Final = chat._hidden_params["response_cost"]
assert isinstance(chat_cost, float) and chat_cost > 0
assert bridged._hidden_params["response_cost"] == pytest.approx(chat_cost)
_ANTHROPIC_MESSAGE: Final = {
"id": "msg_bridge_headers",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-5",
"content": [{"type": "text", "text": "hi"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 10, "output_tokens": 5},
}
@pytest.mark.asyncio
async def test_bridged_responses_call_names_the_provider_request_id_in_its_headers(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
with respx.mock(assert_all_called=False) as upstream:
upstream.post("http://anthropic.test/v1/messages").mock(
return_value=httpx.Response(200, json=_ANTHROPIC_MESSAGE, headers={"x-request-id": "req-upstream-1"})
)
bridged: Final = await litellm.aresponses(
model="anthropic/claude-sonnet-4-5", input="hi", api_base="http://anthropic.test", api_key="sk-test"
)
assert bridged._hidden_params["additional_headers"]["llm_provider-request-id"] == "req-upstream-1"
@pytest.mark.asyncio
async def test_bridged_responses_stream_opens_with_the_cost_breakdown_headers(monkeypatch: pytest.MonkeyPatch):
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.proxy.common_request_processing import cost_headers
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
logging_obj: Final = Logging(
model="anthropic/claude-sonnet-4-5",
messages=[{"role": "user", "content": "hi"}],
stream=True,
call_type="aresponses",
start_time=datetime.datetime.now(datetime.timezone.utc),
litellm_call_id="bridge-stream-headers-0000-4000-8000-000000000001",
function_id="1234",
)
events: Final = (
{"type": "message_start", "message": {**_ANTHROPIC_MESSAGE, "content": [], "stop_reason": None}},
{"type": "message_stop"},
)
sse: Final = "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events)
with respx.mock(assert_all_called=False) as upstream:
upstream.post("http://anthropic.test/v1/messages").mock(
return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, text=sse)
)
stream: Final = await litellm.aresponses(
model="anthropic/claude-sonnet-4-5",
input="hi",
stream=True,
api_base="http://anthropic.test",
api_key="sk-test",
litellm_logging_obj=logging_obj,
)
headers: Final = cost_headers(None, logging_obj)
async for _ in stream:
pass
assert headers["x-litellm-response-cost-output"] == "0.0"
assert headers["x-litellm-response-cost-discount-amount"] == "0.0"
assert headers["x-litellm-response-cost-margin-amount"] == "0.0"

View file

@ -23,16 +23,14 @@ async def test_mock_basic_google_ai_studio_responses_api_with_tools():
choices=[
litellm.utils.Choices(
index=0,
message=litellm.utils.Message(
role="assistant", content="Test response"
),
message=litellm.utils.Message(role="assistant", content="Test response"),
finish_reason="stop",
)
],
)
with patch("litellm.acompletion", new_callable=AsyncMock) as mock_acompletion:
mock_acompletion.return_value = mock_response
with patch("litellm.native.chat_plan.aexecute_planned_chat", new_callable=AsyncMock) as mock_chat:
mock_chat.return_value = mock_response
request_model = "gemini/gemini-2.5-flash"
await litellm.aresponses(
@ -41,11 +39,10 @@ async def test_mock_basic_google_ai_studio_responses_api_with_tools():
tools=[{"type": "web_search_preview", "search_context_size": "low"}],
)
# Verify that acompletion was called
assert mock_acompletion.called
assert mock_chat.called
# Get the call arguments
call_args, call_kwargs = mock_acompletion.call_args
plan = mock_chat.call_args.args[0]
call_kwargs = {**plan.kwargs, **plan.named}
# Verify the expected parameters were passed
print(
@ -65,5 +62,5 @@ async def test_mock_basic_google_ai_studio_responses_api_with_tools():
call_kwargs["messages"][0]["content"]
== "what is the latest version of supabase python package and when was it released?"
)
assert "tools" not in call_kwargs
assert "tool_choice" not in call_kwargs
assert call_kwargs.get("tools") is None
assert call_kwargs.get("tool_choice") is None

View file

@ -1,31 +1,24 @@
"""Regression tests for the responses -> completion fallback bridge guard.
When the Responses API falls back to chat completions (no native responses
config), it must tag the forwarded ``litellm.completion`` / ``litellm.acompletion``
call with ``_skip_responses_api_bridge=True`` so ``completion()`` does not bridge
the request straight back to the Responses API and mutually recurse forever.
Both fallback paths are covered: the sync ``response_api_handler`` (``_is_async``
False) and the async ``async_response_api_handler`` (``_is_async`` True). The
module-level ``litellm.completion`` / ``litellm.acompletion`` are patched to
capture the forwarded kwargs; if the flag-setting line is removed the captured
kwargs lack the flag and these tests fail.
"""
import asyncio
import json
import math
from collections.abc import Mapping, Sequence
from datetime import datetime
from types import SimpleNamespace
from typing import Final
from unittest.mock import patch
import httpx
import pytest
import respx
from pydantic import TypeAdapter
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.responses.litellm_completion_transformation.handler import (
LiteLLMCompletionTransformationHandler,
)
from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD
@ -35,50 +28,6 @@ class _StopForwarding(Exception):
"""Raised by the mocked (a)completion once the forwarded kwargs are captured."""
def test_sync_fallback_tags_skip_responses_api_bridge():
handler = LiteLLMCompletionTransformationHandler()
captured: dict = {}
def fake_completion(**kwargs):
captured.update(kwargs)
raise _StopForwarding()
with patch("litellm.completion", fake_completion):
with pytest.raises(_StopForwarding):
handler.response_api_handler(
model="gpt-4o",
input="hello",
responses_api_request={},
custom_llm_provider="openai",
_is_async=False,
)
assert captured.get("_skip_responses_api_bridge") is True
@pytest.mark.asyncio
async def test_async_fallback_tags_skip_responses_api_bridge():
handler = LiteLLMCompletionTransformationHandler()
captured: dict = {}
async def fake_acompletion(**kwargs):
captured.update(kwargs)
raise _StopForwarding()
with patch("litellm.acompletion", fake_acompletion):
coro = handler.response_api_handler(
model="gpt-4o",
input="hello",
responses_api_request={},
custom_llm_provider="openai",
_is_async=True,
)
with pytest.raises(_StopForwarding):
await coro
assert captured.get("_skip_responses_api_bridge") is True
_CODEX_ADDITIONAL_TOOLS_ITEM = {
"type": "additional_tools",
"id": "at_codex",
@ -303,10 +252,88 @@ async def test_bridged_next_turn_replays_a_turn_stored_under_the_id_the_client_h
)
assert provider.request_body is not None, "the bridged turn never reached the provider"
assert [
(message["role"], json.dumps(message["content"])) for message in provider.request_body["messages"]
] == [
assert [(message["role"], json.dumps(message["content"])) for message in provider.request_body["messages"]] == [
("user", json.dumps([{"type": "text", "text": "My favorite number is 7."}])),
("assistant", json.dumps([{"type": "text", "text": "Noted."}])),
("user", json.dumps([{"type": "text", "text": "Double it"}])),
]
_LOGGED_ROW: Final = TypeAdapter(dict[str, object])
class _SuccessRowRecorder(CustomLogger):
rows: tuple[dict[str, object], ...] = ()
async def async_log_success_event(
self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime
) -> None:
self.rows = (*self.rows, _LOGGED_ROW.validate_python(kwargs["standard_logging_object"]))
@pytest.mark.asyncio
async def test_bridged_responses_stream_logs_one_responses_row_billed_from_provider_usage(
monkeypatch: pytest.MonkeyPatch,
):
recorder: Final = _SuccessRowRecorder()
monkeypatch.setattr(litellm, "callbacks", [recorder])
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
prices: Final = {"input_cost_per_token": 1e-6, "output_cost_per_token": 4e-6, "cache_read_input_token_cost": 1e-7}
litellm.register_model(
{"deepseek/bridge-stream-model": {"litellm_provider": "deepseek", "mode": "chat", "max_tokens": 100, **prices}}
)
chunk: Final[dict[str, object]] = {"id": "chatcmpl-stream", "object": "chat.completion.chunk", "created": 1}
chunks: Final[tuple[dict[str, object], ...]] = (
{**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "streamed"}}]},
{**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]},
{
**chunk,
"choices": [],
"usage": {
"prompt_tokens": 100,
"completion_tokens": 20,
"total_tokens": 120,
"prompt_tokens_details": {"cached_tokens": 40},
},
},
)
sse: Final = ("".join(f"data: {json.dumps(c)}\n\n" for c in chunks) + "data: [DONE]\n\n").encode()
with respx.mock(assert_all_called=False) as upstream:
chat_route: Final = upstream.post("https://example.invalid/v1/chat/completions").mock(
return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, content=sse)
)
stream: Final = await litellm.aresponses(
model="deepseek/bridge-stream-model",
api_base="https://example.invalid/v1",
api_key="x",
input="hi",
stream=True,
)
assert isinstance(stream, ResponsesAPIStreamingIterator)
async for _ in stream:
pass
for _ in range(100):
if recorder.rows:
break
await asyncio.sleep(0.01)
await asyncio.sleep(0.1)
assert stream.completed_response is not None
assert chat_route.call_count == 1
assert len(recorder.rows) == 1
row: Final = recorder.rows[0]
logged: Final = _LOGGED_ROW.validate_python(row["response"])
assert logged["object"] == "response"
assert logged["status"] == "completed"
assert row["prompt_tokens"] == 100
assert row["completion_tokens"] == 20
cost: Final = row["response_cost"]
assert isinstance(cost, float)
assert math.isclose(
cost,
60 * prices["input_cost_per_token"]
+ 40 * prices["cache_read_input_token_cost"]
+ 20 * prices["output_cost_per_token"],
)

View file

@ -23,33 +23,20 @@ class TestExtractBase64FromDataUrl:
def test_extracts_base64_from_data_url(self):
"""Should extract pure base64 from data URL with prefix"""
data_url = "data:image/png;base64,iVBORw0KGgoAAAANS"
result = LiteLLMCompletionResponsesConfig._extract_base64_from_data_url(
data_url
)
result = LiteLLMCompletionResponsesConfig._extract_base64_from_data_url(data_url)
assert result == "iVBORw0KGgoAAAANS"
def test_returns_base64_as_is_if_no_prefix(self):
"""Should return base64 as-is if no data: prefix"""
pure_base64 = "iVBORw0KGgoAAAANS"
result = LiteLLMCompletionResponsesConfig._extract_base64_from_data_url(
pure_base64
)
result = LiteLLMCompletionResponsesConfig._extract_base64_from_data_url(pure_base64)
assert result == pure_base64
def test_handles_invalid_inputs(self):
"""Should return None for empty/None/malformed inputs"""
assert (
LiteLLMCompletionResponsesConfig._extract_base64_from_data_url("") is None
)
assert (
LiteLLMCompletionResponsesConfig._extract_base64_from_data_url(None) is None
)
assert (
LiteLLMCompletionResponsesConfig._extract_base64_from_data_url(
"data:image/png;base64"
)
is None
)
assert LiteLLMCompletionResponsesConfig._extract_base64_from_data_url("") is None
assert LiteLLMCompletionResponsesConfig._extract_base64_from_data_url(None) is None
assert LiteLLMCompletionResponsesConfig._extract_base64_from_data_url("data:image/png;base64") is None
class TestExtractImageGenerationOutputItems:
@ -75,10 +62,8 @@ class TestExtractImageGenerationOutputItems:
mock_choice.message = mock_message
mock_choice.finish_reason = "stop"
result = (
LiteLLMCompletionResponsesConfig._extract_image_generation_output_items(
choice=mock_choice,
)
result = LiteLLMCompletionResponsesConfig._extract_image_generation_output_items(
choice=mock_choice,
)
assert len(result) == 2
@ -99,10 +84,8 @@ class TestExtractImageGenerationOutputItems:
mock_choice.message = mock_message
mock_choice.finish_reason = "stop"
result = (
LiteLLMCompletionResponsesConfig._extract_image_generation_output_items(
choice=mock_choice,
)
result = LiteLLMCompletionResponsesConfig._extract_image_generation_output_items(
choice=mock_choice,
)
assert result == []
@ -122,10 +105,8 @@ class TestExtractImageGenerationOutputItems:
mock_choice.message = mock_message
mock_choice.finish_reason = "length"
result = (
LiteLLMCompletionResponsesConfig._extract_image_generation_output_items(
choice=mock_choice,
)
result = LiteLLMCompletionResponsesConfig._extract_image_generation_output_items(
choice=mock_choice,
)
assert result[0].status == "incomplete"
@ -200,10 +181,7 @@ class TestImageGenerationOutputItemIds:
def _choice_with_images(self, count):
mock_message = Mock(spec=Message)
mock_message.images = [
{"image_url": {"url": f"data:image/png;base64,IMG{idx}"}}
for idx in range(count)
]
mock_message.images = [{"image_url": {"url": f"data:image/png;base64,IMG{idx}"}} for idx in range(count)]
mock_choice = Mock(spec=Choices)
mock_choice.message = mock_message
mock_choice.finish_reason = "stop"

View file

@ -178,16 +178,12 @@ class TestEncryptedReasoningRoundTrip:
item = {
"type": "reasoning",
"id": "rs_1",
"encrypted_content": json.dumps(
[{"type": "thinking", "thinking": "hidden", "signature": "sig-one"}]
),
"encrypted_content": json.dumps([{"type": "thinking", "thinking": "hidden", "signature": "sig-one"}]),
}
messages = _transform_item(item)
assert len(messages) == 1
assert messages[0]["content"] is None
assert messages[0]["thinking_blocks"] == [
{"type": "thinking", "thinking": "hidden", "signature": "sig-one"}
]
assert messages[0]["thinking_blocks"] == [{"type": "thinking", "thinking": "hidden", "signature": "sig-one"}]
def test_unsigned_blocks_dropped(self):
"""Blocks without a signature or redacted payload are not replayed."""
@ -229,9 +225,7 @@ class TestEncryptedReasoningRoundTrip:
)
assert len(messages) == 1
assert messages[0]["reasoning_content"] == "look it up"
assert messages[0]["thinking_blocks"] == [
{"type": "thinking", "thinking": "hidden", "signature": "sig-one"}
]
assert messages[0]["thinking_blocks"] == [{"type": "thinking", "thinking": "hidden", "signature": "sig-one"}]
assert len(messages[0]["tool_calls"]) == 1
def test_replayed_blocks_precede_existing_blocks(self):

View file

@ -58,5 +58,7 @@ def test_a_reasoning_item_is_litellm_minted_by_its_id_or_by_its_encoded_thinking
"encrypted_content": A_PROVIDER_OWNED_ENCRYPTED_BLOB,
}
)
assert not is_litellm_minted_reasoning_item({"type": "message", "id": mint_reasoning_item_id(), "role": "assistant"})
assert not is_litellm_minted_reasoning_item(
{"type": "message", "id": mint_reasoning_item_id(), "role": "assistant"}
)
assert not is_litellm_minted_reasoning_item("a bare string input")

View file

@ -1245,3 +1245,134 @@ async def test_session_lookup_follows_the_owner_across_keys_and_keeps_ownerless_
assert await _history_request_ids(sqlite_db, "alice-new-key") == ["alice-old-key", "alice-new-key"]
assert await _history_request_ids(sqlite_db, "ownerless-2") == ["ownerless-2"]
def _text(content: object) -> object:
if isinstance(content, list):
return "".join(part["text"] for part in content)
return content
def _responses_spend_log(request_id: str, session_id: str, prompt: str, output: list[dict[str, object]]) -> dict:
return {
"request_id": request_id,
"call_type": "aresponses",
"session_id": session_id,
"proxy_server_request": {"input": [{"role": "user", "content": prompt}], "model": "claude-bridge"},
"response": {"id": request_id, "object": "response", "status": "completed", "output": output},
}
@pytest.mark.asyncio
async def test_message_history_reads_responses_shaped_rows_next_to_chat_shaped_rows():
session_id = "0f6f2d8e-6c8a-4b55-9a39-3d0b1f3c9e11"
fake_prisma_client = _FakePrismaClient(
results=[
[
_spend_log("chatcmpl-3333", session_id, "My favorite color is chartreuse.", "Got it."),
_responses_spend_log(
"resp_4444",
session_id,
"What is the weather in Lisbon?",
[
{
"type": "message",
"id": "msg_1",
"role": "assistant",
"status": "completed",
"content": [{"type": "output_text", "text": "Let me check.", "annotations": []}],
},
{
"type": "function_call",
"id": "fc_1",
"call_id": "call_weather",
"name": "get_weather",
"arguments": '{"city": "Lisbon"}',
},
],
),
]
]
)
with patch("litellm.proxy.proxy_server.prisma_client", fake_prisma_client):
result = await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id("resp_4444")
messages: Final = result["messages"]
assert [(message.get("role"), _text(message.get("content"))) for message in messages[:4]] == [
("user", "My favorite color is chartreuse."),
("assistant", "Got it."),
("user", "What is the weather in Lisbon?"),
("assistant", "Let me check."),
]
tool_calls: Final = [call for message in messages[3:] for call in message.get("tool_calls") or []]
assert [(call["id"], call["function"]["name"], call["function"]["arguments"]) for call in tool_calls] == [
("call_weather", "get_weather", '{"city": "Lisbon"}')
]
_WEB_SEARCH_RESULT: Final = {
"type": "web_search_tool_result",
"tool_use_id": "srvtoolu_1",
"content": [{"type": "web_search_result", "url": "https://x.com", "title": "X", "encrypted_content": "EC"}],
}
def _bridged_anthropic_web_search_turn() -> dict[str, object]:
"""A Claude turn served through the Responses to chat bridge, stored as its Responses object."""
return {
"id": "resp_1",
"object": "response",
"output": [
{
"id": "ws_srvtoolu_1",
"type": "web_search_call",
"status": "completed",
"action": {
"type": "search",
"query": "weather sf",
"sources": [{"type": "url", "url": "https://x.com"}],
},
},
{
"type": "message",
"id": "msg_1",
"role": "assistant",
"status": "completed",
"content": [{"type": "output_text", "text": "It is sunny.", "annotations": []}],
},
],
"provider_specific_fields": {"web_search_results": [_WEB_SEARCH_RESULT]},
}
def test_message_history_replays_a_bridged_anthropic_web_search_as_server_tool_blocks():
from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
history: Final = ResponsesSessionHandler.extend_chat_completion_message_with_spend_log_payload(
spend_log={"request_id": "resp_1", "response": _bridged_anthropic_web_search_turn()},
proxy_server_request_dict={"input": "weather in sf?"},
chat_completion_message_history=[],
)
anthropic_messages: Final = anthropic_messages_pt(
messages=history, model="claude-sonnet-4-5", llm_provider="anthropic"
)
assert [message["role"] for message in anthropic_messages] == ["user", "assistant"]
assistant_content: Final = anthropic_messages[1]["content"]
assert [block["type"] for block in assistant_content] == ["text", "server_tool_use", "web_search_tool_result"]
assert assistant_content[1]["id"] == "srvtoolu_1"
assert assistant_content[1]["input"] == {"query": "weather sf"}
assert assistant_content[2]["content"] == _WEB_SEARCH_RESULT["content"]
def test_message_history_keeps_a_web_search_without_a_stored_result_as_text():
turn: Final = {**_bridged_anthropic_web_search_turn(), "provider_specific_fields": None}
history: Final = ResponsesSessionHandler.extend_chat_completion_message_with_spend_log_payload(
spend_log={"request_id": "resp_1", "response": turn},
proxy_server_request_dict=None,
chat_completion_message_history=[],
)
assert str(_text_of(history[0]["content"])).startswith("Hosted web search: ")

View file

@ -78,9 +78,7 @@ class TestColdStorageObjectKeyIntegration:
metadata = {"user_api_key": "test_key", "user_api_key_team_id": "test_team"}
# Call the function
result = _get_spend_logs_metadata(
metadata=metadata, cold_storage_object_key="test/path/to/object.json"
)
result = _get_spend_logs_metadata(metadata=metadata, cold_storage_object_key="test/path/to/object.json")
# Verify the object key is stored in the result
assert result.get("cold_storage_object_key") == "test/path/to/object.json"
@ -104,11 +102,7 @@ class TestColdStorageObjectKeyIntegration:
}
# Test the extraction method
object_key = (
ResponsesSessionHandler._get_cold_storage_object_key_from_spend_log(
spend_log
)
)
object_key = ResponsesSessionHandler._get_cold_storage_object_key_from_spend_log(spend_log)
assert object_key == "test/path/to/object.json"
@ -128,11 +122,7 @@ class TestColdStorageObjectKeyIntegration:
}
# Test the extraction method
object_key = (
ResponsesSessionHandler._get_cold_storage_object_key_from_spend_log(
spend_log
)
)
object_key = ResponsesSessionHandler._get_cold_storage_object_key_from_spend_log(spend_log)
assert object_key == "test/path/to/object.json"
@ -148,20 +138,17 @@ class TestColdStorageObjectKeyIntegration:
# Mock the custom logger
mock_logger = AsyncMock()
mock_logger.get_proxy_server_request_from_cold_storage_with_object_key = (
AsyncMock(return_value={"test": "data"})
mock_logger.get_proxy_server_request_from_cold_storage_with_object_key = AsyncMock(
return_value={"test": "data"}
)
with (
patch.object(
handler, "_select_custom_logger_for_cold_storage", return_value="s3_v2"
),
patch.object(handler, "_select_custom_logger_for_cold_storage", return_value="s3_v2"),
patch(
"litellm.logging_callback_manager.get_active_custom_logger_for_callback_name",
return_value=mock_logger,
),
):
result = await handler.get_proxy_server_request_from_cold_storage_with_object_key(
object_key="test/path/to/object.json"
)
@ -184,9 +171,7 @@ class TestColdStorageObjectKeyIntegration:
s3_logger = S3Logger(s3_bucket_name="test-bucket")
# Mock the _download_object_from_s3 method
with patch.object(
s3_logger, "_download_object_from_s3", return_value={"test": "data"}
) as mock_download:
with patch.object(s3_logger, "_download_object_from_s3", return_value={"test": "data"}) as mock_download:
result = await s3_logger.get_proxy_server_request_from_cold_storage_with_object_key(
object_key="test/path/to/object.json"
)

View file

@ -38,9 +38,7 @@ from litellm.types.utils import (
)
CHAT_COMPLETION_ID = "chatcmpl-77d33d09-effa-4cd2-9c0d-c742d4358256"
RESPONSE_ID_EVENT_TYPES = frozenset(
{"response.created", "response.in_progress", "response.completed"}
)
RESPONSE_ID_EVENT_TYPES = frozenset({"response.created", "response.in_progress", "response.completed"})
def _chunk(content: str, finish_reason: str | None = None) -> ModelResponseStream:
@ -93,11 +91,7 @@ def _build_iterator(chunks) -> LiteLLMCompletionStreamingIterator:
def _response_ids(events) -> list[str]:
return [
event.response.id
for event in events
if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES
]
return [event.response.id for event in events if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES]
def test_tool_call_delta_is_emitted_as_responses_events():
@ -228,9 +222,7 @@ async def test_web_search_stream_preserves_hosted_and_client_calls(sync_mode, to
content=None,
provider_specific_fields={
"web_search_results": [first_result],
"web_search_calls": [
build_web_search_call(call_ids[0], {"query": "one"}, first_result)
]
"web_search_calls": [build_web_search_call(call_ids[0], {"query": "one"}, first_result)]
if tool_type != "function" and first_result["type"] == "web_search_tool_result"
else [],
},
@ -329,9 +321,7 @@ async def test_web_search_stream_preserves_hosted_and_client_calls(sync_mode, to
assert {event["output_index"] for event in search_events} == {completed_indexes[item["id"]]}
assert search_events[0]["item"]["status"] == "in_progress"
assert search_events[-1]["item"] == item
assert item["status"] == (
"failed" if result_kind == "error" and call_id.endswith("01Search") else "completed"
)
assert item["status"] == ("failed" if result_kind == "error" and call_id.endswith("01Search") else "completed")
assert item["action"]["type"] == "search"
assert item["action"]["query"] == ("one" if call_id.endswith("01Search") else "two")
assert item["action"]["queries"] == [item["action"]["query"]]
@ -440,9 +430,7 @@ def test_tool_call_arguments_are_chunked_to_match_openai_behavior():
)
# Create a chunk with a large arguments string that should be split
large_arguments = (
'{"param1": "value1", "param2": "value2", "param3": "value3"}' # 67 chars
)
large_arguments = '{"param1": "value1", "param2": "value2", "param3": "value3"}' # 67 chars
chunk = ModelResponseStream(
id="chunk-1",
created=123,
@ -497,9 +485,7 @@ def test_tool_call_arguments_are_chunked_to_match_openai_behavior():
# Keep draining pending events (expected: ceil(67 / 10) = 7 delta events)
while iterator._pending_tool_events:
evt = iterator._transform_chat_completion_chunk_to_response_api_chunk(
empty_chunk
)
evt = iterator._transform_chat_completion_chunk_to_response_api_chunk(empty_chunk)
if evt and evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA:
delta_events.append(evt)
@ -552,20 +538,12 @@ def test_tool_call_delta_without_id_uses_index_mapping():
while iterator._pending_tool_events:
all_events.append(iterator._pending_tool_events.pop(0))
delta_events = [
evt
for evt in all_events
if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA
]
delta_events = [evt for evt in all_events if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA]
streamed_arguments = "".join(evt.delta for evt in delta_events)
assert streamed_arguments == '{"location": "New York"}'
output_item_added_events = [
evt
for evt in all_events
if evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED
]
output_item_added_events = [evt for evt in all_events if evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED]
assert len(output_item_added_events) == 1
assert output_item_added_events[0].item.id == "fc_call_abc123"
assert output_item_added_events[0].item.call_id == "call_abc123"
@ -606,18 +584,10 @@ def test_parallel_tool_calls_without_ids_use_index_mapping():
while iterator._pending_tool_events:
all_events.append(iterator._pending_tool_events.pop(0))
output_item_added_events = [
evt
for evt in all_events
if evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED
]
output_item_added_events = [evt for evt in all_events if evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED]
assert len(output_item_added_events) == 2
delta_events = [
evt
for evt in all_events
if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA
]
delta_events = [evt for evt in all_events if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA]
arguments_by_call_id = {}
for evt in delta_events:
arguments_by_call_id.setdefault(evt.item_id, "")
@ -670,11 +640,7 @@ def test_reused_index_with_new_call_id_marks_fallback_ambiguous():
while iterator._pending_tool_events:
all_events.append(iterator._pending_tool_events.pop(0))
delta_events = [
evt
for evt in all_events
if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA
]
delta_events = [evt for evt in all_events if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA]
arguments_by_call_id = {}
for evt in delta_events:
arguments_by_call_id.setdefault(evt.item_id, "")
@ -741,22 +707,17 @@ def test_sync_streaming_events_share_the_chat_completion_response_id():
assert len(response_ids) == 3
assert len(set(response_ids)) == 1
assert (
ResponsesAPIRequestUtils.decode_responses_api_response_id(response_ids[0])["response_id"]
== CHAT_COMPLETION_ID
ResponsesAPIRequestUtils.decode_responses_api_response_id(response_ids[0])["response_id"] == CHAT_COMPLETION_ID
)
@pytest.mark.asyncio
async def test_streaming_emits_every_chunk_after_priming_the_response_id():
iterator = _build_iterator(
[_chunk("Hel"), _chunk("lo"), _chunk("!", finish_reason="stop")]
)
iterator = _build_iterator([_chunk("Hel"), _chunk("lo"), _chunk("!", finish_reason="stop")])
events = [event async for event in iterator]
deltas = "".join(
event.delta for event in events if getattr(event, "type", None) == "response.output_text.delta"
)
deltas = "".join(event.delta for event in events if getattr(event, "type", None) == "response.output_text.delta")
assert deltas == "Hello!"
@ -802,7 +763,9 @@ async def test_leading_empty_choices_chunk_does_not_kill_the_stream():
event_types = [getattr(event, "type", None) for event in events]
assert event_types.count(ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED) == 1
assert "".join(event.delta for event in events if event.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA) == "Hello!"
assert (
"".join(event.delta for event in events if event.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA) == "Hello!"
)
assert event_types[-1] == ResponsesAPIStreamEvents.RESPONSE_COMPLETED
@ -986,9 +949,7 @@ def test_streamed_unrecognized_tool_choice_is_echoed_as_auto() -> None:
litellm_metadata={},
)
response_events: Final = [
event for event in iterator if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES
]
response_events: Final = [event for event in iterator if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES]
assert [event.response.tool_choice for event in response_events] == ["auto", "auto", "auto"]
@ -1038,9 +999,7 @@ def _signed_thinking_and_text_in_one_chunk() -> tuple[ModelResponseStream, ...]:
chunk: Final = _chunk("Hello!", finish_reason="stop")
signed_block: Final = {"type": "thinking", "thinking": "", "signature": "sig_only"}
delta: Final = Delta(**{**chunk.choices[0].delta.model_dump(), "thinking_blocks": [signed_block]})
return (
chunk.model_copy(update={"choices": [StreamingChoices(index=0, delta=delta, finish_reason="stop")]}),
)
return (chunk.model_copy(update={"choices": [StreamingChoices(index=0, delta=delta, finish_reason="stop")]}),)
StreamChunks = Callable[[], tuple[ModelResponseStream, ...]]
@ -1560,9 +1519,7 @@ async def test_anthropic_tool_search_call_streams_as_tool_search_call_item(sync_
model="claude-sonnet-5-5",
object="chat.completion.chunk",
choices=[
StreamingChoices(
index=0, delta=delta, finish_reason="tool_calls" if index == len(deltas) - 1 else None
)
StreamingChoices(index=0, delta=delta, finish_reason="tool_calls" if index == len(deltas) - 1 else None)
],
)
for index, delta in enumerate(deltas)

View file

@ -60,11 +60,7 @@ def test_function_call_output_stays_adjacent_to_tool_call():
assistant_ok_idx = i
elif isinstance(content, list):
for block in content:
if (
isinstance(block, dict)
and block.get("type") == "text"
and block.get("text") == "Done."
):
if isinstance(block, dict) and block.get("type") == "text" and block.get("text") == "Done.":
assistant_ok_idx = i
break

View file

@ -0,0 +1,87 @@
import json
from datetime import datetime
from typing import Final
import httpx
import pytest
import respx
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.responses.bridge_executor import native_logging_obj
API_BASE: Final = "https://bridge-executor.invalid/v1"
_CHAT_REPLY: Final = {
"id": "chatcmpl-1",
"object": "chat.completion",
"created": 1,
"model": "m",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
}
@pytest.fixture(autouse=True)
def _offline(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
@pytest.mark.parametrize(
("surface", "litellm_params", "model_info", "upstream_model"),
[
("messages", {"model": "openai/gpt-4.1", "use_chat_completions_api": True}, None, "gpt-4.1"),
("messages", {"model": "openai/gpt-4.1", "use_chat_completions_api": True}, {"mode": "responses"}, "gpt-4.1"),
("messages", {"model": "openai/chat_completions/o3-pro"}, None, "o3-pro"),
("responses", {"model": "openai/chat_completions/o3-pro"}, None, "o3-pro"),
],
)
@pytest.mark.asyncio
async def test_a_deployment_asking_for_chat_completions_is_sent_to_chat_without_the_prefix(
surface: str,
litellm_params: dict[str, object],
model_info: dict[str, object] | None,
upstream_model: str,
respx_mock: respx.MockRouter,
):
chat: Final = respx_mock.post(f"{API_BASE}/chat/completions").mock(
return_value=httpx.Response(200, json=_CHAT_REPLY)
)
router: Final = litellm.Router(
model_list=[
{
"model_name": "m",
"litellm_params": {"api_base": API_BASE, "api_key": "k", **litellm_params},
**({"model_info": model_info} if model_info is not None else {}),
}
],
num_retries=0,
)
if surface == "messages":
await router.aanthropic_messages(model="m", max_tokens=32, messages=[{"role": "user", "content": "hi"}])
else:
await router.aresponses(model="m", input="hi")
assert chat.call_count == 1
assert json.loads(chat.calls.last.request.content)["model"] == upstream_model
@pytest.mark.parametrize(("response_cache", "native"), [(None, True), (litellm.Cache(type="local"), False)])
def test_a_bridged_responses_call_goes_through_aresponses_when_the_response_cache_is_on(
response_cache: litellm.Cache | None, native: bool, monkeypatch: pytest.MonkeyPatch
):
monkeypatch.setattr(litellm, "cache", response_cache)
logging_obj: Final = Logging(
model="gpt-4.1",
messages=[{"role": "user", "content": "hi"}],
stream=False,
call_type="anthropic_messages",
start_time=datetime.now(),
litellm_call_id="bridge-executor-cache",
function_id="bridge-executor-cache",
)
chosen: Final = native_logging_obj(logging_obj, {"api_key": "k"}, {"model": "openai/gpt-4.1", "input": "hi"})
assert (chosen is logging_obj) is native

View file

@ -15,7 +15,7 @@ import litellm
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig
from litellm.responses.main import _resolve_responses_api_provider_config
from litellm.responses.main import resolve_responses_api_provider_config
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import ModelResponse
@ -115,13 +115,13 @@ def _respx_interceptable_httpx_client(monkeypatch):
],
)
def test_resolver_opt_in_gates_openai_like_config(model_info, expected_type):
config = _resolve_responses_api_provider_config("my-model", "custom_openai", model_info, None)
config = resolve_responses_api_provider_config("my-model", "custom_openai", model_info, None)
assert type(config) is expected_type
def test_resolver_keeps_native_provider_config():
"""`openai/` already routes /v1/responses natively; the opt-in must not swap its config."""
config = _resolve_responses_api_provider_config("gpt-4.1", "openai", OPT_IN, None)
config = resolve_responses_api_provider_config("gpt-4.1", "openai", OPT_IN, None)
assert type(config) is OpenAIResponsesAPIConfig

View file

@ -0,0 +1,68 @@
import subprocess
import sys
from pathlib import Path
from typing import Final
import pytest
from scripts.check_native_imports import main, violations
CHECKER: Final = Path(__file__).resolve().parents[2] / "scripts/check_native_imports.py"
@pytest.mark.parametrize(
"path, source",
(
("litellm/native/chat.py", "from litellm.main import execute_chat"),
("litellm/native/chat.py", "from litellm import main"),
("litellm/native/chat.py", "import litellm.main"),
("litellm/native/chat.py", "import litellm.main as legacy"),
("litellm/native/chat.py", "from ..main import completion"),
("litellm/native/chat.py", "from .. import main"),
("litellm/native/chat_providers/azure.py", "from ...main import completion"),
("litellm/native/chat.py", "import litellm\n\ndef run():\n return litellm.main.completion()"),
),
)
def test_native_module_importing_main_is_rejected(path, source):
assert violations(Path(path), source)
@pytest.mark.parametrize(
"path, source",
(
("litellm/native/chat.py", "from litellm.native.chat_providers.azure import complete_azure"),
("litellm/native/chat.py", "import litellm\n\ndef run():\n return litellm.completion"),
("litellm/native/chat.py", "from litellm.maintenance import x"),
("litellm/native/chat.py", "from .chat_mock import mock_completion"),
("litellm/native/chat_providers/azure.py", "from ..chat_mock import mock_completion"),
("litellm/native/chat.py", "from litellm import utils"),
),
)
def test_native_module_without_main_dependency_passes(path, source):
assert violations(Path(path), source) == ()
def test_command_reports_the_offending_line_and_passes_once_it_is_gone(tmp_path, monkeypatch, capsys):
package: Final = tmp_path / "litellm/native/chat_providers"
package.mkdir(parents=True)
module: Final = package / "azure.py"
module.write_text("import os\nfrom litellm.main import completion\n")
monkeypatch.chdir(tmp_path)
assert main() == 1
assert "litellm/native/chat_providers/azure.py:2:" in capsys.readouterr().err
rejected: Final = subprocess.run(
[sys.executable, "-I", str(CHECKER)], cwd=tmp_path, capture_output=True, text=True, check=False
)
assert rejected.returncode == 1
module.write_text("import os\n")
assert main() == 0
accepted: Final = subprocess.run(
[sys.executable, "-I", str(CHECKER)], cwd=tmp_path, capture_output=True, text=True, check=False
)
assert accepted.returncode == 0
def test_the_native_package_has_no_main_dependency(monkeypatch):
monkeypatch.chdir(CHECKER.parents[1])
assert main() == 0

View file

@ -7,6 +7,7 @@ import json
import logging
import os
import threading
import time
import urllib.parse
from collections.abc import Callable, Mapping
from concurrent.futures import ThreadPoolExecutor
@ -20,7 +21,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
import respx
from openai import APITimeoutError, AsyncOpenAI, OpenAI
from openai import AsyncOpenAI, OpenAI
from openai._legacy_response import LegacyAPIResponse
from openai.types.chat.chat_completion import ChatCompletion
from pydantic import BaseModel, ConfigDict, TypeAdapter
@ -45,10 +46,12 @@ from litellm.litellm_core_utils.get_litellm_params import stored_control_options
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.native import chat as native_chat
from litellm.types.litellm_params import ControlOptions
from litellm.types.llms.openai import AllMessageValues, HttpxBinaryResponseContent
from litellm.types.prompts.init_prompts import PromptSpec
from litellm.types.utils import (
CallTypes,
Delta,
ModelResponseStream,
StandardCallbackDynamicParams,
@ -517,7 +520,7 @@ def test_drop_anthropic_only_tool_keys_strips_tool_and_function_levels(key: str)
"opaque_tool",
]
cleaned = litellm_main._drop_anthropic_only_tool_keys(tools=tools)
cleaned = native_chat._drop_anthropic_only_tool_keys(tools=tools)
assert cleaned == [
{"type": "function", "name": "example_tool", "function": {"name": "example_tool"}},
@ -553,6 +556,98 @@ def test_completion_strips_eager_input_streaming_before_openai(respx_mock: respx
assert sent_tool["function"]["name"] == "write_file"
class _SyncSuccessCounter(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.calls: list[str] = [] # mutable-ok: test recorder of success-callback events
def log_success_event(self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object):
self.calls.append(str(kwargs.get("call_type")))
def _settled_success_calls(counter: _SyncSuccessCounter, expected: int) -> int:
for _ in range(50):
if len(counter.calls) >= expected:
break
time.sleep(0.02)
return len(counter.calls)
class _DeploymentHookCounter(CustomLogger):
call_types: tuple[str, ...] = ()
async def async_pre_call_deployment_hook(self, kwargs: dict[str, object], call_type: CallTypes | None) -> None:
self.call_types = (*self.call_types, str(call_type))
@pytest.mark.asyncio
async def test_responses_mode_model_without_a_responses_api_stays_on_chat(
respx_mock: respx.MockRouter, openai_api_response: dict[str, object], monkeypatch: pytest.MonkeyPatch
):
counter: Final = _DeploymentHookCounter()
monkeypatch.setattr(litellm, "callbacks", [counter])
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
litellm.register_model(
{"groq/responses-only-model": {"litellm_provider": "groq", "mode": "responses", "max_tokens": 100}}
)
api_base: Final = "https://example.invalid/openai/v1"
chat_route: Final = respx_mock.post(f"{api_base}/chat/completions").mock(
return_value=httpx.Response(status_code=200, json=openai_api_response)
)
response: Final = await litellm.acompletion(
model="groq/responses-only-model",
api_base=api_base,
api_key="sk-x",
messages=[{"role": "user", "content": "hi"}],
)
assert isinstance(response, litellm.ModelResponse)
assert response.id == openai_api_response["id"]
assert chat_route.call_count == 1
assert counter.call_types == ("CallTypes.acompletion",)
def test_execute_chat_calls_the_provider_without_logging(
respx_mock: respx.MockRouter, openai_api_response, monkeypatch: pytest.MonkeyPatch
):
counter: Final = _SyncSuccessCounter()
monkeypatch.setattr(litellm, "callbacks", [counter])
api_base: Final = "http://localhost:12347/v1"
mock_route: Final = respx_mock.post(url__regex=rf"{api_base}/chat/completions.*").mock(
return_value=httpx.Response(status_code=200, json=openai_api_response)
)
messages: Final = [{"role": "user", "content": "hi"}]
logging_obj: Final = LiteLLMLogging(
model="hosted_vllm/served-model",
messages=messages,
stream=False,
call_type="anthropic_messages",
start_time=datetime.now(),
litellm_call_id="execute-chat-call",
function_id="execute-chat-fn",
)
response: Final = litellm_main.execute_chat(
model="hosted_vllm/served-model",
messages=list(messages),
kwargs={"api_base": api_base, "api_key": "sk-x", "litellm_logging_obj": logging_obj},
non_default_params={},
control_options=ControlOptions(),
prompt_id=None,
prompt_variables=None,
)
assert isinstance(response, litellm.ModelResponse)
assert response.id == openai_api_response["id"]
assert mock_route.call_count == 1
assert _settled_success_calls(counter, expected=1) == 0
litellm.completion(model="hosted_vllm/served-model", messages=messages, api_base=api_base, api_key="sk-x")
assert _settled_success_calls(counter, expected=1) == 1
_CLAUDE_TOOL_SEARCH_CALL: Final = {
"id": "srvtoolu_01search",
"type": "function",
@ -2591,7 +2686,7 @@ def test_completion_optional_params_base_model(
Regression for #29618: overwriting ``model`` with a friendly ``base_model``
label made Bedrock drop ``tools``/``tool_choice`` under ``drop_params``."""
with patch("litellm.main.get_optional_params") as mock_get_optional_params:
with patch("litellm.native.chat.get_optional_params") as mock_get_optional_params:
mock_get_optional_params.return_value = MagicMock()
import litellm
@ -5683,7 +5778,7 @@ def test_bridged_responses_with_openai_http_handler_keeps_forwarded_headers_out_
def test_aiohttp_openai_warns_only_when_http2_enabled(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, http2_on: bool
):
from litellm.main import base_llm_aiohttp_handler
from litellm.native.chat_providers.handlers import base_llm_aiohttp_handler
monkeypatch.setattr(litellm, "http2", http2_on)
monkeypatch.delenv("LITELLM_HTTP2", raising=False)
@ -5984,7 +6079,6 @@ def test_drop_params_false_still_rejects_an_invalid_stream_chunk_size() -> None:
def test_acompletion_params():
import inspect
from litellm.types.completion import CompletionRequest
acompletion_params_odict = inspect.signature(acompletion).parameters
completion_params_dict = inspect.signature(completion).parameters

View file

@ -20,7 +20,7 @@ import pytest
import litellm
from litellm.litellm_core_utils.get_model_cost_map import _expand_model_aliases, adopt_model_cost_map
from litellm.main import _build_custom_pricing_entry
from litellm.native.request_support import _build_custom_pricing_entry
from litellm.utils import _invalidate_model_cost_lowercase_map
@ -1677,7 +1677,7 @@ def test_register_model_scopes_builtin_match_to_the_given_provider():
def test_per_request_custom_pricing_scopes_a_colliding_deployment_id_to_its_provider():
from litellm.main import _register_custom_pricing_for_request
from litellm.native.request_support import register_custom_pricing_for_request
deployment_id: Final = "baseten/zai-org/glm-5.2"
builtin_key: Final = "baseten/zai-org/GLM-5.2"
@ -1686,7 +1686,7 @@ def test_per_request_custom_pricing_scopes_a_colliding_deployment_id_to_its_prov
builtin_row_before: Final = copy.deepcopy(litellm.model_cost[builtin_key])
openai_models_before: Final = frozenset(litellm.open_ai_chat_completion_models)
try:
_register_custom_pricing_for_request(
register_custom_pricing_for_request(
model="zai-org/GLM-5.2",
custom_llm_provider="openai",
kwargs={

View file

@ -1843,7 +1843,9 @@ def test_arouter_responses_api_bridge():
## CONFIRM BRIDGE IS CALLED
# an empty stream: the Router reads the first chunk before returning, and a stream of mocks never produces one
with patch.object(litellm, "responses", return_value=iter([])) as mock_responses:
from litellm.responses import bridge_executor
with patch.object(bridge_executor, "native_responses", return_value=iter([])) as mock_responses:
result = router.completion(
model="[IP-approved] o3-pro",
messages=[{"role": "user", "content": "Hello, world!"}],