mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
refactor: make cross-API bridges call native executors instead of re-entering acompletion and aresponses
Some checks failed
Cost map pins recheck / rerun-cost-map-guard (push) Has been cancelled
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Some checks failed
Cost map pins recheck / rerun-cost-map-guard (push) Has been cancelled
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Each client surface (/v1/chat/completions, /v1/responses, /v1/messages) now decides the provider's native API format once with litellm/native/route_plan.py, owns logging for the request, and hands the translated request to a native executor (execute_chat, execute_responses, execute_messages) instead of calling back into a public entry point. - Messages to Responses, Messages to chat, Responses to chat and chat to Responses bridges call the native executors - Responses-mode models on providers with no Responses API stay on chat - Bridges price the provider reply before translating it, keep its headers, map provider errors and run JSON-schema validation as re-entry did - Bridged streams are logged once, under the client's call type, when the client stream ends - execute_chat and the chat provider helpers move from main.py to litellm/native/, and a lint check keeps litellm.native from importing litellm.main
This commit is contained in:
parent
60601ca995
commit
0ba4e85496
98 changed files with 10674 additions and 6616 deletions
2
.github/workflows/test-unit.yml
vendored
2
.github/workflows/test-unit.yml
vendored
|
|
@ -424,6 +424,7 @@ jobs:
|
|||
tests/unit/images
|
||||
tests/unit/interactions
|
||||
tests/unit/messages
|
||||
tests/unit/native
|
||||
tests/unit/ocr
|
||||
tests/unit/passthrough
|
||||
tests/unit/rag
|
||||
|
|
@ -715,6 +716,7 @@ jobs:
|
|||
test-path: |-
|
||||
tests/unit/responses/litellm_completion_transformation
|
||||
tests/unit/responses/test_additional_tools.py
|
||||
tests/unit/responses/test_bridge_executor.py
|
||||
tests/unit/responses/test_custom_tool_call.py
|
||||
tests/unit/responses/test_metadata_codex_callback.py
|
||||
tests/unit/responses/test_no_duplicate_spend_logs.py
|
||||
|
|
|
|||
1
Makefile
1
Makefile
|
|
@ -164,6 +164,7 @@ lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
|
|||
# Linting targets
|
||||
lint-ruff: $(LINT_DEP_INSTALL)
|
||||
$(UV_RUN) python scripts/check_mcp_operation_boundary.py
|
||||
$(UV_RUN) python scripts/check_native_imports.py
|
||||
cd litellm && $(UV_RUN) ruff check . && cd ..
|
||||
$(UV_RUN) ruff check --config ruff-tests.toml tests
|
||||
|
||||
|
|
|
|||
|
|
@ -11,8 +11,10 @@ from typing import (
|
|||
cast, # noqa: TID251 # optional_params reaches the bridge as an untyped dict
|
||||
)
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -36,6 +38,60 @@ def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str:
|
|||
return f"{custom_llm_provider}/{model}"
|
||||
|
||||
|
||||
_TOOLS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
_BRIDGED_REQUEST: Final = TypeAdapter(dict[str, object])
|
||||
_TOOLS_ONLY_RESPONSES_SERVES: Final = frozenset({"mcp", "file_search"})
|
||||
|
||||
|
||||
def _needs_responses_surface(request_data: Mapping[str, object]) -> bool:
|
||||
"""`responses()` also checks the cache and dispatches MCP and emulated file_search tools, which the executor skips."""
|
||||
if litellm.cache is not None or request_data.get("use_chat_completions_api"):
|
||||
return True
|
||||
model: Final = request_data.get("model")
|
||||
if not isinstance(model, str) or model.startswith("openai/chat_completions/"):
|
||||
return True
|
||||
try:
|
||||
tools: Final = _TOOLS.validate_python(request_data.get("tools") or ())
|
||||
except ValidationError:
|
||||
return True
|
||||
return any(tool.get("type") in _TOOLS_ONLY_RESPONSES_SERVES for tool in tools)
|
||||
|
||||
|
||||
def _executor_logging_obj(
|
||||
request_data: Mapping[str, object], logging_obj: "LiteLLMLoggingObj"
|
||||
) -> "LiteLLMLoggingObj | None":
|
||||
from litellm.responses.bridge_executor import has_native_responses_config, native_logging_obj
|
||||
|
||||
model: Final = request_data.get("model")
|
||||
if _needs_responses_surface(request_data) or not isinstance(model, str):
|
||||
return None
|
||||
if not has_native_responses_config(model, request_data):
|
||||
return None
|
||||
return native_logging_obj(logging_obj, request_data, request_data)
|
||||
|
||||
|
||||
def _logged_by_client_stream(_completed: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _hand_completion_to_client_stream(result: object, executor_logging: "LiteLLMLoggingObj | None") -> None:
|
||||
"""The chat stream wrapping the executor's Responses stream logs the request once, in the shape the client got."""
|
||||
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
|
||||
|
||||
if executor_logging is not None and isinstance(result, BaseResponsesAPIStreamingIterator):
|
||||
result.hand_completion_to(_logged_by_client_stream)
|
||||
|
||||
|
||||
def _retagged_as_responses(
|
||||
request_data: Mapping[str, object], logging_obj: "LiteLLMLoggingObj"
|
||||
) -> Mapping[str, object]:
|
||||
"""`responses()` logs the Responses object it returns, so the shared logging object takes its call type."""
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
setattr(logging_obj, "call_type", CallTypes.responses.value)
|
||||
return request_data
|
||||
|
||||
|
||||
class ResponsesToCompletionBridgeHandler:
|
||||
def __init__(self):
|
||||
from .transformation import LiteLLMResponsesTransformationHandler
|
||||
|
|
@ -172,6 +228,7 @@ class ResponsesToCompletionBridgeHandler:
|
|||
|
||||
from litellm import responses
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.responses.bridge_executor import native_responses
|
||||
|
||||
validated_kwargs: Final = self.validate_input_kwargs(kwargs)
|
||||
model: Final = validated_kwargs["model"]
|
||||
|
|
@ -199,11 +256,16 @@ class ResponsesToCompletionBridgeHandler:
|
|||
# `sanitized_litellm_params` already carries both, so passing them again
|
||||
# would raise a duplicate-keyword TypeError.
|
||||
request_data["custom_llm_provider"] = custom_llm_provider
|
||||
request_data["model"] = _restore_routing_prefix(model, custom_llm_provider)
|
||||
routed_model: Final = _restore_routing_prefix(model, custom_llm_provider)
|
||||
request_data["model"] = routed_model
|
||||
if kwargs.get("cache") is not None:
|
||||
request_data["cache"] = kwargs["cache"]
|
||||
result: Final = responses(
|
||||
**request_data,
|
||||
bridged_request: Final = _BRIDGED_REQUEST.validate_python(request_data)
|
||||
executor_logging: Final = _executor_logging_obj(bridged_request, logging_obj)
|
||||
result: Final = (
|
||||
responses(**_retagged_as_responses(request_data, logging_obj))
|
||||
if executor_logging is None
|
||||
else native_responses(bridged_request, routed_model, bridged_request, executor_logging)
|
||||
)
|
||||
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
|
@ -253,6 +315,7 @@ class ResponsesToCompletionBridgeHandler:
|
|||
return self._apply_post_stream_processing(result, model, custom_llm_provider)
|
||||
from .transformation import is_structured_output_request
|
||||
|
||||
_hand_completion_to_client_stream(result, executor_logging)
|
||||
completion_stream: Final = self.transformation_handler.get_model_response_iterator(
|
||||
streaming_response=result,
|
||||
sync_stream=True,
|
||||
|
|
@ -273,6 +336,7 @@ class ResponsesToCompletionBridgeHandler:
|
|||
async def acompletion(self, *args, **kwargs) -> Union["ModelResponse", "CustomStreamWrapper"]:
|
||||
from litellm import aresponses
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.responses.bridge_executor import anative_responses
|
||||
|
||||
validated_kwargs: Final = self.validate_input_kwargs(kwargs)
|
||||
model: Final = validated_kwargs["model"]
|
||||
|
|
@ -302,12 +366,16 @@ class ResponsesToCompletionBridgeHandler:
|
|||
# `sanitized_litellm_params` already carries both, so passing them again
|
||||
# would raise a duplicate-keyword TypeError.
|
||||
request_data["custom_llm_provider"] = custom_llm_provider
|
||||
request_data["model"] = _restore_routing_prefix(model, custom_llm_provider)
|
||||
routed_model: Final = _restore_routing_prefix(model, custom_llm_provider)
|
||||
request_data["model"] = routed_model
|
||||
if kwargs.get("cache") is not None:
|
||||
request_data["cache"] = kwargs["cache"]
|
||||
result: Final = await aresponses(
|
||||
**request_data,
|
||||
aresponses=True,
|
||||
bridged_request: Final = _BRIDGED_REQUEST.validate_python(request_data)
|
||||
executor_logging: Final = _executor_logging_obj(bridged_request, logging_obj)
|
||||
result: Final = (
|
||||
await aresponses(**_retagged_as_responses(request_data, logging_obj), aresponses=True)
|
||||
if executor_logging is None
|
||||
else await anative_responses(bridged_request, routed_model, bridged_request, executor_logging)
|
||||
)
|
||||
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
|
@ -357,6 +425,7 @@ class ResponsesToCompletionBridgeHandler:
|
|||
return self._apply_post_stream_processing(result, model, custom_llm_provider)
|
||||
from .transformation import is_structured_output_request
|
||||
|
||||
_hand_completion_to_client_stream(result, executor_logging)
|
||||
completion_stream: Final = self.transformation_handler.get_model_response_iterator(
|
||||
streaming_response=result,
|
||||
sync_stream=False,
|
||||
|
|
|
|||
|
|
@ -1195,10 +1195,6 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
|
||||
api_model: Final = model
|
||||
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
setattr(litellm_logging_obj, "call_type", CallTypes.responses.value)
|
||||
|
||||
sanitized_litellm_params: Final = self._build_sanitized_litellm_params(litellm_params)
|
||||
|
||||
request_data: Final = {
|
||||
|
|
|
|||
|
|
@ -41,13 +41,13 @@ from litellm.llms.black_forest_labs.image_generation.handler import bfl_image_ge
|
|||
from litellm.llms.openai.workload_identity import resolve_openai_workload_identity_config
|
||||
from litellm.main import (
|
||||
azure_chat_completions,
|
||||
base_llm_aiohttp_handler,
|
||||
base_llm_http_handler,
|
||||
bedrock_image_edit,
|
||||
bedrock_image_generation,
|
||||
openai_chat_completions,
|
||||
openai_image_variations,
|
||||
)
|
||||
from litellm.native.chat_providers.handlers import base_llm_aiohttp_handler
|
||||
|
||||
###########################################
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
|
|
|||
|
|
@ -79,6 +79,7 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
get_provider_response_headers_from_hidden_params,
|
||||
is_expected_client_error,
|
||||
native_messages_reported_cost,
|
||||
process_response_headers,
|
||||
provider_reported_usage_cost,
|
||||
proxy_stamped_kubernetes_pod_routing,
|
||||
proxy_stamped_used_client_oauth_token,
|
||||
|
|
@ -645,6 +646,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
baseline_observation: "CapturedBaselineObservation | None" = None
|
||||
_on_failure_payload_built: "Callable[[], None] | None" = None
|
||||
|
||||
def hold_stream_completion(self, completion: tuple[object, ...]) -> None:
|
||||
"""Parks stream-end logging for the proxy to release once its end-of-stream guardrails finish."""
|
||||
setattr(self, "_deferred_stream_complete_args", completion) # noqa: B010 # declaring the slot here turns the proxy's reads of it into private-usage findings
|
||||
|
||||
@property
|
||||
def _defer_async_logging(self) -> bool:
|
||||
return self.defer_async_logging
|
||||
|
|
@ -4944,6 +4949,29 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
row is built from this response rather than re-parsed from the served body."""
|
||||
self.bridged_model_response = response
|
||||
|
||||
def price_bridged_provider_response(self, response: ModelResponse | ResponsesAPIResponse) -> None:
|
||||
"""Stamps the cost and headers of the provider response a bridge translates into its client's shape.
|
||||
|
||||
The client shape has no field for some of the usage the price depends on (built-in tool calls,
|
||||
the 1h cache-write split), and the translation carries the stamped cost over, so the request is
|
||||
billed for what the provider returned. The headers get the round of processing the call that made
|
||||
the response gave them, which also names the provider request id `llm_provider-request-id`."""
|
||||
set_hidden_param(response, "response_cost", self.response_cost_calculator(result=response))
|
||||
provider_headers: Final = (get_hidden_params(response) or {}).get("additional_headers")
|
||||
if isinstance(provider_headers, dict):
|
||||
set_hidden_param(
|
||||
response,
|
||||
"additional_headers",
|
||||
process_response_headers(provider_headers, preserve_litellm_internal_headers=True),
|
||||
)
|
||||
|
||||
def price_bridged_provider_stream(self, stream: object) -> None:
|
||||
"""Prices a provider stream a bridge wraps for its client, as the call that opened it did.
|
||||
|
||||
Nothing has streamed yet, so this only records the zero cost breakdown the proxy builds the
|
||||
response headers from; the request is billed when the stream ends."""
|
||||
self.response_cost_calculator(result=stream)
|
||||
|
||||
def record_bridged_responses_api_response(self, response: ResponsesAPIResponse) -> None:
|
||||
self.record_bridged_model_response(self._translate_responses_api_response_to_model_response(response))
|
||||
|
||||
|
|
|
|||
|
|
@ -305,6 +305,7 @@ def _provider_hidden_params(
|
|||
|
||||
class CustomStreamWrapper:
|
||||
_earlier_rounds_spend: "BilledRoundsSpend | None" = None
|
||||
_on_complete: Callable[[object], None] | None = None
|
||||
|
||||
@property
|
||||
def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation
|
||||
|
|
@ -1755,7 +1756,7 @@ class CustomStreamWrapper:
|
|||
setattr(e, "message", str(e))
|
||||
raise self._map_stream_exception(e)
|
||||
|
||||
def set_logging_event_loop(self, loop):
|
||||
def set_logging_event_loop(self, loop: asyncio.AbstractEventLoop) -> None:
|
||||
"""
|
||||
import litellm, asyncio
|
||||
|
||||
|
|
@ -1907,11 +1908,19 @@ class CustomStreamWrapper:
|
|||
if not cache_hit and self.logging_obj.llm_caching_handler is not None:
|
||||
await self.logging_obj.llm_caching_handler.add_streaming_response_to_cache(processed_chunk)
|
||||
|
||||
def hand_completion_to(self, on_complete: Callable[[object], None]) -> None:
|
||||
"""A bridge whose own client stream logs the request takes the assembled response instead of the logger."""
|
||||
self._on_complete = on_complete
|
||||
|
||||
def take_completion_back(self) -> None:
|
||||
"""The logger gets the assembled response again, as before a bridge took it."""
|
||||
self._on_complete = None
|
||||
|
||||
def run_success_logging_and_cache_storage(self, processed_chunk, cache_hit: bool):
|
||||
"""
|
||||
Runs success logging in a thread and adds the response to the cache
|
||||
"""
|
||||
if litellm.disable_streaming_logging is True:
|
||||
if litellm.disable_streaming_logging is True or self._on_complete is not None:
|
||||
"""
|
||||
[NOT RECOMMENDED]
|
||||
Set this via `litellm.disable_streaming_logging = True`.
|
||||
|
|
@ -2174,13 +2183,18 @@ class CustomStreamWrapper:
|
|||
processed_chunk=_cache_copy,
|
||||
cache_hit=cache_hit,
|
||||
)
|
||||
executor.submit(
|
||||
self.logging_obj.success_handler,
|
||||
_log_copy,
|
||||
None,
|
||||
None,
|
||||
cache_hit,
|
||||
)
|
||||
if self._on_complete is not None:
|
||||
self._on_complete(_log_copy)
|
||||
else:
|
||||
executor.submit(
|
||||
self.logging_obj.success_handler,
|
||||
_log_copy,
|
||||
None,
|
||||
None,
|
||||
cache_hit,
|
||||
)
|
||||
elif self._on_complete is not None:
|
||||
self._on_complete(response)
|
||||
else:
|
||||
executor.submit(
|
||||
self.logging_obj.success_handler,
|
||||
|
|
@ -2415,7 +2429,9 @@ class CustomStreamWrapper:
|
|||
"_on_deferred_stream_complete",
|
||||
None,
|
||||
)
|
||||
if _deferred_cb is not None:
|
||||
if self._on_complete is not None:
|
||||
self._on_complete(complete_streaming_response)
|
||||
elif _deferred_cb is not None:
|
||||
# Proxy has post-call guardrails. Store the assembled
|
||||
# response so the outer streaming consumer
|
||||
# (ProxyLogging.async_post_call_streaming_iterator_hook)
|
||||
|
|
@ -2423,10 +2439,7 @@ class CustomStreamWrapper:
|
|||
# end-of-stream blocks complete. Scheduling here via
|
||||
# create_task would race with unified_guardrail's
|
||||
# end-of-stream block for short-stream providers.
|
||||
self.logging_obj._deferred_stream_complete_args = (
|
||||
complete_streaming_response,
|
||||
cache_hit,
|
||||
)
|
||||
self.logging_obj.hold_stream_completion((complete_streaming_response, cache_hit))
|
||||
else:
|
||||
# prefer_async_handlers routes CustomLogger to async_success_handler
|
||||
# when consumers use ``async for`` on sync-SDK streams. Legacy string
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ from litellm.llms.anthropic.messages.context_management import (
|
|||
apply_context_management,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.convertible_blocks import reject_malformed_tool_result_text
|
||||
from litellm.llms.anthropic.messages.held_stream_completion import hold_completion_of
|
||||
from litellm.llms.anthropic.messages.safeguards import (
|
||||
SafeguardsEvaluator,
|
||||
build_safeguards_evaluator,
|
||||
|
|
@ -39,6 +40,7 @@ from litellm.llms.anthropic.messages.utils import (
|
|||
proxy_sub_call_logging_kwargs,
|
||||
proxy_sub_call_trace_id,
|
||||
)
|
||||
from litellm.native.chat_plan import aexecute_planned_chat, execute_planned_chat, plan_chat
|
||||
from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import (
|
||||
AnthropicMessagesResponse,
|
||||
|
|
@ -797,8 +799,13 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
extra_kwargs=kwargs,
|
||||
)
|
||||
|
||||
completion_response: Final = await litellm.acompletion(**completion_kwargs)
|
||||
logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs)
|
||||
planned_chat: Final = None if logging_obj is None else plan_chat(completion_kwargs, logging_obj)
|
||||
completion_response: Final = (
|
||||
await litellm.acompletion(**completion_kwargs)
|
||||
if planned_chat is None
|
||||
else await aexecute_planned_chat(planned_chat)
|
||||
)
|
||||
|
||||
if stream:
|
||||
transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming(
|
||||
|
|
@ -810,6 +817,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
completion_request=completion_kwargs,
|
||||
litellm_logging_obj=logging_obj,
|
||||
safeguards_evaluator=safeguards_evaluator,
|
||||
held_completion=None if planned_chat is None else hold_completion_of(completion_response, logging_obj),
|
||||
)
|
||||
if transformed_stream is not None:
|
||||
return transformed_stream
|
||||
|
|
@ -944,8 +952,11 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
extra_kwargs=kwargs,
|
||||
)
|
||||
|
||||
completion_response: Final = litellm.completion(**completion_kwargs)
|
||||
logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs)
|
||||
planned_chat: Final = None if logging_obj is None else plan_chat(completion_kwargs, logging_obj)
|
||||
completion_response: Final = (
|
||||
litellm.completion(**completion_kwargs) if planned_chat is None else execute_planned_chat(planned_chat)
|
||||
)
|
||||
|
||||
if stream:
|
||||
transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming(
|
||||
|
|
@ -956,6 +967,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
is_async=False,
|
||||
completion_request=completion_kwargs,
|
||||
litellm_logging_obj=logging_obj,
|
||||
held_completion=None if planned_chat is None else hold_completion_of(completion_response, logging_obj),
|
||||
)
|
||||
if transformed_stream is not None:
|
||||
return transformed_stream
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@ from litellm.types.utils import AdapterCompletionStreamWrapper, ChatCompletionDe
|
|||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
|
||||
from litellm.llms.anthropic.messages.held_stream_completion import HeldStreamCompletion
|
||||
from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator
|
||||
from litellm.types.utils import ModelResponseStream
|
||||
|
||||
|
|
@ -637,6 +638,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
|
||||
safeguards_evaluator: "SafeguardsEvaluator | None" = None,
|
||||
completion_request: Mapping[str, object] | None = None,
|
||||
held_completion: "HeldStreamCompletion | None" = None,
|
||||
):
|
||||
# Wrap the upstream stream so chunks that carry both content and a
|
||||
# finish_reason (fake-streamed providers) are split into two — see
|
||||
|
|
@ -651,6 +653,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
litellm_logging_obj.record_streamed_anthropic_message_id(self._message_id)
|
||||
self.litellm_logging_obj = litellm_logging_obj
|
||||
self.safeguards_evaluator: Final = safeguards_evaluator
|
||||
self._held_completion: Final = held_completion
|
||||
# Mapping of truncated tool names to original names (for OpenAI's 64-char limit)
|
||||
self.tool_name_mapping = tool_name_mapping or {}
|
||||
# Polyfill applied_edits on final message_delta.
|
||||
|
|
@ -1350,14 +1353,18 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
|
||||
This wrapper ensures dict chunks are SSE formatted with both event and data lines.
|
||||
"""
|
||||
for chunk in self:
|
||||
if isinstance(chunk, dict):
|
||||
event_type: str = str(chunk.get("type", "message"))
|
||||
payload = f"event: {event_type}\ndata: {json.dumps(chunk)}\n\n"
|
||||
yield payload.encode()
|
||||
else:
|
||||
# For non-dict chunks, forward the original value unchanged
|
||||
yield chunk
|
||||
try:
|
||||
for chunk in self:
|
||||
if isinstance(chunk, dict):
|
||||
event_type: str = str(chunk.get("type", "message"))
|
||||
payload = f"event: {event_type}\ndata: {json.dumps(chunk)}\n\n"
|
||||
yield payload.encode()
|
||||
else:
|
||||
# For non-dict chunks, forward the original value unchanged
|
||||
yield chunk
|
||||
finally:
|
||||
if self._held_completion is not None:
|
||||
self._held_completion.log(is_async=False)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
await self._combined_chunk_splitter.aclose()
|
||||
|
|
@ -1379,6 +1386,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
else:
|
||||
yield anthropic_sse_frame(await safeguard_results.observe(_SSE_EVENT.validate_python(chunk)))
|
||||
except Exception as e: # noqa: BLE001 # boundary before the socket: any upstream failure becomes an Anthropic error event
|
||||
if self._held_completion is not None:
|
||||
self._held_completion.discard()
|
||||
verbose_logger.exception("Anthropic Adapter - mid-stream error: %s", e)
|
||||
logging_obj: Final = self.litellm_logging_obj
|
||||
provider_error: Final = _provider_error(e)
|
||||
|
|
@ -1402,6 +1411,12 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
failure_handler_error,
|
||||
)
|
||||
yield _mid_stream_error_sse_event(e)
|
||||
else:
|
||||
if self._held_completion is not None:
|
||||
self._held_completion.log(is_async=True)
|
||||
finally:
|
||||
if self._held_completion is not None:
|
||||
self._held_completion.log(is_async=True, client_detached=True)
|
||||
|
||||
def _increment_content_block_index(self):
|
||||
self.current_content_block_index += 1
|
||||
|
|
|
|||
|
|
@ -213,6 +213,7 @@ from .streaming_iterator import AnthropicSSEStream, AnthropicStreamWrapper
|
|||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
|
||||
from litellm.llms.anthropic.messages.held_stream_completion import HeldStreamCompletion
|
||||
from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator
|
||||
from litellm.types.llms.anthropic import ContentBlockContentBlockDict
|
||||
|
||||
|
|
@ -326,6 +327,7 @@ class AnthropicAdapter:
|
|||
litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
|
||||
safeguards_evaluator: "SafeguardsEvaluator | None" = None,
|
||||
completion_request: Mapping[str, object] | None = None,
|
||||
held_completion: "HeldStreamCompletion | None" = None,
|
||||
) -> AsyncIterator[bytes] | Iterator[bytes] | None:
|
||||
"""
|
||||
Translate OpenAI streaming response to Anthropic format.
|
||||
|
|
@ -355,6 +357,7 @@ class AnthropicAdapter:
|
|||
litellm_logging_obj=litellm_logging_obj,
|
||||
safeguards_evaluator=safeguards_evaluator,
|
||||
completion_request=completion_request,
|
||||
held_completion=held_completion,
|
||||
)
|
||||
# Return the SSE-wrapped version for proper event formatting.
|
||||
if is_async:
|
||||
|
|
@ -1620,7 +1623,8 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
else translated_finish_reason
|
||||
)
|
||||
# extract usage
|
||||
usage: Final[Usage] = getattr(response, "usage")
|
||||
# A provider reply with no usage block leaves the response without one; it reads as zero tokens.
|
||||
usage: Final[Usage] = getattr(response, "usage", None) or Usage()
|
||||
message_usage: Final = self.translate_openai_usage_to_anthropic_usage(usage)
|
||||
polyfill_iterations: Final = polyfill_result.iterations_usage if polyfill_result is not None else None
|
||||
anthropic_usage: Final[AnthropicUsage] = (
|
||||
|
|
|
|||
64
litellm/llms/anthropic/messages/held_stream_completion.py
Normal file
64
litellm/llms/anthropic/messages/held_stream_completion.py
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING, Final, Protocol, runtime_checkable
|
||||
|
||||
from litellm.responses.streaming_iterator import dispatch_stream_success
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class _HandsOffCompletion(Protocol):
|
||||
def hand_completion_to(self, on_complete: Callable[[object], None]) -> None: ...
|
||||
|
||||
|
||||
class _NothingHeld:
|
||||
pass
|
||||
|
||||
|
||||
_NOTHING_HELD: Final = _NothingHeld()
|
||||
|
||||
|
||||
class HeldStreamCompletion:
|
||||
"""The executor stream's assembled response, logged by the client-facing Messages stream when it ends.
|
||||
|
||||
A stream that ended with no content hands over ``None``; it is still logged, as the success
|
||||
handler logs an empty stream when it is not held.
|
||||
"""
|
||||
|
||||
def __init__(self, logging_obj: "LiteLLMLoggingObj") -> None:
|
||||
self._logging_obj: Final = logging_obj
|
||||
self._response: object = _NOTHING_HELD
|
||||
|
||||
def take(self, response: object) -> None:
|
||||
self._response = response
|
||||
|
||||
def _release(self) -> object:
|
||||
response: Final = self._response
|
||||
self._response = _NOTHING_HELD
|
||||
return response
|
||||
|
||||
def discard(self) -> None:
|
||||
self._release()
|
||||
|
||||
def log(self, *, is_async: bool, client_detached: bool = False) -> None:
|
||||
response: Final = self._release()
|
||||
if response is _NOTHING_HELD:
|
||||
return
|
||||
dispatch_stream_success(
|
||||
self._logging_obj,
|
||||
response,
|
||||
start_time=None,
|
||||
end_time=None,
|
||||
cache_hit=None,
|
||||
is_async=is_async,
|
||||
client_detached=client_detached,
|
||||
)
|
||||
|
||||
|
||||
def hold_completion_of(stream: object, logging_obj: "LiteLLMLoggingObj | None") -> HeldStreamCompletion | None:
|
||||
if logging_obj is None or not isinstance(stream, _HandsOffCompletion):
|
||||
return None
|
||||
held: Final = HeldStreamCompletion(logging_obj)
|
||||
stream.hand_completion_to(held.take)
|
||||
return held
|
||||
|
|
@ -16,7 +16,6 @@ from pydantic import TypeAdapter
|
|||
import litellm
|
||||
from litellm.anthropic_beta_headers_manager import aload_beta_headers_config
|
||||
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
|
|
@ -29,23 +28,17 @@ from litellm.llms.base_llm.anthropic_messages.transformation import (
|
|||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from litellm.native.messages import execute_messages
|
||||
from litellm.native.route_plan import NativeChat, NativeResponses, native_messages_config, plan_messages_route
|
||||
from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway
|
||||
from litellm.router_utils.reasoning_effort_capability import (
|
||||
RequestedReasoningEffort,
|
||||
deployment_reasoning_effort_levels,
|
||||
keeps_effort,
|
||||
model_group_from_metadata,
|
||||
resolve_deployment_reasoning_effort,
|
||||
with_adaptive_thinking_as_effort,
|
||||
)
|
||||
from litellm.types.llms.anthropic import AllAnthropicToolsValues
|
||||
from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import (
|
||||
AnthropicMessagesResponse,
|
||||
)
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import CallTypes, LlmProviders
|
||||
from litellm.utils import ProviderConfigManager, client
|
||||
from litellm.types.utils import CallTypes
|
||||
from litellm.utils import client
|
||||
|
||||
from ..adapters.handler import LiteLLMMessagesToCompletionTransformationHandler, request_safeguards_evaluator
|
||||
from ..responses_adapters.handler import LiteLLMMessagesToResponsesAPIHandler
|
||||
|
|
@ -56,69 +49,14 @@ from ..safeguards import (
|
|||
with_safeguard_results,
|
||||
)
|
||||
from .interceptors import get_messages_interceptors
|
||||
from .utils import AnthropicMessagesRequestUtils, mock_response, prepare_native_messages
|
||||
from .utils import mock_response, prepare_native_messages
|
||||
|
||||
__all__ = ("anthropic_messages", "anthropic_messages_handler")
|
||||
|
||||
# Providers that are routed directly to the OpenAI Responses API instead of
|
||||
# going through chat/completions.
|
||||
_RESPONSES_API_PROVIDERS: Final = frozenset({"openai"})
|
||||
_SAFEGUARD_KWARGS: Final = TypeAdapter(Mapping[str, object])
|
||||
_SAFEGUARD_MESSAGES: Final = TypeAdapter(Sequence[Mapping[str, object]])
|
||||
|
||||
|
||||
def _bridges_to_responses_api(model: str, custom_llm_provider: str, deployment_model_info: object) -> bool:
|
||||
from litellm.responses.bridge_check import responses_api_bridge_check, str_keyed_mapping
|
||||
|
||||
model_info, _ = responses_api_bridge_check(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
deployment_model_info=str_keyed_mapping(deployment_model_info),
|
||||
)
|
||||
return model_info.get("mode") == "responses"
|
||||
|
||||
|
||||
def _responses_mode_is_lost_by_prefix_strip(
|
||||
requested_model: str, resolved_model: str, custom_llm_provider: str, deployment_model_info: object
|
||||
) -> bool:
|
||||
"""Whether a Responses-only deployment stops looking like one once its provider prefix is stripped.
|
||||
|
||||
``litellm.completion`` re-derives the Responses bridge from the stripped id alone, so a
|
||||
deployment id such as ``perplexity/perplexity/sonar`` (mode ``responses``) is shadowed by the
|
||||
chat entry ``perplexity/sonar`` and would otherwise be sent to chat/completions.
|
||||
"""
|
||||
if requested_model == resolved_model:
|
||||
return False
|
||||
return _bridges_to_responses_api(
|
||||
requested_model, custom_llm_provider, deployment_model_info
|
||||
) and not _bridges_to_responses_api(resolved_model, custom_llm_provider, deployment_model_info)
|
||||
|
||||
|
||||
def _should_route_to_responses_api(
|
||||
custom_llm_provider: str | None,
|
||||
requested_model: str | None = None,
|
||||
resolved_model: str | None = None,
|
||||
deployment_model_info: object = None,
|
||||
stop_sequences: Sequence[str] | None = None,
|
||||
) -> bool:
|
||||
"""Return True when the request should use the Responses API path.
|
||||
|
||||
Set ``litellm.use_chat_completions_url_for_anthropic_messages = True`` to
|
||||
opt out and route OpenAI/Azure requests through chat/completions instead.
|
||||
"""
|
||||
if litellm.use_chat_completions_url_for_anthropic_messages:
|
||||
return False
|
||||
if stop_sequences:
|
||||
return False
|
||||
if custom_llm_provider in _RESPONSES_API_PROVIDERS:
|
||||
return True
|
||||
if custom_llm_provider is None or requested_model is None or resolved_model is None:
|
||||
return False
|
||||
return _responses_mode_is_lost_by_prefix_strip(
|
||||
requested_model, resolved_model, custom_llm_provider, deployment_model_info
|
||||
)
|
||||
|
||||
|
||||
async def _with_native_safeguard_results(
|
||||
pending: Coroutine[object, object, AnthropicMessagesResponse | AsyncIterator[bytes]],
|
||||
proxy_classifies: Callable[[], Awaitable[bool]],
|
||||
|
|
@ -146,88 +84,6 @@ def _native_safeguards_evaluator(kwargs: Mapping[str, object], messages: object)
|
|||
return request_safeguards_evaluator(kwargs, _SAFEGUARD_MESSAGES.validate_python(messages))
|
||||
|
||||
|
||||
def _deployment_passes_through_anthropic_messages(model_info: object) -> bool:
|
||||
"""Whether the deployment opted into forwarding /v1/messages untranslated.
|
||||
|
||||
The opt-in is ``model_info.supported_endpoints`` containing ``"/v1/messages"``,
|
||||
declared per deployment in config.yaml and plumbed here as ``kwargs["model_info"]``
|
||||
by the router.
|
||||
"""
|
||||
if not isinstance(model_info, dict):
|
||||
return False
|
||||
supported_endpoints: Final = model_info.get("supported_endpoints")
|
||||
return isinstance(supported_endpoints, (list, tuple)) and "/v1/messages" in supported_endpoints
|
||||
|
||||
|
||||
def _with_output_config(params: Mapping[str, object], output_config: RequestedReasoningEffort) -> Mapping[str, object]:
|
||||
rest: Final = {key: value for key, value in params.items() if key != "output_config"}
|
||||
return rest if output_config is None else {**rest, "output_config": output_config}
|
||||
|
||||
|
||||
def _with_deployment_output_effort(
|
||||
params: Mapping[str, object],
|
||||
*,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
kwargs: Mapping[str, object],
|
||||
drop_params: bool,
|
||||
) -> Mapping[str, object]:
|
||||
deployment_model_info: Final = kwargs.get("model_info")
|
||||
|
||||
def resolve(requested: RequestedReasoningEffort) -> RequestedReasoningEffort:
|
||||
return resolve_deployment_reasoning_effort(
|
||||
requested,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
deployment_model_info=deployment_model_info,
|
||||
model_group=model_group_from_metadata(kwargs.get("litellm_metadata"), kwargs.get("metadata")),
|
||||
drop_params=drop_params,
|
||||
).value
|
||||
|
||||
def translate(source: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return with_adaptive_thinking_as_effort(
|
||||
source, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params
|
||||
)
|
||||
|
||||
requested_output: Final = as_str_mapping(params.get("output_config"))
|
||||
caller_output: Final = None if requested_output is None else resolve(requested_output)
|
||||
caller_params: Final = params if requested_output is None else _with_output_config(params, caller_output)
|
||||
if deployment_reasoning_effort_levels(deployment_model_info) is None:
|
||||
return caller_params
|
||||
if requested_output is not None and keeps_effort(requested_output, caller_output):
|
||||
return _with_output_config(translate(params), caller_output)
|
||||
translated: Final = translate(caller_params)
|
||||
budget_output: Final = as_str_mapping(translated.get("output_config"))
|
||||
return translated if budget_output is None else _with_output_config(translated, resolve(budget_output))
|
||||
|
||||
|
||||
def _deployment_supports_cache_control_ttl(model_info: object) -> bool:
|
||||
return isinstance(model_info, dict) and model_info.get("cache_control_ttl") is True
|
||||
|
||||
|
||||
def _native_messages_config(
|
||||
model: str, custom_llm_provider: str | None, model_info: object, api_base: str | None
|
||||
) -> BaseAnthropicMessagesConfig | None:
|
||||
provider_config: Final = (
|
||||
ProviderConfigManager.get_provider_anthropic_messages_config(
|
||||
model=model,
|
||||
provider=litellm.LlmProviders(custom_llm_provider),
|
||||
)
|
||||
if custom_llm_provider is not None and custom_llm_provider in [provider.value for provider in LlmProviders]
|
||||
else None
|
||||
)
|
||||
native_config: Final = (
|
||||
provider_config if provider_config is not None and provider_config.serves_api_base(model, api_base) else None
|
||||
)
|
||||
if native_config is not None or not _deployment_passes_through_anthropic_messages(model_info):
|
||||
return native_config
|
||||
from litellm.llms.openai_like.messages.transformation import (
|
||||
OpenAILikeAnthropicMessagesConfig,
|
||||
)
|
||||
|
||||
return OpenAILikeAnthropicMessagesConfig(cache_control_ttl=_deployment_supports_cache_control_ttl(model_info))
|
||||
|
||||
|
||||
async def _proxy_classifies(model: str, safeguards: object, native_config: BaseAnthropicMessagesConfig | None) -> bool:
|
||||
if not proxy_serves_safeguards(safeguards):
|
||||
return False
|
||||
|
|
@ -244,7 +100,7 @@ async def proxy_classifies_tool_uses(model: str, custom_llm_provider: str | None
|
|||
return await _proxy_classifies(
|
||||
model,
|
||||
kwargs.get("safeguards"),
|
||||
_native_messages_config(
|
||||
native_messages_config(
|
||||
model, custom_llm_provider, kwargs.get("model_info"), api_base if isinstance(api_base, str) else None
|
||||
),
|
||||
)
|
||||
|
|
@ -687,40 +543,44 @@ def anthropic_messages_handler(
|
|||
**kwargs,
|
||||
)
|
||||
|
||||
anthropic_messages_provider_config: Final = _native_messages_config(
|
||||
model, custom_llm_provider, kwargs.get("model_info"), api_base
|
||||
route: Final = plan_messages_route(
|
||||
requested_model=original_model,
|
||||
resolved_model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
model_info=kwargs.get("model_info"),
|
||||
api_base=api_base,
|
||||
compaction_requested=kwargs.get("compaction") is not None,
|
||||
force_chat_completions=bool(litellm.use_chat_completions_url_for_anthropic_messages),
|
||||
stop_requested=bool(stop_sequences),
|
||||
)
|
||||
if anthropic_messages_provider_config is None:
|
||||
# Route to Responses API for OpenAI / Azure, chat/completions for everything else.
|
||||
if kwargs.get("compaction") is None and _should_route_to_responses_api(
|
||||
custom_llm_provider, original_model, model, kwargs.get("model_info"), stop_sequences=stop_sequences
|
||||
):
|
||||
return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
|
||||
max_tokens=max_tokens,
|
||||
messages=messages,
|
||||
model=original_model,
|
||||
metadata=metadata,
|
||||
stop_sequences=stop_sequences,
|
||||
stream=stream,
|
||||
system=cast( # cast-ok: anthropic accepts list system blocks; handler signature is narrower
|
||||
"str | None", system
|
||||
),
|
||||
temperature=temperature,
|
||||
thinking=thinking,
|
||||
tool_choice=tool_choice,
|
||||
tools=cast( # cast-ok: dict tools already fit the handler's union
|
||||
"list[AllAnthropicToolsValues | dict[str, object]] | None", tools
|
||||
),
|
||||
top_k=top_k,
|
||||
top_p=top_p,
|
||||
_is_async=is_async,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
**kwargs,
|
||||
)
|
||||
if isinstance(route, NativeResponses):
|
||||
return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
|
||||
max_tokens=max_tokens,
|
||||
messages=messages,
|
||||
model=original_model,
|
||||
metadata=metadata,
|
||||
stop_sequences=stop_sequences,
|
||||
stream=stream,
|
||||
system=cast( # cast-ok: anthropic accepts list system blocks; handler signature is narrower
|
||||
"str | None", system
|
||||
),
|
||||
temperature=temperature,
|
||||
thinking=thinking,
|
||||
tool_choice=tool_choice,
|
||||
tools=cast( # cast-ok: dict tools already fit the handler's union
|
||||
"list[AllAnthropicToolsValues | dict[str, object]] | None", tools
|
||||
),
|
||||
top_k=top_k,
|
||||
top_p=top_p,
|
||||
_is_async=is_async,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
if isinstance(route, NativeChat):
|
||||
# The in-gateway context_management polyfill runs inside
|
||||
# ``async_anthropic_messages_handler`` so it can ``await`` the
|
||||
# summarization model for ``compact_20260112``. ``context_management``
|
||||
|
|
@ -761,23 +621,11 @@ def anthropic_messages_handler(
|
|||
if marked_tools is not None:
|
||||
AnthropicCacheControlHook.record_gateway_injection(request_kwargs, 1)
|
||||
local_vars.update({**request_kwargs, **({"tools": list(marked_tools)} if marked_tools is not None else {})})
|
||||
anthropic_messages_optional_request_params: Final = (
|
||||
AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param(
|
||||
params=local_vars,
|
||||
model=model,
|
||||
drop_params=litellm_params.get("drop_params") is True,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
)
|
||||
resolved_api_base: Final = (
|
||||
dynamic_api_base
|
||||
if dynamic_api_base is not None and anthropic_messages_provider_config.uses_get_llm_provider_api_base()
|
||||
else api_base
|
||||
)
|
||||
safeguard_kwargs: Final = (
|
||||
_SAFEGUARD_KWARGS.validate_python(kwargs) if is_async is True and "safeguards" in kwargs else None
|
||||
)
|
||||
native_response: Final = base_llm_http_handler.anthropic_messages_handler(
|
||||
native_response: Final = execute_messages(
|
||||
http_handler=base_llm_http_handler,
|
||||
model=model,
|
||||
messages=cast( # cast-ok: anthropic messages are dicts at runtime
|
||||
"list[dict[str, object]]",
|
||||
|
|
@ -785,30 +633,23 @@ def anthropic_messages_handler(
|
|||
cast(Sequence[object], messages) # cast-ok: anthropic payloads arrive as untyped dicts
|
||||
),
|
||||
),
|
||||
anthropic_messages_provider_config=anthropic_messages_provider_config,
|
||||
anthropic_messages_optional_request_params=dict(
|
||||
_with_deployment_output_effort(
|
||||
anthropic_messages_optional_request_params,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
kwargs=cast(Mapping[str, object], kwargs), # cast-ok: router kwargs arrive untyped
|
||||
drop_params=litellm.drop_params is True or litellm_params.get("drop_params") is True,
|
||||
)
|
||||
),
|
||||
_is_async=is_async,
|
||||
client=client,
|
||||
provider_config=route.config,
|
||||
request_params=local_vars,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params,
|
||||
logging_obj=litellm_logging_obj,
|
||||
api_key=api_key,
|
||||
api_base=resolved_api_base,
|
||||
api_base=api_base,
|
||||
dynamic_api_base=dynamic_api_base,
|
||||
client=client,
|
||||
is_async=is_async,
|
||||
stream=stream,
|
||||
kwargs=kwargs,
|
||||
request_kwargs=kwargs,
|
||||
)
|
||||
if safeguard_kwargs is None or not isinstance(native_response, Coroutine):
|
||||
return native_response
|
||||
return _with_native_safeguard_results(
|
||||
native_response,
|
||||
partial(_proxy_classifies, model, safeguard_kwargs.get("safeguards"), anthropic_messages_provider_config),
|
||||
partial(_proxy_classifies, model, safeguard_kwargs.get("safeguards"), route.config),
|
||||
partial(_native_safeguards_evaluator, safeguard_kwargs, messages),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -19,11 +19,14 @@ from litellm.litellm_core_utils.asyncify import (
|
|||
run_async_function, # pyright: ignore[reportUnknownVariableType] # untyped helper
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
cache_control_as_prompt_cache_breakpoint,
|
||||
prompt_cache_breakpoint_for_wire,
|
||||
with_prompt_cache_breakpoint,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.held_stream_completion import hold_completion_of
|
||||
from litellm.responses.bridge_executor import anative_responses, native_logging_obj, native_responses
|
||||
from litellm.responses.utils import auto_reasoning_summary
|
||||
from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS
|
||||
from litellm.types.llms.anthropic import (
|
||||
|
|
@ -38,7 +41,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
|
|||
AnthropicMessagesResponse,
|
||||
)
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.types.utils import CountContextManagement
|
||||
from litellm.types.utils import CallTypes, CountContextManagement
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
from ..adapters.handler import context_management_explicitly_dropped, request_safeguards_evaluator
|
||||
|
|
@ -159,6 +162,9 @@ def _with_applied_edits(
|
|||
return {**response, "context_management": ContextManagementResponse(applied_edits=list(applied_edits))}
|
||||
|
||||
|
||||
_NEVER_FORWARDED: Final = CONVERTED_STREAM_KEYS | {"anthropic_messages"}
|
||||
|
||||
|
||||
def _record_bridged_response(responses_kwargs: Mapping[str, object], result: ResponsesAPIResponse) -> None:
|
||||
logging_obj: Final = litellm_logging_obj_from_kwargs(responses_kwargs)
|
||||
if logging_obj is not None:
|
||||
|
|
@ -166,8 +172,10 @@ def _record_bridged_response(responses_kwargs: Mapping[str, object], result: Res
|
|||
|
||||
|
||||
def _forwarded_kwargs(extra_kwargs: Mapping[str, object] | None) -> Mapping[str, object]:
|
||||
"""The litellm-specific kwargs forwarded verbatim onto the Responses API request."""
|
||||
return extra_kwargs or {}
|
||||
"""The litellm-specific kwargs (credentials, routing, metadata) that carry over to the Responses API call."""
|
||||
return {
|
||||
key: value for key, value in (extra_kwargs or {}).items() if key not in _NEVER_FORWARDED and value is not None
|
||||
}
|
||||
|
||||
|
||||
def _provider_returns_encrypted_reasoning(model: str, custom_llm_provider: object) -> bool:
|
||||
|
|
@ -247,7 +255,7 @@ def _rewritten_blocks(content: object, rewrite: BlockRewrite) -> Sequence[Mappin
|
|||
return ()
|
||||
|
||||
|
||||
def _build_responses_kwargs(
|
||||
def _build_responses_request(
|
||||
*,
|
||||
max_tokens: int,
|
||||
messages: AnthropicRequestMessages,
|
||||
|
|
@ -267,9 +275,7 @@ def _build_responses_kwargs(
|
|||
output_format: AnthropicOutputSchema | None = None,
|
||||
extra_kwargs: Mapping[str, object] | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""
|
||||
Build the kwargs dict to pass directly to litellm.responses() / litellm.aresponses().
|
||||
"""
|
||||
"""The Responses API request the Messages request translates to, without the forwarded litellm kwargs."""
|
||||
forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs)
|
||||
reject_malformed_tool_result_text(messages, model, forwarded_kwargs)
|
||||
block_rewrite: Final = _prompt_cache_block_rewrite(model, forwarded_kwargs)
|
||||
|
|
@ -314,7 +320,7 @@ def _build_responses_kwargs(
|
|||
request_data["output_format"] = output_format
|
||||
|
||||
anthropic_request: Final = AnthropicMessagesRequest(**request_data)
|
||||
responses_kwargs: Final = _ADAPTER.translate_request(
|
||||
responses_request: Final = _ADAPTER.translate_request(
|
||||
anthropic_request,
|
||||
include_encrypted_reasoning=_provider_returns_encrypted_reasoning(
|
||||
model, forwarded_kwargs.get("custom_llm_provider")
|
||||
|
|
@ -325,7 +331,7 @@ def _build_responses_kwargs(
|
|||
|
||||
# Normalize reasoning effort based on model capabilities
|
||||
# (e.g. "max" → "xhigh"/"high", "minimal" → "low" if unsupported)
|
||||
reasoning: Final = responses_kwargs.get("reasoning")
|
||||
reasoning: Final = responses_request.get("reasoning")
|
||||
if isinstance(reasoning, dict):
|
||||
effort: Final[object] = reasoning.get("effort")
|
||||
if isinstance(effort, str):
|
||||
|
|
@ -341,49 +347,39 @@ def _build_responses_kwargs(
|
|||
deployment_model_info=forwarded_kwargs.get("model_info"),
|
||||
)
|
||||
if normalized != effort:
|
||||
responses_kwargs["reasoning"] = {**reasoning, "effort": normalized}
|
||||
responses_request["reasoning"] = {**reasoning, "effort": normalized}
|
||||
|
||||
if stream:
|
||||
responses_kwargs["stream"] = True
|
||||
|
||||
# Forward litellm-specific kwargs (api_key, api_base, logging obj, etc.)
|
||||
excluded: Final = CONVERTED_STREAM_KEYS | {"anthropic_messages"}
|
||||
for key, value in forwarded_kwargs.items():
|
||||
if key == "litellm_logging_obj" and value is not None:
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
Logging as LiteLLMLoggingObject,
|
||||
)
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
if isinstance(value, LiteLLMLoggingObject):
|
||||
# Keep call_type as anthropic_messages so spend_logs are billed
|
||||
# against /v1/messages; the success handler translates the
|
||||
# Responses API result back to a ModelResponse for the row.
|
||||
setattr(value, "call_type", CallTypes.anthropic_messages.value)
|
||||
responses_kwargs[key] = value
|
||||
elif key not in excluded and key not in responses_kwargs and value is not None:
|
||||
responses_kwargs[key] = value
|
||||
responses_request["stream"] = True
|
||||
|
||||
explicit_prompt_cache_key: Final = forwarded_kwargs.get("prompt_cache_key")
|
||||
if explicit_prompt_cache_key is not None:
|
||||
responses_kwargs["prompt_cache_key"] = explicit_prompt_cache_key
|
||||
responses_request["prompt_cache_key"] = explicit_prompt_cache_key
|
||||
|
||||
deployment_include: Final = forwarded_kwargs.get("include")
|
||||
bridge_include: Final = responses_kwargs.get("include")
|
||||
bridge_include: Final = responses_request.get("include")
|
||||
if isinstance(deployment_include, list) and isinstance(bridge_include, list):
|
||||
responses_kwargs["include"] = [
|
||||
responses_request["include"] = [
|
||||
*bridge_include,
|
||||
*(item for item in deployment_include if item not in bridge_include),
|
||||
]
|
||||
|
||||
return responses_kwargs
|
||||
return responses_request
|
||||
|
||||
|
||||
def _reentrant_kwargs(
|
||||
forwarded_kwargs: Mapping[str, object], request: Mapping[str, object], logging_obj: LiteLLMLoggingObj | None
|
||||
) -> Mapping[str, object]:
|
||||
if logging_obj is not None:
|
||||
setattr(logging_obj, "call_type", CallTypes.anthropic_messages.value)
|
||||
return {**forwarded_kwargs, **request}
|
||||
|
||||
|
||||
class LiteLLMMessagesToResponsesAPIHandler:
|
||||
"""
|
||||
Handles Anthropic /v1/messages requests for OpenAI / Azure models by
|
||||
calling litellm.responses() / litellm.aresponses() directly and translating
|
||||
the response back to Anthropic format.
|
||||
calling the native Responses executor and translating the response back
|
||||
to Anthropic format.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -410,7 +406,7 @@ class LiteLLMMessagesToResponsesAPIHandler:
|
|||
polyfill_result: Final = await _polyfilled(
|
||||
model=model, messages=messages, tools=tools, system=system, edits=polyfilled_edits, kwargs=kwargs
|
||||
)
|
||||
responses_kwargs: Final = _build_responses_kwargs(
|
||||
request: Final = _build_responses_request(
|
||||
max_tokens=max_tokens,
|
||||
messages=polyfill_result.messages if polyfill_result is not None else messages,
|
||||
model=model,
|
||||
|
|
@ -429,17 +425,25 @@ class LiteLLMMessagesToResponsesAPIHandler:
|
|||
output_format=output_format,
|
||||
extra_kwargs=kwargs,
|
||||
)
|
||||
forwarded_kwargs: Final = _forwarded_kwargs(kwargs)
|
||||
logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs)
|
||||
|
||||
safeguards_evaluator: Final = request_safeguards_evaluator(kwargs, messages)
|
||||
result: Final = await litellm.aresponses(**responses_kwargs)
|
||||
native_logging: Final = native_logging_obj(logging_obj, forwarded_kwargs, request)
|
||||
result: Final = (
|
||||
await litellm.aresponses(**_reentrant_kwargs(forwarded_kwargs, request, logging_obj))
|
||||
if native_logging is None
|
||||
else await anative_responses(request, model, forwarded_kwargs, native_logging)
|
||||
)
|
||||
|
||||
if stream:
|
||||
wrapper: Final = AnthropicResponsesStreamWrapper(
|
||||
responses_stream=result,
|
||||
model=local_model_name(model, kwargs.get("custom_llm_provider")),
|
||||
litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
|
||||
litellm_logging_obj=logging_obj,
|
||||
safeguards_evaluator=safeguards_evaluator,
|
||||
applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None,
|
||||
held_completion=None if native_logging is None else hold_completion_of(result, native_logging),
|
||||
)
|
||||
return wrapper.async_anthropic_sse_wrapper()
|
||||
|
||||
|
|
@ -449,7 +453,7 @@ class LiteLLMMessagesToResponsesAPIHandler:
|
|||
served: Final = await with_safeguard_results(
|
||||
_with_applied_edits(_ADAPTER.translate_response(result), polyfill_result), safeguards_evaluator
|
||||
)
|
||||
_record_bridged_response(responses_kwargs, result)
|
||||
_record_bridged_response(kwargs, result)
|
||||
return served
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -514,7 +518,7 @@ class LiteLLMMessagesToResponsesAPIHandler:
|
|||
else None
|
||||
)
|
||||
polyfill_result: Final = ran if isinstance(ran, PolyfillResult) else None
|
||||
responses_kwargs: Final = _build_responses_kwargs(
|
||||
request: Final = _build_responses_request(
|
||||
max_tokens=max_tokens,
|
||||
messages=polyfill_result.messages if polyfill_result is not None else messages,
|
||||
model=model,
|
||||
|
|
@ -533,15 +537,22 @@ class LiteLLMMessagesToResponsesAPIHandler:
|
|||
output_format=output_format,
|
||||
extra_kwargs=kwargs,
|
||||
)
|
||||
|
||||
result: Final = litellm.responses(**responses_kwargs)
|
||||
forwarded_kwargs: Final = _forwarded_kwargs(kwargs)
|
||||
logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs)
|
||||
native_logging: Final = native_logging_obj(logging_obj, forwarded_kwargs, request)
|
||||
result: Final = (
|
||||
litellm.responses(**_reentrant_kwargs(forwarded_kwargs, request, logging_obj))
|
||||
if native_logging is None
|
||||
else native_responses(request, model, forwarded_kwargs, native_logging)
|
||||
)
|
||||
|
||||
if stream:
|
||||
wrapper: Final = AnthropicResponsesStreamWrapper(
|
||||
responses_stream=result,
|
||||
model=local_model_name(model, kwargs.get("custom_llm_provider")),
|
||||
litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
|
||||
litellm_logging_obj=logging_obj,
|
||||
applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None,
|
||||
held_completion=None if native_logging is None else hold_completion_of(result, native_logging),
|
||||
)
|
||||
return wrapper.async_anthropic_sse_wrapper()
|
||||
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ from .transformation import (
|
|||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
|
||||
from litellm.llms.anthropic.messages.held_stream_completion import HeldStreamCompletion
|
||||
from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator
|
||||
from litellm.types.llms.openai import ResponseIncompleteEvent
|
||||
|
||||
|
|
@ -131,12 +132,14 @@ class AnthropicResponsesStreamWrapper:
|
|||
litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
|
||||
safeguards_evaluator: "SafeguardsEvaluator | None" = None,
|
||||
applied_edits: Sequence[AppliedEdit] | None = None,
|
||||
held_completion: "HeldStreamCompletion | None" = None,
|
||||
) -> None:
|
||||
self.responses_stream = responses_stream
|
||||
self._applied_edits: Final = tuple(applied_edits or ())
|
||||
self.model = model
|
||||
self.litellm_logging_obj = litellm_logging_obj
|
||||
self.safeguards_evaluator: Final = safeguards_evaluator
|
||||
self._held_completion: Final = held_completion
|
||||
self._message_id: str = f"msg_{uuid.uuid4()}"
|
||||
if litellm_logging_obj is not None:
|
||||
litellm_logging_obj.record_streamed_anthropic_message_id(self._message_id)
|
||||
|
|
@ -545,10 +548,22 @@ class AnthropicResponsesStreamWrapper:
|
|||
|
||||
async def _sse_frames(self) -> AsyncGenerator[bytes, None]:
|
||||
safeguard_results: Final = StreamedSafeguardResults(self.safeguards_evaluator)
|
||||
held: Final = self._held_completion
|
||||
try:
|
||||
async for chunk in self:
|
||||
yield anthropic_sse_frame(await safeguard_results.observe(chunk))
|
||||
except Exception:
|
||||
if held is not None:
|
||||
held.discard()
|
||||
raise
|
||||
else:
|
||||
if held is not None and self._stream_failed:
|
||||
held.discard()
|
||||
if held is not None:
|
||||
held.log(is_async=True)
|
||||
finally:
|
||||
if held is not None:
|
||||
held.log(is_async=True, client_detached=True)
|
||||
from litellm.llms.anthropic.messages.native.streaming_iterator import closing_superseded_attempt
|
||||
|
||||
if not closing_superseded_attempt():
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import asyncio
|
||||
import json
|
||||
import time
|
||||
from collections.abc import Callable, Coroutine
|
||||
from collections.abc import Callable, Coroutine, Mapping
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -114,7 +114,7 @@ class AzureOpenAIAssistantsAPIConfig:
|
|||
|
||||
|
||||
def check_dynamic_azure_params(
|
||||
azure_client_params: dict,
|
||||
azure_client_params: Mapping[str, object],
|
||||
azure_client: AzureOpenAI | AsyncAzureOpenAI | None,
|
||||
) -> bool:
|
||||
"""
|
||||
|
|
|
|||
5905
litellm/main.py
5905
litellm/main.py
File diff suppressed because it is too large
Load diff
|
|
@ -3,9 +3,11 @@ from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Itera
|
|||
from types import MappingProxyType
|
||||
from typing import Final, TypeAlias, cast # noqa: TID251 # native binding selects a sync result or an async awaitable
|
||||
|
||||
import litellm
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.llms.anthropic.messages.native import handler as main
|
||||
from litellm.native.route_plan import NativeMessages, NativeResponses, plan_messages_route
|
||||
from litellm.rust_bridge import detectors
|
||||
from litellm.rust_bridge.catalog import Feature, Route, RouteContext, Target
|
||||
from litellm.rust_bridge.dispatch import PublicDispatch
|
||||
|
|
@ -86,16 +88,20 @@ def _resolved(request: NativeCall) -> tuple[str, str | None]:
|
|||
|
||||
|
||||
def _target(request: NativeCall, model: str, provider: str | None) -> Target:
|
||||
requested_model: Final = str(request.resolved["model"])
|
||||
model_info: Final = request.resolved.get("model_info")
|
||||
api_base: Final = request.resolved.get("api_base")
|
||||
native_config: Final = main._native_messages_config( # pyright: ignore[reportPrivateUsage] # the handler's own passthrough lookup
|
||||
model, provider, model_info, api_base if isinstance(api_base, str) else None
|
||||
route: Final = plan_messages_route(
|
||||
requested_model=str(request.resolved["model"]),
|
||||
resolved_model=model,
|
||||
custom_llm_provider=provider,
|
||||
model_info=request.resolved.get("model_info"),
|
||||
api_base=api_base if isinstance(api_base, str) else None,
|
||||
compaction_requested=request.resolved.get("compaction") is not None,
|
||||
force_chat_completions=bool(litellm.use_chat_completions_url_for_anthropic_messages),
|
||||
stop_requested=bool(request.resolved.get("stop_sequences")),
|
||||
)
|
||||
if native_config is not None:
|
||||
if isinstance(route, NativeMessages):
|
||||
return Target.MESSAGES
|
||||
compacting: Final = request.resolved.get("compaction") is not None
|
||||
if not compacting and main._should_route_to_responses_api(provider, requested_model, model, model_info): # pyright: ignore[reportPrivateUsage] # the handler's own translation choice
|
||||
if isinstance(route, NativeResponses):
|
||||
return Target.RESPONSES
|
||||
return Target.CHAT_COMPLETIONS
|
||||
|
||||
|
|
|
|||
1342
litellm/native/chat.py
Normal file
1342
litellm/native/chat.py
Normal file
File diff suppressed because it is too large
Load diff
281
litellm/native/chat_mock.py
Normal file
281
litellm/native/chat_mock.py
Normal file
|
|
@ -0,0 +1,281 @@
|
|||
import asyncio
|
||||
import time
|
||||
from typing import (
|
||||
Final,
|
||||
cast, # noqa: TID251 # casts moved verbatim from litellm/main.py
|
||||
)
|
||||
|
||||
import httpx
|
||||
import openai
|
||||
|
||||
import litellm
|
||||
from litellm.constants import DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT
|
||||
from litellm.llms.base_llm.base_model_iterator import convert_model_response_to_streaming
|
||||
from litellm.native.request_support import admitted_prompt_token_counter
|
||||
from litellm.types.utils import ChatCompletionMessageToolCall, ModelResponseStream
|
||||
from litellm.utils import (
|
||||
CustomStreamWrapper,
|
||||
ModelResponse,
|
||||
Usage,
|
||||
async_mock_completion_streaming_obj,
|
||||
mock_completion_streaming_obj,
|
||||
)
|
||||
|
||||
MOCK_RESPONSE_TYPE = str | Exception | dict | ModelResponse | ModelResponseStream
|
||||
|
||||
|
||||
def handle_mock_potential_exceptions(
|
||||
mock_response: str | Exception,
|
||||
model: str,
|
||||
custom_llm_provider: str | None = None,
|
||||
) -> None:
|
||||
if isinstance(mock_response, Exception):
|
||||
if isinstance(mock_response, openai.APIError):
|
||||
raise mock_response
|
||||
raise litellm.MockException(
|
||||
status_code=getattr(mock_response, "status_code", 500),
|
||||
message=getattr(mock_response, "text", str(mock_response)),
|
||||
llm_provider=getattr(mock_response, "llm_provider", custom_llm_provider or "openai"),
|
||||
model=model,
|
||||
request=httpx.Request(method="POST", url="https://api.openai.com/v1/"),
|
||||
)
|
||||
elif isinstance(mock_response, str) and mock_response == "litellm.RateLimitError":
|
||||
raise litellm.RateLimitError(
|
||||
message="this is a mock rate limit error",
|
||||
llm_provider=getattr(mock_response, "llm_provider", custom_llm_provider or "openai"),
|
||||
model=model,
|
||||
)
|
||||
elif isinstance(mock_response, str) and mock_response == "litellm.ContextWindowExceededError":
|
||||
raise litellm.ContextWindowExceededError(
|
||||
message="this is a mock context window exceeded error",
|
||||
llm_provider=getattr(mock_response, "llm_provider", custom_llm_provider or "openai"),
|
||||
model=model,
|
||||
)
|
||||
elif isinstance(mock_response, str) and mock_response == "litellm.InternalServerError":
|
||||
raise litellm.InternalServerError(
|
||||
message="this is a mock internal server error",
|
||||
llm_provider=getattr(mock_response, "llm_provider", custom_llm_provider or "openai"),
|
||||
model=model,
|
||||
)
|
||||
elif isinstance(mock_response, str) and mock_response.startswith("Exception: content_filter_policy"):
|
||||
raise litellm.MockException(
|
||||
status_code=400,
|
||||
message=mock_response,
|
||||
llm_provider="azure",
|
||||
model=model,
|
||||
request=httpx.Request(method="POST", url="https://api.openai.com/v1/"),
|
||||
)
|
||||
|
||||
|
||||
def handle_mock_timeout(
|
||||
mock_timeout: bool | None,
|
||||
timeout: float | str | httpx.Timeout | None,
|
||||
model: str,
|
||||
) -> None:
|
||||
if mock_timeout is True and timeout is not None:
|
||||
_sleep_for_timeout(timeout)
|
||||
raise litellm.Timeout(
|
||||
message="This is a mock timeout error",
|
||||
llm_provider="openai",
|
||||
model=model,
|
||||
)
|
||||
|
||||
|
||||
async def handle_mock_timeout_async(
|
||||
mock_timeout: bool | None,
|
||||
timeout: float | str | httpx.Timeout | None,
|
||||
model: str,
|
||||
) -> None:
|
||||
if mock_timeout is True and timeout is not None:
|
||||
await _sleep_for_timeout_async(timeout)
|
||||
raise litellm.Timeout(
|
||||
message="This is a mock timeout error",
|
||||
llm_provider="openai",
|
||||
model=model,
|
||||
)
|
||||
|
||||
|
||||
def _sleep_for_timeout(timeout: float | str | httpx.Timeout):
|
||||
if isinstance(timeout, float):
|
||||
time.sleep(timeout)
|
||||
elif isinstance(timeout, str):
|
||||
time.sleep(float(timeout))
|
||||
elif isinstance(timeout, httpx.Timeout) and timeout.connect is not None:
|
||||
time.sleep(timeout.connect)
|
||||
|
||||
|
||||
async def _sleep_for_timeout_async(timeout: float | str | httpx.Timeout):
|
||||
if isinstance(timeout, float):
|
||||
await asyncio.sleep(timeout)
|
||||
elif isinstance(timeout, str):
|
||||
await asyncio.sleep(float(timeout))
|
||||
elif isinstance(timeout, httpx.Timeout) and timeout.connect is not None:
|
||||
await asyncio.sleep(timeout.connect)
|
||||
|
||||
|
||||
def mock_completion(
|
||||
model: str,
|
||||
messages: list,
|
||||
stream: bool | None = False,
|
||||
n: int | None = None,
|
||||
mock_response: MOCK_RESPONSE_TYPE | None = "This is a mock request",
|
||||
mock_tool_calls: list | None = None,
|
||||
mock_timeout: bool | None = False,
|
||||
logging=None,
|
||||
custom_llm_provider=None,
|
||||
timeout: float | str | httpx.Timeout | None = None,
|
||||
prompt_tokens: int | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
Generate a mock completion response for testing or debugging purposes.
|
||||
|
||||
This is a helper function that simulates the response structure of the OpenAI completion API.
|
||||
|
||||
Parameters:
|
||||
model (str): The name of the language model for which the mock response is generated.
|
||||
messages (List): A list of message objects representing the conversation context.
|
||||
stream (bool, optional): If True, returns a mock streaming response (default is False).
|
||||
mock_response (str, optional): The content of the mock response (default is "This is a mock request").
|
||||
mock_timeout (bool, optional): If True, the mock response will be a timeout error (default is False).
|
||||
timeout (float, optional): The timeout value to use for the mock response (default is None).
|
||||
**kwargs: Additional keyword arguments that can be used but are not required.
|
||||
|
||||
Returns:
|
||||
litellm.ModelResponse: A ModelResponse simulating a completion response with the specified model, messages, and mock response.
|
||||
|
||||
Raises:
|
||||
Exception: If an error occurs during the generation of the mock completion response.
|
||||
Note:
|
||||
- This function is intended for testing or debugging purposes to generate mock completion responses.
|
||||
- If 'stream' is True, it returns a response that mimics the behavior of a streaming completion.
|
||||
"""
|
||||
try:
|
||||
is_acompletion: Final = kwargs.get("acompletion") or False
|
||||
if mock_response is None:
|
||||
mock_response = "This is a mock request"
|
||||
|
||||
handle_mock_timeout(mock_timeout=mock_timeout, timeout=timeout, model=model)
|
||||
|
||||
## LOGGING
|
||||
if logging is not None:
|
||||
logging.pre_call(
|
||||
input=messages,
|
||||
api_key="mock-key",
|
||||
)
|
||||
|
||||
if isinstance(mock_response, str) or isinstance(mock_response, Exception):
|
||||
handle_mock_potential_exceptions(
|
||||
mock_response=mock_response,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
mock_response = cast(
|
||||
str | dict | ModelResponse | ModelResponseStream, mock_response
|
||||
) # after this point, mock_response is a string, dict, ModelResponse, or ModelResponseStream
|
||||
if isinstance(mock_response, str) and mock_response.startswith("Exception: mock_streaming_error"):
|
||||
mock_response = litellm.MockException(
|
||||
message="This is a mock error raised mid-stream",
|
||||
llm_provider="anthropic",
|
||||
model=model,
|
||||
status_code=529,
|
||||
)
|
||||
time_delay: Final = kwargs.get("mock_delay", None)
|
||||
if time_delay is not None and not is_acompletion:
|
||||
time.sleep(time_delay)
|
||||
|
||||
if isinstance(mock_response, dict):
|
||||
return ModelResponse(**mock_response)
|
||||
|
||||
if isinstance(mock_response, ModelResponse):
|
||||
if not stream:
|
||||
return mock_response
|
||||
# convert to ModelResponseStream
|
||||
mock_response = convert_model_response_to_streaming(mock_response)
|
||||
|
||||
model_response: ModelResponse | ModelResponseStream = ModelResponse()
|
||||
|
||||
if stream is True:
|
||||
model_response = ModelResponseStream()
|
||||
count_prompt_tokens: Final = admitted_prompt_token_counter(prompt_tokens)
|
||||
# don't try to access stream object,
|
||||
if kwargs.get("acompletion", False) is True:
|
||||
return CustomStreamWrapper(
|
||||
completion_stream=async_mock_completion_streaming_obj(
|
||||
model_response, mock_response=mock_response, model=model, n=n, prompt_tokens=prompt_tokens
|
||||
),
|
||||
model=model,
|
||||
custom_llm_provider="openai",
|
||||
logging_obj=logging,
|
||||
count_prompt_tokens=count_prompt_tokens,
|
||||
)
|
||||
return CustomStreamWrapper(
|
||||
completion_stream=mock_completion_streaming_obj(
|
||||
model_response, mock_response=mock_response, model=model, n=n, prompt_tokens=prompt_tokens
|
||||
),
|
||||
model=model,
|
||||
custom_llm_provider="openai",
|
||||
logging_obj=logging,
|
||||
count_prompt_tokens=count_prompt_tokens,
|
||||
)
|
||||
if isinstance(mock_response, litellm.MockException):
|
||||
raise mock_response
|
||||
# At this point, mock_response must be a string (all other types have been handled or returned early)
|
||||
mock_response = cast(str, mock_response)
|
||||
|
||||
if n is None:
|
||||
model_response.choices[0].message.content = mock_response
|
||||
else:
|
||||
_all_choices: Final = []
|
||||
for i in range(n):
|
||||
_choice = litellm.utils.Choices(
|
||||
index=i,
|
||||
message=litellm.utils.Message(content=mock_response, role="assistant"),
|
||||
)
|
||||
_all_choices.append(_choice)
|
||||
model_response.choices = _all_choices
|
||||
model_response.created = int(time.time())
|
||||
model_response.model = model
|
||||
|
||||
if mock_tool_calls:
|
||||
model_response.choices[0].message.tool_calls = [
|
||||
ChatCompletionMessageToolCall(**tool_call) for tool_call in mock_tool_calls
|
||||
]
|
||||
|
||||
usage_prompt_tokens: Final = (
|
||||
prompt_tokens if prompt_tokens is not None else DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT
|
||||
)
|
||||
setattr(
|
||||
model_response,
|
||||
"usage",
|
||||
Usage(
|
||||
prompt_tokens=usage_prompt_tokens,
|
||||
completion_tokens=DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT,
|
||||
total_tokens=usage_prompt_tokens + DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT,
|
||||
),
|
||||
)
|
||||
|
||||
if custom_llm_provider is not None:
|
||||
model_response.hidden_params["custom_llm_provider"] = custom_llm_provider
|
||||
else:
|
||||
try:
|
||||
_, inferred_provider, _, _ = litellm.utils.get_llm_provider(model=model)
|
||||
model_response.hidden_params["custom_llm_provider"] = inferred_provider
|
||||
except Exception:
|
||||
# dont let setting a hidden param block a mock_respose
|
||||
pass
|
||||
|
||||
if logging is not None:
|
||||
logging.post_call(
|
||||
input=messages,
|
||||
api_key="my-secret-key",
|
||||
original_response="my-original-response",
|
||||
)
|
||||
|
||||
return model_response
|
||||
|
||||
except Exception as e:
|
||||
if isinstance(e, openai.APIError):
|
||||
raise e
|
||||
raise Exception(f"Mock completion response failed - {e}")
|
||||
283
litellm/native/chat_plan.py
Normal file
283
litellm/native/chat_plan.py
Normal file
|
|
@ -0,0 +1,283 @@
|
|||
import asyncio
|
||||
import contextvars
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from types import MappingProxyType
|
||||
from typing import (
|
||||
Final,
|
||||
cast, # noqa: TID251 # casts moved verbatim from litellm/main.py
|
||||
)
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.chat_completion_agentic_loop import maybe_run_chat_completion_agentic_loop
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import get_completion_messages
|
||||
from litellm.litellm_core_utils.rules import Rules
|
||||
from litellm.native.chat import OPTIONAL_PARAMS_ADAPTER, execute_chat, resolve_dispatched_chat_response
|
||||
from litellm.native.request_support import resolve_control_options
|
||||
from litellm.types.litellm_params import ControlOptions
|
||||
from litellm.types.llms.anthropic import AnthropicThinkingParam
|
||||
from litellm.utils import (
|
||||
CustomStreamWrapper,
|
||||
ModelResponse,
|
||||
exception_type,
|
||||
get_llm_provider,
|
||||
get_non_default_completion_params,
|
||||
post_call_processing,
|
||||
validate_and_fix_openai_messages,
|
||||
validate_and_fix_openai_tools,
|
||||
validate_and_fix_thinking_param,
|
||||
validate_chat_completion_tool_choice,
|
||||
validate_openai_optional_params,
|
||||
)
|
||||
|
||||
_PLANNED_CHAT_ARGS: Final = frozenset(
|
||||
("kwargs", "non_default_params", "control_options", "prompt_id", "prompt_variables")
|
||||
)
|
||||
|
||||
|
||||
_EXECUTE_CHAT_CODE: Final = execute_chat.__code__
|
||||
|
||||
|
||||
_EXECUTE_CHAT_PARAMS: Final = (
|
||||
frozenset(_EXECUTE_CHAT_CODE.co_varnames[: _EXECUTE_CHAT_CODE.co_kwonlyargcount]) - _PLANNED_CHAT_ARGS
|
||||
)
|
||||
|
||||
|
||||
_SKIP_RESPONSES_API_BRIDGE_KWARG: Final = "_skip_responses_api_bridge"
|
||||
|
||||
|
||||
_CHAT_SURFACE_KWARGS: Final = (
|
||||
"fallbacks",
|
||||
"model_list",
|
||||
"prompt_id",
|
||||
"litellm_system_prompt",
|
||||
"ensure_alternating_roles",
|
||||
"user_continue_message",
|
||||
"assistant_continue_message",
|
||||
"mock_delay",
|
||||
"mock_timeout",
|
||||
)
|
||||
|
||||
|
||||
_TEXT_COMPLETION_PROVIDERS: Final = frozenset(
|
||||
("text-completion-openai", "text-completion-codestral", "text-completion-inception")
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PlannedChat:
|
||||
model: str
|
||||
custom_llm_provider: str
|
||||
named: Mapping[str, object]
|
||||
kwargs: Mapping[str, object]
|
||||
non_default_params: Mapping[str, object]
|
||||
control_options: ControlOptions
|
||||
|
||||
|
||||
def _optional_str(value: object) -> str | None:
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
_OBJECT_LIST: Final = TypeAdapter(list[object])
|
||||
|
||||
|
||||
def _object_list(value: object) -> list[object] | None: # mutable-ok: feeds validate_and_fix_openai_* which take a list
|
||||
return _OBJECT_LIST.validate_python(value) if isinstance(value, list) else None
|
||||
|
||||
|
||||
def _tool_choice(value: object) -> str | dict[str, object] | None: # mutable-ok: its validator takes a dict
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
return OPTIONAL_PARAMS_ADAPTER.validate_python(value) if isinstance(value, dict) else None
|
||||
|
||||
|
||||
def _stop(value: object) -> str | list[str] | None: # mutable-ok: validate_openai_optional_params takes a list
|
||||
if isinstance(value, list):
|
||||
return [item for item in _OBJECT_LIST.validate_python(value) if isinstance(item, str)]
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
def _thinking(value: object) -> AnthropicThinkingParam | bool | None:
|
||||
if isinstance(value, dict):
|
||||
return cast(AnthropicThinkingParam, value) # cast-ok: the caller's thinking dict, normalized by the validator
|
||||
return value if isinstance(value, bool) else None
|
||||
|
||||
|
||||
def _chat_entry_points_wrapped() -> bool:
|
||||
"""Tracers such as ddtrace wrap `litellm.completion`/`acompletion`, so bridged calls must go through them."""
|
||||
from litellm.chat_completions import dispatch
|
||||
|
||||
return litellm.completion is not dispatch.completion or litellm.acompletion is not dispatch.acompletion
|
||||
|
||||
|
||||
def _reads_chat_cache(request: Mapping[str, object]) -> bool:
|
||||
"""The chat response cache is read and written by the `acompletion` surface, keyed on the chat request."""
|
||||
return litellm.cache is not None or request.get("caching") is True
|
||||
|
||||
|
||||
def _uses_chat_surface(request: Mapping[str, object]) -> bool:
|
||||
from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler
|
||||
from litellm.types.llms.openai import ToolParam
|
||||
|
||||
tools: Final = request.get("tools")
|
||||
return (
|
||||
litellm.model_fallbacks is not None
|
||||
or _reads_chat_cache(request)
|
||||
or _chat_entry_points_wrapped()
|
||||
or any(request.get(key) is not None for key in _CHAT_SURFACE_KWARGS)
|
||||
or (
|
||||
isinstance(tools, list)
|
||||
and LiteLLM_Proxy_MCP_Handler.should_use_litellm_mcp_gateway(
|
||||
tools=cast(Iterable[ToolParam], tools) # cast-ok: OpenAI-format tools from the caller's request
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def plan_chat(request: Mapping[str, object], logging_obj: LiteLLMLoggingObj) -> PlannedChat | None:
|
||||
model: Final = request.get("model")
|
||||
if not isinstance(model, str) or _uses_chat_surface(request):
|
||||
return None
|
||||
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
|
||||
|
||||
custom_llm_provider: Final = _optional_str(request.get("custom_llm_provider"))
|
||||
api_base: Final = _optional_str(request.get("api_base"))
|
||||
_, resolved_provider, _, _ = get_llm_provider(
|
||||
model=model, custom_llm_provider=custom_llm_provider, api_base=api_base
|
||||
)
|
||||
if resolved_provider in _TEXT_COMPLETION_PROVIDERS:
|
||||
return None
|
||||
kwargs: Final = {
|
||||
key: value
|
||||
for key, value in request.items()
|
||||
if key not in _EXECUTE_CHAT_PARAMS and key != _SKIP_RESPONSES_API_BRIDGE_KWARG
|
||||
}
|
||||
tools: Final = validate_and_fix_openai_tools(tools=_object_list(request.get("tools")))
|
||||
messages: Final = get_completion_messages(
|
||||
messages=validate_and_fix_openai_messages(messages=_object_list(request.get("messages")) or []),
|
||||
ensure_alternating_roles=False,
|
||||
user_continue_message=None,
|
||||
assistant_continue_message=None,
|
||||
)
|
||||
non_default_params: Final = get_non_default_completion_params(kwargs=kwargs)
|
||||
AnthropicCacheControlHook.maybe_seed_default_injection_points(
|
||||
non_default_params=non_default_params,
|
||||
messages=messages,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
tools=tools,
|
||||
enable_prompt_caching=request.get("enable_prompt_caching") is True,
|
||||
api_base=api_base,
|
||||
)
|
||||
if logging_obj.should_run_prompt_management_hooks(non_default_params=non_default_params, tools=tools):
|
||||
return None
|
||||
named: Final = {key: value for key, value in request.items() if key in _EXECUTE_CHAT_PARAMS}
|
||||
tool_choice: Final = validate_chat_completion_tool_choice(
|
||||
tool_choice=_tool_choice(request.get("tool_choice")), model=model
|
||||
)
|
||||
return PlannedChat(
|
||||
model=model,
|
||||
custom_llm_provider=resolved_provider,
|
||||
named=MappingProxyType(
|
||||
{
|
||||
**named,
|
||||
"messages": messages,
|
||||
"tools": tools,
|
||||
"tool_choice": tool_choice,
|
||||
"stop": validate_openai_optional_params(stop=_stop(request.get("stop"))),
|
||||
"thinking": validate_and_fix_thinking_param(thinking=_thinking(request.get("thinking"))),
|
||||
"skip_responses_api_bridge": request.get(_SKIP_RESPONSES_API_BRIDGE_KWARG) is True,
|
||||
}
|
||||
),
|
||||
kwargs=MappingProxyType(kwargs),
|
||||
non_default_params=MappingProxyType(non_default_params),
|
||||
control_options=resolve_control_options(kwargs, model),
|
||||
)
|
||||
|
||||
|
||||
def _run_planned_chat(plan: PlannedChat, is_async: bool) -> ModelResponse | CustomStreamWrapper:
|
||||
return execute_chat(
|
||||
**plan.named, # pyright: ignore[reportArgumentType] # named holds the caller's validated OpenAI-format params
|
||||
kwargs={**plan.kwargs, "acompletion": is_async},
|
||||
non_default_params=dict(plan.non_default_params),
|
||||
control_options=plan.control_options,
|
||||
prompt_id=None,
|
||||
prompt_variables=None,
|
||||
)
|
||||
|
||||
|
||||
def _post_call_checked(
|
||||
plan: PlannedChat, response: ModelResponse | CustomStreamWrapper, is_async: bool
|
||||
) -> ModelResponse | CustomStreamWrapper:
|
||||
"""Runs the post-call rules and JSON-schema validation the ``completion`` client wrapper runs on its result."""
|
||||
post_call_processing(
|
||||
original_response=response,
|
||||
model=plan.model,
|
||||
optional_params={**plan.kwargs, **plan.named},
|
||||
original_function=litellm.acompletion if is_async else litellm.completion,
|
||||
rules_obj=Rules(),
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def execute_planned_chat(plan: PlannedChat) -> ModelResponse | CustomStreamWrapper:
|
||||
return _post_call_checked(plan, _run_planned_chat(plan, is_async=False), is_async=False)
|
||||
|
||||
|
||||
async def _settled_chat_response(
|
||||
plan: PlannedChat, loop: asyncio.AbstractEventLoop
|
||||
) -> ModelResponse | CustomStreamWrapper:
|
||||
pending: Final[object] = await loop.run_in_executor(
|
||||
None, partial(contextvars.copy_context().run, partial(_run_planned_chat, plan, True))
|
||||
)
|
||||
if asyncio.iscoroutine(pending):
|
||||
return await resolve_dispatched_chat_response(pending)
|
||||
if isinstance(pending, (ModelResponse, CustomStreamWrapper)):
|
||||
return pending
|
||||
raise TypeError(f"chat provider returned {type(pending).__name__}, expected a ModelResponse or a stream")
|
||||
|
||||
|
||||
async def _with_agentic_loop(plan: PlannedChat, response: ModelResponse) -> ModelResponse | CustomStreamWrapper:
|
||||
looped: Final = await maybe_run_chat_completion_agentic_loop(
|
||||
response=response,
|
||||
model=plan.model,
|
||||
messages=_object_list(plan.named["messages"]) or [],
|
||||
optional_params={
|
||||
key: value
|
||||
for key, value in plan.named.items()
|
||||
if value is not None and key not in ("model", "messages", "stream", "deployment_id")
|
||||
},
|
||||
kwargs=dict(plan.kwargs),
|
||||
logging_obj=plan.kwargs.get("litellm_logging_obj"),
|
||||
custom_llm_provider=plan.custom_llm_provider,
|
||||
stream=plan.named.get("stream") is True,
|
||||
)
|
||||
return response if looped is None else looped
|
||||
|
||||
|
||||
async def _aexecute_planned_chat(plan: PlannedChat) -> ModelResponse | CustomStreamWrapper:
|
||||
loop: Final = asyncio.get_event_loop()
|
||||
response: Final = await _settled_chat_response(plan, loop)
|
||||
if isinstance(response, CustomStreamWrapper):
|
||||
response.set_logging_event_loop(loop=loop)
|
||||
return response
|
||||
return await _with_agentic_loop(plan, response)
|
||||
|
||||
|
||||
async def aexecute_planned_chat(plan: PlannedChat) -> ModelResponse | CustomStreamWrapper:
|
||||
"""Maps a provider failure raised while awaiting the call, as ``acompletion`` does."""
|
||||
try:
|
||||
response: Final = await _aexecute_planned_chat(plan)
|
||||
except Exception as e: # noqa: BLE001 # every provider failure maps to the litellm exception acompletion raises
|
||||
raise exception_type(
|
||||
model=plan.model,
|
||||
custom_llm_provider=plan.custom_llm_provider or "openai",
|
||||
original_exception=e,
|
||||
completion_kwargs=dict(plan.named),
|
||||
extra_kwargs=dict(plan.kwargs),
|
||||
) from e
|
||||
return _post_call_checked(plan, response, is_async=True)
|
||||
0
litellm/native/chat_providers/__init__.py
Normal file
0
litellm/native/chat_providers/__init__.py
Normal file
273
litellm/native/chat_providers/agents.py
Normal file
273
litellm/native/chat_providers/agents.py
Normal file
|
|
@ -0,0 +1,273 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final, cast # noqa: TID251 # casts moved verbatim from litellm/main.py
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.native.chat_providers.handlers import base_llm_http_handler
|
||||
from litellm.native.chat_providers.support import dispatch_client_http, get_encoding
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
||||
|
||||
def complete_ragflow(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base: Final = ctx.api_base
|
||||
api_key: Final = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
provider_config: Final = ctx.provider_config
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
try:
|
||||
response: Final = base_llm_http_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
model_response=model_response,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
encoding=get_encoding(),
|
||||
stream=stream,
|
||||
provider_config=provider_config,
|
||||
)
|
||||
except Exception as e:
|
||||
logging.post_call(
|
||||
input=messages,
|
||||
api_key=api_key,
|
||||
original_response=str(e),
|
||||
additional_args={"headers": headers},
|
||||
)
|
||||
raise e
|
||||
|
||||
return response
|
||||
|
||||
|
||||
def complete_a2a(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
provider_config: Final = ctx.provider_config
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
(
|
||||
api_base,
|
||||
api_key,
|
||||
headers,
|
||||
) = litellm.A2AConfig.resolve_agent_config_from_registry(
|
||||
agent_name=model,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
headers=headers,
|
||||
optional_params=optional_params,
|
||||
)
|
||||
|
||||
# Fall back to environment variables and defaults
|
||||
api_base = api_base or litellm.api_base or get_secret_str("A2A_API_BASE")
|
||||
|
||||
if api_base is None:
|
||||
raise Exception(
|
||||
"api_base is required for A2A provider. "
|
||||
"Either provide api_base parameter, set A2A_API_BASE environment variable, "
|
||||
"or register the agent in the proxy with model='a2a/<agent-name>'."
|
||||
)
|
||||
|
||||
headers = headers or litellm.headers
|
||||
|
||||
return base_llm_http_handler.completion(
|
||||
model=model,
|
||||
stream=stream,
|
||||
messages=messages,
|
||||
acompletion=acompletion,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
timeout=timeout,
|
||||
headers=headers,
|
||||
encoding=get_encoding(),
|
||||
api_key=api_key,
|
||||
logging_obj=logging,
|
||||
client=client,
|
||||
provider_config=provider_config,
|
||||
)
|
||||
|
||||
|
||||
def complete_langgraph(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
from litellm.llms.langgraph.chat.transformation import LangGraphConfig
|
||||
|
||||
(
|
||||
api_base,
|
||||
api_key,
|
||||
) = LangGraphConfig().get_openai_compatible_provider_info(
|
||||
api_base=api_base or litellm.api_base,
|
||||
api_key=api_key or litellm.api_key,
|
||||
)
|
||||
|
||||
headers = headers or litellm.headers
|
||||
|
||||
return base_llm_http_handler.completion(
|
||||
model=model,
|
||||
stream=stream,
|
||||
messages=messages,
|
||||
acompletion=acompletion,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
timeout=timeout,
|
||||
headers=headers,
|
||||
encoding=get_encoding(),
|
||||
api_key=api_key,
|
||||
logging_obj=logging,
|
||||
client=client,
|
||||
)
|
||||
|
||||
|
||||
def complete_microsoft_365_copilot(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
from litellm.llms.microsoft_365_copilot.chat.handler import acompletion, completion
|
||||
from litellm.llms.microsoft_365_copilot.common_utils import as_secret_fields
|
||||
|
||||
messages: Final[Sequence[AllMessageValues]] = cast( # cast-ok: dispatch messages are prevalidated
|
||||
Sequence[AllMessageValues],
|
||||
ctx.messages,
|
||||
)
|
||||
optional_params_value: Final[object] = cast( # cast-ok: dispatch params are validated below
|
||||
object,
|
||||
ctx.optional_params,
|
||||
)
|
||||
optional_params: Final = TypeAdapter(Mapping[str, object]).validate_python(optional_params_value)
|
||||
kwargs_value: Final[object] = cast( # cast-ok: dispatch kwargs are validated below
|
||||
object,
|
||||
ctx.kwargs,
|
||||
)
|
||||
kwargs: Final = TypeAdapter(Mapping[str, object]).validate_python(kwargs_value)
|
||||
secret_fields: Final = as_secret_fields(kwargs.get("secret_fields"))
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
if ctx.acompletion:
|
||||
async_client: Final = client if isinstance(client, AsyncHTTPHandler) else None
|
||||
return acompletion(
|
||||
model=ctx.model,
|
||||
messages=messages,
|
||||
api_key=ctx.api_key,
|
||||
litellm_params=ctx.litellm_params,
|
||||
optional_params=optional_params,
|
||||
secret_fields=secret_fields,
|
||||
stream=ctx.stream is True,
|
||||
timeout=ctx.timeout,
|
||||
logging_obj=ctx.logging,
|
||||
client=async_client,
|
||||
shared_session=ctx.shared_session,
|
||||
)
|
||||
sync_client: Final = client if isinstance(client, HTTPHandler) else None
|
||||
return completion(
|
||||
model=ctx.model,
|
||||
messages=messages,
|
||||
api_key=ctx.api_key,
|
||||
litellm_params=ctx.litellm_params,
|
||||
optional_params=optional_params,
|
||||
secret_fields=secret_fields,
|
||||
stream=ctx.stream is True,
|
||||
timeout=ctx.timeout,
|
||||
logging_obj=ctx.logging,
|
||||
client=sync_client,
|
||||
)
|
||||
|
||||
|
||||
def complete_langflow(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
from litellm.llms.langflow.chat.transformation import LangFlowConfig
|
||||
|
||||
(
|
||||
api_base,
|
||||
api_key,
|
||||
) = LangFlowConfig().get_openai_compatible_provider_info(
|
||||
api_base=api_base or litellm.api_base,
|
||||
api_key=api_key or litellm.api_key,
|
||||
)
|
||||
|
||||
headers = headers or litellm.headers
|
||||
|
||||
return base_llm_http_handler.completion(
|
||||
model=model,
|
||||
stream=stream,
|
||||
messages=messages,
|
||||
acompletion=acompletion,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
timeout=timeout,
|
||||
headers=headers,
|
||||
encoding=get_encoding(),
|
||||
api_key=api_key,
|
||||
logging_obj=logging,
|
||||
client=client,
|
||||
)
|
||||
132
litellm/native/chat_providers/anthropic.py
Normal file
132
litellm/native/chat_providers/anthropic.py
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
import os
|
||||
from typing import Final, cast # noqa: TID251 # casts moved verbatim from litellm/main.py
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route
|
||||
from litellm.native.chat_providers.handlers import anthropic_chat_completions, base_llm_http_handler
|
||||
from litellm.native.chat_providers.support import dispatch_client_http, get_encoding, print_verbose
|
||||
from litellm.secret_managers.main import get_secret_bool
|
||||
from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult
|
||||
from litellm.utils import get_secret
|
||||
|
||||
|
||||
def complete_anthropic_text(
|
||||
ctx: CompletionDispatchContext,
|
||||
) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
custom_prompt_dict = ctx.custom_prompt_dict
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
api_key = api_key or litellm.anthropic_key or litellm.api_key or os.environ.get("ANTHROPIC_API_KEY")
|
||||
custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict
|
||||
api_base = cast(
|
||||
str | None,
|
||||
api_base
|
||||
or litellm.api_base
|
||||
or get_secret("ANTHROPIC_API_BASE")
|
||||
or get_secret("ANTHROPIC_BASE_URL")
|
||||
or "https://api.anthropic.com/v1/complete",
|
||||
)
|
||||
|
||||
# Check if we should disable automatic URL suffix appending
|
||||
disable_url_suffix: Final = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX")
|
||||
if api_base is not None and not disable_url_suffix:
|
||||
api_base = ensure_api_route(api_base, "v1/complete")
|
||||
elif disable_url_suffix:
|
||||
verbose_logger.debug("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/complete suffix")
|
||||
|
||||
return base_llm_http_handler.completion(
|
||||
model=model,
|
||||
stream=stream,
|
||||
messages=messages,
|
||||
acompletion=acompletion,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
custom_llm_provider="anthropic_text",
|
||||
timeout=timeout,
|
||||
headers=headers,
|
||||
encoding=get_encoding(),
|
||||
api_key=api_key,
|
||||
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
|
||||
)
|
||||
|
||||
|
||||
def complete_anthropic(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
custom_prompt_dict = ctx.custom_prompt_dict
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logger_fn: Final = ctx.logger_fn
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
api_key = api_key or litellm.anthropic_key or litellm.api_key or os.environ.get("ANTHROPIC_API_KEY")
|
||||
custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict
|
||||
# call /messages
|
||||
# default route for all anthropic models
|
||||
api_base = cast(
|
||||
str | None,
|
||||
api_base
|
||||
or litellm.api_base
|
||||
or get_secret("ANTHROPIC_API_BASE")
|
||||
or get_secret("ANTHROPIC_BASE_URL")
|
||||
or "https://api.anthropic.com/v1/messages",
|
||||
)
|
||||
|
||||
# Check if we should disable automatic URL suffix appending
|
||||
disable_url_suffix: Final = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX")
|
||||
if api_base is not None and not disable_url_suffix:
|
||||
api_base = ensure_api_route(api_base, "v1/messages")
|
||||
elif disable_url_suffix:
|
||||
verbose_logger.debug("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/messages suffix")
|
||||
|
||||
response: Final = anthropic_chat_completions.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
custom_prompt_dict=litellm.custom_prompt_dict,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
encoding=get_encoding(), # for calculating input/output tokens
|
||||
api_key=api_key,
|
||||
logging_obj=logging,
|
||||
headers=headers,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
if optional_params.get("stream", False) or acompletion is True:
|
||||
## LOGGING
|
||||
logging.post_call(
|
||||
input=messages,
|
||||
api_key=api_key,
|
||||
original_response=response,
|
||||
)
|
||||
return response
|
||||
400
litellm/native/chat_providers/aws.py
Normal file
400
litellm/native/chat_providers/aws.py
Normal file
|
|
@ -0,0 +1,400 @@
|
|||
from collections.abc import Mapping
|
||||
from functools import partial
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
|
||||
from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import (
|
||||
AmazonAnthropicClaudeConfig,
|
||||
)
|
||||
from litellm.llms.bedrock.chat.native_output_fallback import NativeOutputAttemptResult, call_with_native_output_fallback
|
||||
from litellm.llms.bedrock.common_utils import (
|
||||
BedrockModelInfo,
|
||||
NativeOutputFallback,
|
||||
bedrock_route_for_request,
|
||||
bedrock_strict_tools_fallback,
|
||||
deployment_model_info,
|
||||
without_bedrock_native_output_opt_in,
|
||||
without_bedrock_route_prefix,
|
||||
)
|
||||
from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantle_chat_config
|
||||
from litellm.native.chat_providers.handlers import (
|
||||
base_llm_http_handler,
|
||||
bedrock_converse_chat_completion,
|
||||
openai_like_chat_completion,
|
||||
sagemaker_llm,
|
||||
)
|
||||
from litellm.native.chat_providers.support import (
|
||||
dispatch_client_http,
|
||||
get_encoding,
|
||||
print_verbose,
|
||||
provider_config_items,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import ProviderConfigManager, get_secret
|
||||
|
||||
_ADDITIONAL_DROP_PARAMS_ADAPTER: Final = TypeAdapter(list[str])
|
||||
_OPTIONAL_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
def complete_bedrock_mantle(
|
||||
ctx: CompletionDispatchContext,
|
||||
) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
api_base = api_base or litellm.api_base or get_secret("BEDROCK_MANTLE_API_BASE")
|
||||
api_key = api_key or litellm.api_key or get_secret("BEDROCK_MANTLE_API_KEY")
|
||||
headers = headers or litellm.headers
|
||||
config: Final = bedrock_mantle_chat_config(model).get_config()
|
||||
for k, v in provider_config_items(config):
|
||||
if k not in optional_params:
|
||||
optional_params[k] = v
|
||||
return base_llm_http_handler.completion(
|
||||
model=model,
|
||||
stream=stream,
|
||||
messages=messages,
|
||||
acompletion=acompletion,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
timeout=timeout,
|
||||
headers=headers,
|
||||
encoding=get_encoding(),
|
||||
api_key=api_key,
|
||||
logging_obj=logging,
|
||||
client=client,
|
||||
)
|
||||
|
||||
|
||||
def complete_amazon_nova(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
custom_prompt_dict: Final = ctx.custom_prompt_dict
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logger_fn: Final = ctx.logger_fn
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
api_key = api_key or litellm.amazon_nova_api_key or get_secret_str("AMAZON_NOVA_API_KEY") or litellm.api_key
|
||||
api_base = (
|
||||
api_base or litellm.api_base or get_secret_str("AMAZON_NOVA_API_BASE") or "https://api.nova.amazon.com/v1"
|
||||
)
|
||||
return openai_like_chat_completion.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
encoding=get_encoding(),
|
||||
api_key=api_key,
|
||||
logging_obj=logging,
|
||||
timeout=timeout,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
custom_prompt_dict=custom_prompt_dict,
|
||||
)
|
||||
|
||||
|
||||
def complete_sagemaker_chat(
|
||||
ctx: CompletionDispatchContext,
|
||||
) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base: Final = ctx.api_base
|
||||
api_key: Final = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
return base_llm_http_handler.completion(
|
||||
model=model,
|
||||
stream=stream,
|
||||
messages=messages,
|
||||
acompletion=acompletion,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
timeout=timeout,
|
||||
headers=headers,
|
||||
encoding=get_encoding(),
|
||||
api_key=api_key,
|
||||
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
|
||||
client=client,
|
||||
)
|
||||
|
||||
|
||||
def complete_sagemaker(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
custom_prompt_dict: Final = ctx.custom_prompt_dict
|
||||
hf_model_name: Final = ctx.hf_model_name
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logger_fn: Final = ctx.logger_fn
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
|
||||
return sagemaker_llm.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
custom_prompt_dict=custom_prompt_dict,
|
||||
hf_model_name=hf_model_name,
|
||||
logger_fn=logger_fn,
|
||||
encoding=get_encoding(),
|
||||
logging_obj=logging,
|
||||
acompletion=acompletion,
|
||||
)
|
||||
|
||||
|
||||
_AWS_BEDROCK_CLIENT_CREDENTIAL_KEYS: Final = (
|
||||
"aws_access_key_id",
|
||||
"aws_secret_access_key",
|
||||
"aws_session_token",
|
||||
"aws_region_name",
|
||||
)
|
||||
|
||||
|
||||
def _native_output_fallback(
|
||||
ctx: CompletionDispatchContext,
|
||||
first_attempt_params: Mapping[str, object],
|
||||
first_attempt_litellm_params: Mapping[str, object],
|
||||
) -> NativeOutputFallback:
|
||||
litellm_params: Final = without_bedrock_native_output_opt_in(first_attempt_litellm_params)
|
||||
remapped_params: Final = ctx.optional_params_for_model_info(deployment_model_info(litellm_params))
|
||||
return NativeOutputFallback(
|
||||
optional_params={
|
||||
**{key: value for key, value in remapped_params.items() if key != "aws_bedrock_client"},
|
||||
**{
|
||||
key: first_attempt_params[key]
|
||||
for key in _AWS_BEDROCK_CLIENT_CREDENTIAL_KEYS
|
||||
if key in first_attempt_params
|
||||
},
|
||||
},
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
|
||||
|
||||
def _converse_native_output_fallback(
|
||||
ctx: CompletionDispatchContext,
|
||||
first_attempt_params: Mapping[str, object],
|
||||
first_attempt_litellm_params: Mapping[str, object],
|
||||
) -> NativeOutputFallback | None:
|
||||
fallback: Final = _native_output_fallback(ctx, first_attempt_params, first_attempt_litellm_params)
|
||||
return None if AmazonConverseConfig.sends_native_text_format(fallback.optional_params) else fallback
|
||||
|
||||
|
||||
def _native_output_attempt_timeout(
|
||||
ctx: CompletionDispatchContext, fallback: NativeOutputFallback | None
|
||||
) -> float | str | httpx.Timeout | None:
|
||||
if fallback is None:
|
||||
return ctx.timeout
|
||||
retry_timeout: Final = ctx.timeout_for_optional_params(fallback.optional_params)
|
||||
ctx.logging.record_applied_timeout(retry_timeout)
|
||||
return retry_timeout
|
||||
|
||||
|
||||
def complete_bedrock(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base: Final = ctx.api_base
|
||||
api_key: Final = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_prompt_dict = ctx.custom_prompt_dict
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logger_fn: Final = ctx.logger_fn
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
provider_config = ctx.provider_config
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict
|
||||
|
||||
if "aws_bedrock_client" in optional_params:
|
||||
verbose_logger.warning(
|
||||
"'aws_bedrock_client' is a deprecated param. Please move to another auth method - https://docs.litellm.ai/docs/providers/bedrock#boto3---authentication."
|
||||
)
|
||||
# Extract credentials for legacy boto3 client and pass thru to httpx
|
||||
aws_bedrock_client: Final = optional_params.pop("aws_bedrock_client")
|
||||
creds: Final = aws_bedrock_client._get_credentials().get_frozen_credentials()
|
||||
|
||||
if creds.access_key:
|
||||
optional_params["aws_access_key_id"] = creds.access_key
|
||||
if creds.secret_key:
|
||||
optional_params["aws_secret_access_key"] = creds.secret_key
|
||||
if creds.token:
|
||||
optional_params["aws_session_token"] = creds.token
|
||||
if "aws_region_name" not in optional_params or optional_params["aws_region_name"] is None:
|
||||
optional_params["aws_region_name"] = aws_bedrock_client.meta.region_name
|
||||
|
||||
additional_drop_params: Final = (
|
||||
_ADDITIONAL_DROP_PARAMS_ADAPTER.validate_python(ctx.kwargs["additional_drop_params"])
|
||||
if ctx.kwargs.get("additional_drop_params") is not None
|
||||
else None
|
||||
)
|
||||
bedrock_route: Final = bedrock_route_for_request(model, ctx.request_params, additional_drop_params)
|
||||
if bedrock_route == "claude_platform":
|
||||
provider_config = ProviderConfigManager.get_provider_chat_config(
|
||||
model=model,
|
||||
provider=LlmProviders.BEDROCK,
|
||||
)
|
||||
model = BedrockModelInfo.get_claude_platform_model(model)
|
||||
return base_llm_http_handler.completion(
|
||||
model=model,
|
||||
stream=stream,
|
||||
messages=messages,
|
||||
acompletion=acompletion,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
custom_llm_provider="bedrock",
|
||||
timeout=timeout,
|
||||
headers=headers,
|
||||
encoding=get_encoding(),
|
||||
api_key=api_key,
|
||||
logging_obj=logging,
|
||||
client=client,
|
||||
provider_config=provider_config,
|
||||
)
|
||||
elif bedrock_route == "converse":
|
||||
model = without_bedrock_route_prefix(model)
|
||||
converse_model: Final = model
|
||||
|
||||
def converse_attempt(fallback: NativeOutputFallback | None) -> NativeOutputAttemptResult:
|
||||
return bedrock_converse_chat_completion.completion(
|
||||
model=converse_model,
|
||||
messages=messages,
|
||||
custom_prompt_dict=custom_prompt_dict,
|
||||
model_response=model_response,
|
||||
optional_params=optional_params if fallback is None else dict(fallback.optional_params),
|
||||
litellm_params=litellm_params if fallback is None else dict(fallback.litellm_params),
|
||||
logger_fn=logger_fn,
|
||||
encoding=get_encoding(),
|
||||
logging_obj=logging,
|
||||
extra_headers=headers, # Use merged headers instead of original extra_headers
|
||||
timeout=_native_output_attempt_timeout(ctx, fallback),
|
||||
acompletion=acompletion,
|
||||
client=client,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
)
|
||||
|
||||
converse_params: Final = _OPTIONAL_PARAMS_ADAPTER.validate_python(optional_params)
|
||||
response = call_with_native_output_fallback(
|
||||
converse_attempt,
|
||||
(
|
||||
partial(_converse_native_output_fallback, ctx, converse_params, dict(litellm_params))
|
||||
if AmazonConverseConfig().uses_opt_in_native_output(converse_model, converse_params, litellm_params)
|
||||
else None
|
||||
),
|
||||
converse_model,
|
||||
strict_tools_fallback=(
|
||||
partial(bedrock_strict_tools_fallback, converse_model, converse_params, dict(litellm_params))
|
||||
if drops_unsupported_params(litellm_params.get("drop_params"))
|
||||
else None
|
||||
),
|
||||
)
|
||||
elif bedrock_route == "converse_like":
|
||||
model = model.replace("converse_like/", "")
|
||||
response = base_llm_http_handler.completion(
|
||||
model=model,
|
||||
stream=stream,
|
||||
messages=messages,
|
||||
acompletion=acompletion,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
custom_llm_provider="bedrock",
|
||||
timeout=timeout,
|
||||
headers=headers,
|
||||
encoding=get_encoding(),
|
||||
api_key=api_key,
|
||||
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
|
||||
client=client,
|
||||
)
|
||||
else:
|
||||
|
||||
def invoke_attempt(fallback: NativeOutputFallback | None) -> NativeOutputAttemptResult:
|
||||
return base_llm_http_handler.completion(
|
||||
model=model,
|
||||
stream=stream,
|
||||
messages=messages,
|
||||
acompletion=acompletion,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
optional_params=optional_params if fallback is None else dict(fallback.optional_params),
|
||||
litellm_params=litellm_params if fallback is None else dict(fallback.litellm_params),
|
||||
custom_llm_provider="bedrock",
|
||||
timeout=_native_output_attempt_timeout(ctx, fallback),
|
||||
headers=headers,
|
||||
encoding=get_encoding(),
|
||||
api_key=api_key,
|
||||
logging_obj=logging,
|
||||
client=client,
|
||||
)
|
||||
|
||||
invoke_params: Final = _OPTIONAL_PARAMS_ADAPTER.validate_python(optional_params)
|
||||
response = call_with_native_output_fallback(
|
||||
invoke_attempt,
|
||||
(
|
||||
partial(_native_output_fallback, ctx, invoke_params, dict(litellm_params))
|
||||
if isinstance(provider_config, AmazonAnthropicClaudeConfig)
|
||||
and provider_config.uses_opt_in_native_output(model, invoke_params, litellm_params)
|
||||
else None
|
||||
),
|
||||
model,
|
||||
)
|
||||
|
||||
return response
|
||||
399
litellm/native/chat_providers/azure.py
Normal file
399
litellm/native/chat_providers/azure.py
Normal file
|
|
@ -0,0 +1,399 @@
|
|||
from typing import Final, cast # noqa: TID251 # casts moved verbatim from litellm/main.py
|
||||
|
||||
import openai
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import stringify_json_tool_call_content
|
||||
from litellm.llms.azure.azure import check_dynamic_azure_params
|
||||
from litellm.llms.azure_ai.anthropic.transformation import azure_anthropic_messages_url
|
||||
from litellm.native.chat_providers.handlers import (
|
||||
azure_anthropic_chat_completions,
|
||||
azure_chat_completions,
|
||||
azure_o1_chat_completions,
|
||||
azure_text_completions,
|
||||
base_llm_http_handler,
|
||||
)
|
||||
from litellm.native.chat_providers.support import (
|
||||
dispatch_client_azure,
|
||||
dispatch_client_http,
|
||||
get_encoding,
|
||||
print_verbose,
|
||||
provider_config_items,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult
|
||||
from litellm.utils import get_secret
|
||||
|
||||
|
||||
def complete_azure(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
_azure_detection_model: Final = ctx._azure_detection_model
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
api_version = ctx.api_version
|
||||
client: Final = dispatch_client_azure(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
extra_headers: Final = ctx.extra_headers
|
||||
headers = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logger_fn: Final = ctx.logger_fn
|
||||
logging: Final = ctx.logging
|
||||
max_retries: Final = ctx.max_retries
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
dynamic_params = False
|
||||
if client is not None and (isinstance(client, openai.AzureOpenAI) or isinstance(client, openai.AsyncAzureOpenAI)):
|
||||
dynamic_params = check_dynamic_azure_params(
|
||||
azure_client_params={"api_version": api_version},
|
||||
azure_client=client,
|
||||
)
|
||||
|
||||
api_type: Final = get_secret("AZURE_API_TYPE") or "azure"
|
||||
|
||||
api_base = api_base or litellm.api_base or get_secret("AZURE_API_BASE")
|
||||
|
||||
api_version = (
|
||||
api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") or litellm.AZURE_DEFAULT_API_VERSION
|
||||
)
|
||||
|
||||
api_key = (
|
||||
api_key
|
||||
or litellm.api_key
|
||||
or litellm.azure_key
|
||||
or get_secret_str("AZURE_OPENAI_API_KEY")
|
||||
or get_secret_str("AZURE_API_KEY")
|
||||
)
|
||||
|
||||
azure_ad_token: Final = optional_params.get("extra_body", {}).pop("azure_ad_token", None) or get_secret_str(
|
||||
"AZURE_AD_TOKEN"
|
||||
)
|
||||
|
||||
azure_ad_token_provider_value: Final = litellm_params.get("azure_ad_token_provider", None)
|
||||
azure_ad_token_provider: Final = azure_ad_token_provider_value if callable(azure_ad_token_provider_value) else None
|
||||
|
||||
headers = headers or litellm.headers
|
||||
|
||||
if extra_headers is not None:
|
||||
optional_params["extra_headers"] = extra_headers
|
||||
if max_retries is not None:
|
||||
optional_params["max_retries"] = max_retries
|
||||
|
||||
if litellm.AzureOpenAIO1Config().is_o_series_model(model=_azure_detection_model):
|
||||
## LOAD CONFIG - if set
|
||||
config = litellm.AzureOpenAIO1Config.get_config()
|
||||
for k, v in provider_config_items(config):
|
||||
if (
|
||||
k not in optional_params
|
||||
): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in
|
||||
optional_params[k] = v
|
||||
|
||||
response = azure_o1_chat_completions.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
api_version=api_version,
|
||||
dynamic_params=dynamic_params,
|
||||
azure_ad_token=azure_ad_token,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
logging_obj=logging,
|
||||
acompletion=acompletion,
|
||||
timeout=timeout,
|
||||
client=client, # pass AsyncAzureOpenAI, AzureOpenAI client
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
else:
|
||||
## LOAD CONFIG - if set
|
||||
config = litellm.AzureOpenAIConfig.get_config()
|
||||
for k, v in provider_config_items(config):
|
||||
if (
|
||||
k not in optional_params
|
||||
): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in
|
||||
optional_params[k] = v
|
||||
|
||||
## COMPLETION CALL
|
||||
response = azure_chat_completions.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
api_version=api_version,
|
||||
api_type=api_type,
|
||||
dynamic_params=dynamic_params,
|
||||
azure_ad_token=azure_ad_token,
|
||||
azure_ad_token_provider=azure_ad_token_provider,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
logging_obj=logging,
|
||||
acompletion=acompletion,
|
||||
timeout=timeout,
|
||||
client=client, # pass AsyncAzureOpenAI, AzureOpenAI client
|
||||
)
|
||||
|
||||
if optional_params.get("stream", False):
|
||||
## LOGGING
|
||||
logging.post_call(
|
||||
input=messages,
|
||||
api_key=api_key,
|
||||
original_response=response,
|
||||
additional_args={
|
||||
"headers": headers,
|
||||
"api_version": api_version,
|
||||
"api_base": api_base,
|
||||
},
|
||||
)
|
||||
|
||||
return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
|
||||
|
||||
|
||||
def complete_azure_text(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
api_version = ctx.api_version
|
||||
client: Final = dispatch_client_azure(ctx)
|
||||
extra_headers: Final = ctx.extra_headers
|
||||
headers = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logger_fn: Final = ctx.logger_fn
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
api_type: Final = get_secret_str("AZURE_API_TYPE") or "azure"
|
||||
|
||||
api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
|
||||
|
||||
if api_base is None:
|
||||
raise ValueError(
|
||||
"api_base is required for Azure OpenAI LLM provider. Either set it dynamically or set the AZURE_API_BASE environment variable."
|
||||
)
|
||||
|
||||
api_version = api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION")
|
||||
|
||||
api_key = (
|
||||
api_key
|
||||
or litellm.api_key
|
||||
or litellm.azure_key
|
||||
or get_secret_str("AZURE_OPENAI_API_KEY")
|
||||
or get_secret_str("AZURE_API_KEY")
|
||||
)
|
||||
|
||||
azure_ad_token: Final = optional_params.get("extra_body", {}).pop("azure_ad_token", None) or get_secret_str(
|
||||
"AZURE_AD_TOKEN"
|
||||
)
|
||||
|
||||
azure_ad_token_provider_value: Final = litellm_params.get("azure_ad_token_provider", None)
|
||||
azure_ad_token_provider: Final = azure_ad_token_provider_value if callable(azure_ad_token_provider_value) else None
|
||||
|
||||
headers = headers or litellm.headers
|
||||
|
||||
if extra_headers is not None:
|
||||
optional_params["extra_headers"] = extra_headers
|
||||
|
||||
## LOAD CONFIG - if set
|
||||
config: Final = litellm.AzureOpenAIConfig.get_config()
|
||||
for k, v in provider_config_items(config):
|
||||
if (
|
||||
k not in optional_params
|
||||
): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in
|
||||
optional_params[k] = v
|
||||
|
||||
## COMPLETION CALL
|
||||
response: Final = azure_text_completions.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
api_version=cast(str, api_version),
|
||||
api_type=api_type,
|
||||
azure_ad_token=azure_ad_token,
|
||||
azure_ad_token_provider=azure_ad_token_provider,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
logging_obj=logging,
|
||||
acompletion=acompletion,
|
||||
timeout=timeout,
|
||||
client=client, # pass AsyncAzureOpenAI, AzureOpenAI client
|
||||
)
|
||||
|
||||
if optional_params.get("stream", False) or acompletion is True:
|
||||
## LOGGING
|
||||
logging.post_call(
|
||||
input=messages,
|
||||
api_key=api_key,
|
||||
original_response=response,
|
||||
additional_args={
|
||||
"headers": headers,
|
||||
"api_version": api_version,
|
||||
"api_base": api_base,
|
||||
},
|
||||
)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
def complete_azure_ai(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
extra_headers: Final = ctx.extra_headers
|
||||
headers = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logger_fn: Final = ctx.logger_fn
|
||||
logging: Final = ctx.logging
|
||||
messages = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
|
||||
|
||||
azure_ai_route: Final = AzureFoundryModelInfo.get_azure_ai_route(model)
|
||||
|
||||
# Check if this is an agents route - model format: azure_ai/agents/<agent_id>
|
||||
if azure_ai_route == "agents":
|
||||
from litellm.llms.azure_ai.agents import AzureAIAgentsConfig
|
||||
|
||||
api_base = AzureFoundryModelInfo.get_api_base(api_base)
|
||||
if api_base is None:
|
||||
raise ValueError(
|
||||
"Azure AI Agents requests require an api_base. Set `api_base` or the AZURE_AI_API_BASE env var."
|
||||
)
|
||||
api_key = AzureFoundryModelInfo.get_api_key(api_key)
|
||||
|
||||
response = AzureAIAgentsConfig.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
model_response=model_response,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
timeout=timeout,
|
||||
acompletion=acompletion,
|
||||
stream=stream,
|
||||
headers=headers or litellm.headers,
|
||||
)
|
||||
|
||||
# Check if this is a Claude model - route to Azure Anthropic handler
|
||||
elif "claude" in model.lower():
|
||||
# Use Azure Anthropic handler for Claude models
|
||||
api_base = AzureFoundryModelInfo.get_api_base(api_base)
|
||||
if api_base is None:
|
||||
raise ValueError(
|
||||
"Azure Anthropic requests require an api_base. Set `api_base` or the AZURE_AI_API_BASE env var."
|
||||
)
|
||||
api_key = AzureFoundryModelInfo.get_api_key(api_key)
|
||||
|
||||
if api_base:
|
||||
api_base = azure_anthropic_messages_url(api_base)
|
||||
|
||||
response = azure_anthropic_chat_completions.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
custom_prompt_dict=litellm.custom_prompt_dict,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
encoding=get_encoding(),
|
||||
api_key=api_key,
|
||||
logging_obj=logging,
|
||||
headers=headers,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
if optional_params.get("stream", False) or acompletion is True:
|
||||
## LOGGING
|
||||
logging.post_call(
|
||||
input=messages,
|
||||
api_key=api_key,
|
||||
original_response=response,
|
||||
)
|
||||
response = response
|
||||
else:
|
||||
# Non-Claude models use standard Azure AI flow
|
||||
api_base = AzureFoundryModelInfo.get_api_base(api_base)
|
||||
# set API KEY
|
||||
api_key = AzureFoundryModelInfo.get_api_key(api_key)
|
||||
|
||||
headers = headers or litellm.headers
|
||||
|
||||
if extra_headers is not None:
|
||||
optional_params["extra_headers"] = extra_headers
|
||||
|
||||
## FOR COHERE
|
||||
if "command-r" in model: # make sure tool call in messages are str
|
||||
messages = stringify_json_tool_call_content(messages=messages)
|
||||
|
||||
## COMPLETION CALL
|
||||
try:
|
||||
response = base_llm_http_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
model_response=model_response,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
timeout=timeout,
|
||||
client=client, # pass AsyncOpenAI, OpenAI client
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
encoding=get_encoding(),
|
||||
stream=stream,
|
||||
)
|
||||
except Exception as e:
|
||||
## LOGGING - log the original exception returned
|
||||
logging.post_call(
|
||||
input=messages,
|
||||
api_key=api_key,
|
||||
original_response=str(e),
|
||||
additional_args={"headers": headers},
|
||||
)
|
||||
raise e
|
||||
|
||||
if optional_params.get("stream", False):
|
||||
## LOGGING
|
||||
logging.post_call(
|
||||
input=messages,
|
||||
api_key=api_key,
|
||||
original_response=response,
|
||||
additional_args={"headers": headers},
|
||||
)
|
||||
|
||||
return response
|
||||
147
litellm/native/chat_providers/custom.py
Normal file
147
litellm/native/chat_providers/custom.py
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
import time
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
||||
import litellm
|
||||
from litellm.exceptions import LiteLLMUnknownProvider
|
||||
from litellm.llms.custom_llm import CustomLLM, custom_chat_llm_router
|
||||
from litellm.native.chat_providers.support import get_encoding, print_verbose
|
||||
from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult
|
||||
from litellm.utils import CustomStreamWrapper
|
||||
|
||||
|
||||
def _custom_api_first_output(resp: httpx.Response | None) -> str:
|
||||
return resp.json()["data"][0]["output"][0]
|
||||
|
||||
|
||||
def complete_custom(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
api_base: Final = ctx.api_base
|
||||
headers: Final = ctx.headers
|
||||
kwargs: Final = ctx.kwargs
|
||||
max_tokens: Final = ctx.max_tokens
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
temperature: Final = ctx.temperature
|
||||
top_p: Final = ctx.top_p
|
||||
|
||||
url: Final = litellm.api_base or api_base or ""
|
||||
if url is None or url == "":
|
||||
raise ValueError("api_base not set. Set api_base or litellm.api_base for custom endpoints")
|
||||
|
||||
"""
|
||||
assume input to custom LLM api bases follow this format:
|
||||
resp = litellm.module_level_client.post(
|
||||
api_base,
|
||||
json={
|
||||
'model': 'meta-llama/Llama-2-13b-hf', # model name
|
||||
'params': {
|
||||
'prompt': ["The capital of France is P"],
|
||||
'max_tokens': 32,
|
||||
'temperature': 0.7,
|
||||
'top_p': 1.0,
|
||||
'top_k': 40,
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
"""
|
||||
prompt: Final = " ".join([message["content"] for message in messages])
|
||||
resp: Final = litellm.module_level_client.post(
|
||||
url,
|
||||
headers=headers,
|
||||
json={
|
||||
"model": model,
|
||||
"params": {
|
||||
"prompt": [prompt],
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": temperature,
|
||||
"top_p": top_p,
|
||||
"top_k": kwargs.get("top_k"),
|
||||
},
|
||||
**kwargs.get("extra_body", {}),
|
||||
},
|
||||
)
|
||||
"""
|
||||
assume all responses from custom api_bases of this format:
|
||||
{
|
||||
'data': [
|
||||
{
|
||||
'prompt': 'The capital of France is P',
|
||||
'output': ['The capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France'],
|
||||
'params': {'temperature': 0.7, 'top_k': 40, 'top_p': 1}}],
|
||||
'message': 'ok'
|
||||
}
|
||||
]
|
||||
}
|
||||
"""
|
||||
string_response: Final = _custom_api_first_output(resp)
|
||||
## RESPONSE OBJECT
|
||||
model_response.choices[0].message.content = string_response
|
||||
model_response.created = int(time.time())
|
||||
model_response.model = model
|
||||
return model_response
|
||||
|
||||
|
||||
def complete_custom_providers(
|
||||
ctx: CompletionDispatchContext,
|
||||
) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base: Final = ctx.api_base
|
||||
api_key: Final = ctx.api_key
|
||||
client: Final = ctx.client
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
custom_prompt_dict: Final = ctx.custom_prompt_dict
|
||||
headers = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logger_fn: Final = ctx.logger_fn
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
custom_handler: CustomLLM | None = None
|
||||
for item in litellm.custom_provider_map:
|
||||
if item["provider"] == custom_llm_provider:
|
||||
custom_handler = item["custom_handler"]
|
||||
|
||||
if custom_handler is None:
|
||||
raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider)
|
||||
|
||||
## ROUTE LLM CALL ##
|
||||
handler_fn: Final = custom_chat_llm_router(async_fn=acompletion, stream=stream, custom_llm=custom_handler)
|
||||
|
||||
headers = headers or litellm.headers or {}
|
||||
|
||||
## CALL FUNCTION
|
||||
response: Final = handler_fn(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
timeout=timeout,
|
||||
custom_prompt_dict=custom_prompt_dict,
|
||||
client=client, # pass AsyncOpenAI, OpenAI client
|
||||
encoding=get_encoding(),
|
||||
)
|
||||
if stream is True:
|
||||
return CustomStreamWrapper(
|
||||
completion_stream=response,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
logging_obj=logging,
|
||||
)
|
||||
|
||||
return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
|
||||
299
litellm/native/chat_providers/gateways.py
Normal file
299
litellm/native/chat_providers/gateways.py
Normal file
|
|
@ -0,0 +1,299 @@
|
|||
from typing import Final
|
||||
|
||||
import openai
|
||||
|
||||
import litellm
|
||||
from litellm.constants import NADIR_DEFAULT_API_BASE
|
||||
from litellm.native.chat_providers.handlers import base_llm_http_handler
|
||||
from litellm.native.chat_providers.support import dispatch_client_http, get_encoding, provider_config_items
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult
|
||||
from litellm.utils import get_secret
|
||||
|
||||
|
||||
def complete_cometapi(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
provider_config: Final = ctx.provider_config
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
api_key = api_key or litellm.cometapi_key or get_secret_str("COMETAPI_KEY") or litellm.api_key
|
||||
|
||||
api_base = api_base or litellm.api_base or get_secret_str("COMETAPI_API_BASE") or "https://api.cometapi.com/v1"
|
||||
|
||||
## COMPLETION CALL
|
||||
response: Final = base_llm_http_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
model_response=model_response,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
encoding=get_encoding(),
|
||||
stream=stream,
|
||||
provider_config=provider_config,
|
||||
)
|
||||
|
||||
## LOGGING
|
||||
logging.post_call(input=messages, api_key=api_key, original_response=response)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
def complete_openrouter(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
headers = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
api_base = api_base or litellm.api_base or get_secret_str("OPENROUTER_API_BASE") or "https://openrouter.ai/api/v1"
|
||||
|
||||
api_key = (
|
||||
api_key
|
||||
or litellm.api_key
|
||||
or litellm.openrouter_key
|
||||
or get_secret_str("OPENROUTER_API_KEY")
|
||||
or get_secret_str("OR_API_KEY")
|
||||
)
|
||||
|
||||
openrouter_site_url: Final = get_secret("OR_SITE_URL") or "https://litellm.ai"
|
||||
openrouter_app_name: Final = get_secret("OR_APP_NAME") or "liteLLM"
|
||||
|
||||
openrouter_headers: Final = {
|
||||
"HTTP-Referer": openrouter_site_url,
|
||||
"X-Title": openrouter_app_name,
|
||||
}
|
||||
|
||||
_headers: Final = headers or litellm.headers
|
||||
if _headers:
|
||||
openrouter_headers.update(_headers)
|
||||
|
||||
headers = openrouter_headers
|
||||
|
||||
## Load Config
|
||||
config: Final = litellm.OpenrouterConfig.get_config()
|
||||
for k, v in provider_config_items(config):
|
||||
if k == "extra_body":
|
||||
# we use openai 'extra_body' to pass openrouter specific params - transforms, route, models
|
||||
if "extra_body" in optional_params:
|
||||
optional_params[k].update(v)
|
||||
else:
|
||||
optional_params[k] = v
|
||||
elif k not in optional_params:
|
||||
optional_params[k] = v
|
||||
|
||||
## COMPLETION CALL
|
||||
response: Final = base_llm_http_handler.completion(
|
||||
model=model,
|
||||
stream=stream,
|
||||
messages=messages,
|
||||
acompletion=acompletion,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
custom_llm_provider="openrouter",
|
||||
timeout=timeout,
|
||||
headers=headers,
|
||||
encoding=get_encoding(),
|
||||
api_key=api_key,
|
||||
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
|
||||
client=client,
|
||||
)
|
||||
## LOGGING
|
||||
logging.post_call(input=messages, api_key=openai.api_key, original_response=response)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
def complete_nadir(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
api_base: Final = ctx.api_base or litellm.api_base or get_secret_str("NADIR_API_BASE") or NADIR_DEFAULT_API_BASE
|
||||
api_key: Final = ctx.api_key
|
||||
|
||||
response: Final = base_llm_http_handler.completion(
|
||||
model=ctx.model,
|
||||
stream=ctx.stream,
|
||||
messages=ctx.messages,
|
||||
acompletion=ctx.acompletion,
|
||||
api_base=api_base,
|
||||
model_response=ctx.model_response,
|
||||
optional_params=ctx.optional_params,
|
||||
litellm_params=ctx.litellm_params,
|
||||
shared_session=ctx.shared_session,
|
||||
custom_llm_provider="nadir",
|
||||
timeout=ctx.timeout,
|
||||
headers=ctx.headers or litellm.headers,
|
||||
encoding=get_encoding(),
|
||||
api_key=api_key,
|
||||
logging_obj=ctx.logging,
|
||||
client=dispatch_client_http(ctx),
|
||||
)
|
||||
ctx.logging.post_call(input=ctx.messages, api_key=api_key, original_response=response)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
def complete_vercel_ai_gateway(
|
||||
ctx: CompletionDispatchContext,
|
||||
) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
headers = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
api_base = (
|
||||
api_base
|
||||
or litellm.api_base
|
||||
or get_secret_str("VERCEL_AI_GATEWAY_API_BASE")
|
||||
or "https://ai-gateway.vercel.sh/v1"
|
||||
)
|
||||
|
||||
api_key = api_key or litellm.api_key or get_secret("VERCEL_AI_GATEWAY_API_KEY")
|
||||
|
||||
vercel_site_url: Final = get_secret("VERCEL_SITE_URL") or "https://litellm.ai"
|
||||
vercel_app_name: Final = get_secret("VERCEL_APP_NAME") or "liteLLM"
|
||||
|
||||
vercel_headers: Final = {
|
||||
"http-referer": vercel_site_url,
|
||||
"x-title": vercel_app_name,
|
||||
}
|
||||
|
||||
_headers: Final = headers or litellm.headers
|
||||
if _headers:
|
||||
vercel_headers.update(_headers)
|
||||
|
||||
headers = vercel_headers
|
||||
|
||||
## Load Config
|
||||
config: Final = litellm.VercelAIGatewayConfig.get_config()
|
||||
for k, v in provider_config_items(config):
|
||||
if k == "extra_body":
|
||||
# we use openai 'extra_body' to pass vercel specific params - providerOptions
|
||||
if "extra_body" in optional_params:
|
||||
optional_params[k].update(v)
|
||||
else:
|
||||
optional_params[k] = v
|
||||
elif k not in optional_params:
|
||||
optional_params[k] = v
|
||||
|
||||
## COMPLETION CALL
|
||||
response: Final = base_llm_http_handler.completion(
|
||||
model=model,
|
||||
stream=stream,
|
||||
messages=messages,
|
||||
acompletion=acompletion,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
custom_llm_provider="vercel_ai_gateway",
|
||||
timeout=timeout,
|
||||
headers=headers,
|
||||
encoding=get_encoding(),
|
||||
api_key=api_key,
|
||||
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
|
||||
client=client,
|
||||
)
|
||||
## LOGGING
|
||||
logging.post_call(input=messages, api_key=openai.api_key, original_response=response)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
def complete_edenai(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
api_base: Final = litellm.EdenAIChatConfig.get_api_base(ctx.api_base)
|
||||
api_key: Final = litellm.EdenAIChatConfig.get_api_key(ctx.api_key or litellm.api_key)
|
||||
response: Final = base_llm_http_handler.completion(
|
||||
model=ctx.model,
|
||||
messages=ctx.messages,
|
||||
api_base=api_base,
|
||||
custom_llm_provider="edenai",
|
||||
model_response=ctx.model_response,
|
||||
encoding=get_encoding(),
|
||||
logging_obj=ctx.logging,
|
||||
optional_params=ctx.optional_params,
|
||||
timeout=ctx.timeout,
|
||||
litellm_params=ctx.litellm_params,
|
||||
shared_session=ctx.shared_session,
|
||||
acompletion=ctx.acompletion,
|
||||
stream=ctx.stream,
|
||||
api_key=api_key,
|
||||
headers=ctx.headers or litellm.headers,
|
||||
client=dispatch_client_http(ctx),
|
||||
provider_config=ctx.provider_config,
|
||||
)
|
||||
ctx.logging.post_call(input=ctx.messages, api_key=api_key, original_response=response)
|
||||
return response
|
||||
|
||||
|
||||
def complete_fal_ai(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
if ctx.stream:
|
||||
raise litellm.FalAIError(
|
||||
status_code=400,
|
||||
message="fal_ai chat completions do not support streaming",
|
||||
)
|
||||
api_base: Final = litellm.FalAIChatConfig.get_api_base(ctx.api_base)
|
||||
api_key: Final = litellm.FalAIChatConfig.get_api_key(ctx.api_key or litellm.api_key)
|
||||
response: Final = base_llm_http_handler.completion(
|
||||
model=ctx.model,
|
||||
messages=ctx.messages,
|
||||
api_base=api_base,
|
||||
custom_llm_provider="fal_ai",
|
||||
model_response=ctx.model_response,
|
||||
encoding=get_encoding(),
|
||||
logging_obj=ctx.logging,
|
||||
optional_params=ctx.optional_params,
|
||||
timeout=ctx.timeout,
|
||||
litellm_params=ctx.litellm_params,
|
||||
shared_session=ctx.shared_session,
|
||||
acompletion=ctx.acompletion,
|
||||
stream=ctx.stream,
|
||||
api_key=api_key,
|
||||
headers=ctx.headers or litellm.headers,
|
||||
client=dispatch_client_http(ctx),
|
||||
provider_config=ctx.provider_config,
|
||||
)
|
||||
ctx.logging.post_call(input=ctx.messages, api_key=api_key, original_response=response)
|
||||
return response
|
||||
311
litellm/native/chat_providers/google.py
Normal file
311
litellm/native/chat_providers/google.py
Normal file
|
|
@ -0,0 +1,311 @@
|
|||
from typing import Final
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import safe_deep_copy
|
||||
from litellm.llms.gemini.common_utils import get_api_key_from_env
|
||||
from litellm.llms.vertex_ai import vertex_ai_non_gemini
|
||||
from litellm.llms.vertex_ai.common_utils import VertexAIModelRoute, get_vertex_ai_model_route
|
||||
from litellm.native.chat_providers.handlers import (
|
||||
base_llm_http_handler,
|
||||
gdc_transformation,
|
||||
vertex_chat_completion,
|
||||
vertex_gemma_chat_completion,
|
||||
vertex_model_garden_chat_completion,
|
||||
vertex_partner_models_chat_completion,
|
||||
)
|
||||
from litellm.native.chat_providers.support import dispatch_client_http, get_encoding, print_verbose
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult
|
||||
from litellm.utils import CustomStreamWrapper, get_secret
|
||||
|
||||
|
||||
def complete_vertex_ai_beta(
|
||||
ctx: CompletionDispatchContext,
|
||||
) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key: Final = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logger_fn: Final = ctx.logger_fn
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
vertex_ai_project: Final = (
|
||||
optional_params.pop("vertex_project", None)
|
||||
or optional_params.pop("vertex_ai_project", None)
|
||||
or litellm.vertex_project
|
||||
or get_secret("VERTEXAI_PROJECT")
|
||||
)
|
||||
vertex_ai_location: Final = (
|
||||
optional_params.pop("vertex_location", None)
|
||||
or optional_params.pop("vertex_ai_location", None)
|
||||
or litellm.vertex_location
|
||||
or get_secret("VERTEXAI_LOCATION")
|
||||
)
|
||||
vertex_credentials: Final = (
|
||||
optional_params.pop("vertex_credentials", None)
|
||||
or optional_params.pop("vertex_ai_credentials", None)
|
||||
or get_secret("VERTEXAI_CREDENTIALS")
|
||||
)
|
||||
|
||||
gemini_api_key: Final = (
|
||||
api_key
|
||||
or get_api_key_from_env()
|
||||
or get_secret("PALM_API_KEY") # older palm api key should also work
|
||||
or litellm.api_key
|
||||
)
|
||||
|
||||
api_base = api_base or litellm.api_base or get_secret("GEMINI_API_BASE")
|
||||
new_params: Final = safe_deep_copy(optional_params or {})
|
||||
return vertex_chat_completion.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
optional_params=new_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
encoding=get_encoding(),
|
||||
vertex_location=vertex_ai_location,
|
||||
vertex_project=vertex_ai_project,
|
||||
vertex_credentials=vertex_credentials,
|
||||
gemini_api_key=gemini_api_key,
|
||||
logging_obj=logging,
|
||||
acompletion=acompletion,
|
||||
timeout=timeout,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
client=client,
|
||||
api_base=api_base,
|
||||
extra_headers=headers,
|
||||
)
|
||||
|
||||
|
||||
def complete_vertex_ai(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
custom_prompt_dict: Final = ctx.custom_prompt_dict
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logger_fn: Final = ctx.logger_fn
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
vertex_ai_project: Final = (
|
||||
optional_params.pop("vertex_project", None)
|
||||
or optional_params.pop("vertex_ai_project", None)
|
||||
or litellm.vertex_project
|
||||
or get_secret("VERTEXAI_PROJECT")
|
||||
)
|
||||
vertex_ai_location: Final = (
|
||||
optional_params.pop("vertex_location", None)
|
||||
or optional_params.pop("vertex_ai_location", None)
|
||||
or litellm.vertex_location
|
||||
or get_secret("VERTEXAI_LOCATION")
|
||||
)
|
||||
vertex_credentials: Final = (
|
||||
optional_params.pop("vertex_credentials", None)
|
||||
or optional_params.pop("vertex_ai_credentials", None)
|
||||
or get_secret("VERTEXAI_CREDENTIALS")
|
||||
)
|
||||
|
||||
api_base = api_base or litellm.api_base or get_secret("VERTEXAI_API_BASE")
|
||||
|
||||
new_params: Final = safe_deep_copy(optional_params or {})
|
||||
model_route: Final = get_vertex_ai_model_route(model=model, litellm_params=litellm_params)
|
||||
|
||||
if model_route == VertexAIModelRoute.PARTNER_MODELS:
|
||||
model_response = vertex_partner_models_chat_completion.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
optional_params=new_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
encoding=get_encoding(),
|
||||
api_base=api_base,
|
||||
vertex_location=vertex_ai_location,
|
||||
vertex_project=vertex_ai_project,
|
||||
vertex_credentials=vertex_credentials,
|
||||
logging_obj=logging,
|
||||
acompletion=acompletion,
|
||||
headers=headers,
|
||||
custom_prompt_dict=custom_prompt_dict,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
)
|
||||
elif model_route == VertexAIModelRoute.GEMINI:
|
||||
model_response = vertex_chat_completion.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
optional_params=new_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
encoding=get_encoding(),
|
||||
vertex_location=vertex_ai_location,
|
||||
vertex_project=vertex_ai_project,
|
||||
vertex_credentials=vertex_credentials,
|
||||
gemini_api_key=None,
|
||||
logging_obj=logging,
|
||||
acompletion=acompletion,
|
||||
timeout=timeout,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
client=client,
|
||||
api_base=api_base,
|
||||
extra_headers=headers,
|
||||
)
|
||||
elif model_route == VertexAIModelRoute.GEMMA:
|
||||
# Vertex Gemma Models with custom prediction endpoint
|
||||
model_response = vertex_gemma_chat_completion.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
optional_params=new_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
encoding=get_encoding(),
|
||||
api_base=api_base,
|
||||
vertex_location=vertex_ai_location,
|
||||
vertex_project=vertex_ai_project,
|
||||
vertex_credentials=vertex_credentials,
|
||||
logging_obj=logging,
|
||||
acompletion=acompletion,
|
||||
headers=headers,
|
||||
custom_prompt_dict=custom_prompt_dict,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
)
|
||||
elif model_route == VertexAIModelRoute.MODEL_GARDEN:
|
||||
# Vertex Model Garden - OpenAI compatible models
|
||||
model_response = vertex_model_garden_chat_completion.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
optional_params=new_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
encoding=get_encoding(),
|
||||
api_base=api_base,
|
||||
vertex_location=vertex_ai_location,
|
||||
vertex_project=vertex_ai_project,
|
||||
vertex_credentials=vertex_credentials,
|
||||
logging_obj=logging,
|
||||
acompletion=acompletion,
|
||||
headers=headers,
|
||||
custom_prompt_dict=custom_prompt_dict,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
)
|
||||
elif model_route == VertexAIModelRoute.AGENT_ENGINE:
|
||||
# Vertex AI Agent Engine (Reasoning Engines)
|
||||
from litellm.llms.vertex_ai.agent_engine.transformation import (
|
||||
VertexAgentEngineConfig,
|
||||
)
|
||||
|
||||
vertex_agent_engine_config: Final = VertexAgentEngineConfig()
|
||||
|
||||
# Update litellm_params with vertex credentials
|
||||
litellm_params["vertex_project"] = vertex_ai_project
|
||||
litellm_params["vertex_location"] = vertex_ai_location
|
||||
litellm_params["vertex_credentials"] = vertex_credentials
|
||||
|
||||
model_response = base_llm_http_handler.completion(
|
||||
model=model,
|
||||
stream=stream,
|
||||
messages=messages,
|
||||
model_response=model_response,
|
||||
optional_params=new_params,
|
||||
litellm_params=litellm_params,
|
||||
encoding=get_encoding(),
|
||||
api_key=None,
|
||||
api_base=api_base,
|
||||
logging_obj=logging,
|
||||
acompletion=acompletion,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
custom_llm_provider="vertex_ai",
|
||||
provider_config=vertex_agent_engine_config,
|
||||
headers=headers or {},
|
||||
)
|
||||
else: # VertexAIModelRoute.NON_GEMINI
|
||||
model_response = vertex_ai_non_gemini.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
optional_params=new_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
encoding=get_encoding(),
|
||||
vertex_location=vertex_ai_location,
|
||||
vertex_project=vertex_ai_project,
|
||||
vertex_credentials=vertex_credentials,
|
||||
logging_obj=logging,
|
||||
acompletion=acompletion,
|
||||
)
|
||||
|
||||
if "stream" in optional_params and optional_params["stream"] is True and acompletion is False:
|
||||
return CustomStreamWrapper(
|
||||
model_response,
|
||||
model,
|
||||
custom_llm_provider="vertex_ai",
|
||||
logging_obj=logging,
|
||||
)
|
||||
return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
|
||||
|
||||
|
||||
def complete_gdc(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
api_key = api_key or litellm.gdc_key or get_secret_str("GDC_API_KEY") or litellm.api_key
|
||||
api_base = api_base or litellm.gdc_api_base or get_secret_str("GDC_API_BASE") or litellm.api_base
|
||||
|
||||
return base_llm_http_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
model_response=model_response,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
encoding=get_encoding(),
|
||||
stream=stream,
|
||||
provider_config=gdc_transformation,
|
||||
)
|
||||
51
litellm/native/chat_providers/handlers.py
Normal file
51
litellm/native/chat_providers/handlers.py
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
from typing import Final
|
||||
|
||||
from litellm.llms.anthropic.chat import AnthropicChatCompletion
|
||||
from litellm.llms.azure.azure import AzureChatCompletion
|
||||
from litellm.llms.azure.chat.o_series_handler import AzureOpenAIO1ChatCompletion
|
||||
from litellm.llms.azure.completion.handler import AzureTextCompletion
|
||||
from litellm.llms.azure_ai.anthropic.handler import AzureAnthropicChatCompletion
|
||||
from litellm.llms.bedrock.chat import BedrockConverseLLM
|
||||
from litellm.llms.bytez.chat.transformation import BytezChatConfig
|
||||
from litellm.llms.codestral.completion.handler import CodestralTextCompletion
|
||||
from litellm.llms.custom_httpx.aiohttp_handler import BaseLLMAIOHTTPHandler
|
||||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from litellm.llms.gdc.chat.transformation import GDCGeminiConfig
|
||||
from litellm.llms.lemonade.chat.transformation import LemonadeChatConfig
|
||||
from litellm.llms.openai.completion.handler import OpenAITextCompletion
|
||||
from litellm.llms.openai.openai import OpenAIChatCompletion
|
||||
from litellm.llms.openai_like.chat.handler import OpenAILikeChatHandler
|
||||
from litellm.llms.ovhcloud.chat.transformation import OVHCloudChatConfig
|
||||
from litellm.llms.predibase.chat.handler import PredibaseChatCompletion
|
||||
from litellm.llms.sagemaker.completion.handler import SagemakerLLM
|
||||
from litellm.llms.sap.chat.handler import GenAIHubOrchestration
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.main import VertexAIPartnerModels
|
||||
from litellm.llms.vertex_ai.vertex_gemma_models.main import VertexAIGemmaModels
|
||||
from litellm.llms.vertex_ai.vertex_model_garden.main import VertexAIModelGardenModels
|
||||
from litellm.llms.watsonx.chat.handler import WatsonXChatHandler
|
||||
|
||||
openai_chat_completions = OpenAIChatCompletion()
|
||||
openai_text_completions: Final = OpenAITextCompletion()
|
||||
sap_gen_ai_hub_chat_completions: Final = GenAIHubOrchestration()
|
||||
anthropic_chat_completions: Final = AnthropicChatCompletion()
|
||||
azure_anthropic_chat_completions: Final = AzureAnthropicChatCompletion()
|
||||
azure_chat_completions = AzureChatCompletion()
|
||||
azure_o1_chat_completions: Final = AzureOpenAIO1ChatCompletion()
|
||||
azure_text_completions: Final = AzureTextCompletion()
|
||||
predibase_chat_completions: Final = PredibaseChatCompletion()
|
||||
codestral_text_completions: Final = CodestralTextCompletion()
|
||||
bedrock_converse_chat_completion: Final = BedrockConverseLLM()
|
||||
vertex_chat_completion: Final = VertexLLM()
|
||||
vertex_partner_models_chat_completion: Final = VertexAIPartnerModels()
|
||||
vertex_gemma_chat_completion: Final = VertexAIGemmaModels()
|
||||
vertex_model_garden_chat_completion: Final = VertexAIModelGardenModels()
|
||||
gdc_transformation: Final = GDCGeminiConfig()
|
||||
sagemaker_llm: Final = SagemakerLLM()
|
||||
watsonx_chat_completion: Final = WatsonXChatHandler()
|
||||
openai_like_chat_completion: Final = OpenAILikeChatHandler()
|
||||
base_llm_http_handler = BaseLLMHTTPHandler()
|
||||
base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler()
|
||||
bytez_transformation: Final = BytezChatConfig()
|
||||
ovhcloud_transformation: Final = OVHCloudChatConfig()
|
||||
lemonade_transformation: Final = LemonadeChatConfig()
|
||||
926
litellm/native/chat_providers/model_labs.py
Normal file
926
litellm/native/chat_providers/model_labs.py
Normal file
|
|
@ -0,0 +1,926 @@
|
|||
import os
|
||||
from types import MappingProxyType
|
||||
from typing import Final, cast # noqa: TID251 # casts moved verbatim from litellm/main.py
|
||||
|
||||
import httpx
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.cohere.common_utils import CohereModelInfo
|
||||
from litellm.llms.deprecated_providers import aleph_alpha
|
||||
from litellm.llms.nlp_cloud.chat.handler import completion as nlp_cloud_chat_completion
|
||||
from litellm.llms.replicate.chat.handler import completion as replicate_chat_completion
|
||||
from litellm.native.chat_providers.handlers import (
|
||||
base_llm_http_handler,
|
||||
bytez_transformation,
|
||||
codestral_text_completions,
|
||||
openai_like_chat_completion,
|
||||
predibase_chat_completions,
|
||||
)
|
||||
from litellm.native.chat_providers.support import (
|
||||
dispatch_client_http,
|
||||
get_encoding,
|
||||
print_verbose,
|
||||
provider_config_items,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult
|
||||
from litellm.utils import CustomStreamWrapper, get_secret
|
||||
|
||||
|
||||
def complete_deepseek(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base: Final = ctx.api_base
|
||||
api_key: Final = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
provider_config: Final = ctx.provider_config
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
try:
|
||||
response: Final = base_llm_http_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
model_response=model_response,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
encoding=get_encoding(),
|
||||
stream=stream,
|
||||
provider_config=provider_config,
|
||||
)
|
||||
except Exception as e:
|
||||
## LOGGING - log the original exception returned
|
||||
logging.post_call(
|
||||
input=messages,
|
||||
api_key=api_key,
|
||||
original_response=str(e),
|
||||
additional_args={"headers": headers},
|
||||
)
|
||||
raise e
|
||||
|
||||
return response
|
||||
|
||||
|
||||
def complete_fireworks_ai(
|
||||
ctx: CompletionDispatchContext,
|
||||
) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base: Final = ctx.api_base
|
||||
api_key: Final = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
provider_config: Final = ctx.provider_config
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
optional_params: Final = (
|
||||
provider_config.map_extra_body_params(optional_params=ctx.optional_params, model=model)
|
||||
if isinstance(provider_config, litellm.FireworksAIConfig)
|
||||
else ctx.optional_params
|
||||
)
|
||||
|
||||
try:
|
||||
response: Final = base_llm_http_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
model_response=model_response,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
encoding=get_encoding(),
|
||||
stream=stream,
|
||||
provider_config=provider_config,
|
||||
)
|
||||
except Exception as e:
|
||||
## LOGGING - log the original exception returned
|
||||
logging.post_call(
|
||||
input=messages,
|
||||
api_key=api_key,
|
||||
original_response=str(e),
|
||||
additional_args={"headers": headers},
|
||||
)
|
||||
raise e
|
||||
|
||||
return response
|
||||
|
||||
|
||||
def complete_together_ai(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base: Final = ctx.api_base
|
||||
api_key: Final = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
provider_config: Final = ctx.provider_config
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
try:
|
||||
response: Final = base_llm_http_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
model_response=model_response,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
encoding=get_encoding(),
|
||||
stream=stream,
|
||||
provider_config=provider_config,
|
||||
)
|
||||
except Exception as e:
|
||||
logging.post_call(
|
||||
input=messages,
|
||||
api_key=api_key,
|
||||
original_response=str(e),
|
||||
additional_args=MappingProxyType({"headers": headers}),
|
||||
)
|
||||
raise
|
||||
|
||||
return response
|
||||
|
||||
|
||||
def complete_scaledown(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
try:
|
||||
return base_llm_http_handler.completion(
|
||||
model=ctx.model,
|
||||
messages=ctx.messages,
|
||||
headers=ctx.headers,
|
||||
model_response=ctx.model_response,
|
||||
api_key=ctx.api_key,
|
||||
api_base=ctx.api_base,
|
||||
acompletion=ctx.acompletion,
|
||||
logging_obj=ctx.logging,
|
||||
optional_params=ctx.optional_params,
|
||||
litellm_params=ctx.litellm_params,
|
||||
shared_session=ctx.shared_session,
|
||||
timeout=cast( # cast-ok: CompletionTimeout.resolve normalizes this value
|
||||
float | httpx.Timeout, ctx.timeout
|
||||
),
|
||||
client=dispatch_client_http(ctx),
|
||||
custom_llm_provider=ctx.custom_llm_provider,
|
||||
encoding=get_encoding(),
|
||||
stream=ctx.stream,
|
||||
provider_config=ctx.provider_config,
|
||||
)
|
||||
except Exception as exc:
|
||||
ctx.logging.post_call(
|
||||
input=ctx.messages,
|
||||
api_key=ctx.api_key,
|
||||
original_response=str(exc),
|
||||
additional_args={"headers": ctx.headers},
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
def complete_xai(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base: Final = ctx.api_base
|
||||
api_key: Final = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
provider_config: Final = ctx.provider_config
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
try:
|
||||
response: Final = base_llm_http_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
model_response=model_response,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
encoding=get_encoding(),
|
||||
stream=stream,
|
||||
provider_config=provider_config,
|
||||
)
|
||||
except Exception as e:
|
||||
## LOGGING - log the original exception returned
|
||||
logging.post_call(
|
||||
input=messages,
|
||||
api_key=api_key,
|
||||
original_response=str(e),
|
||||
additional_args={"headers": headers},
|
||||
)
|
||||
raise e
|
||||
|
||||
return response
|
||||
|
||||
|
||||
def complete_groq(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
api_base = (
|
||||
api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there
|
||||
or litellm.api_base
|
||||
or get_secret("GROQ_API_BASE")
|
||||
or "https://api.groq.com/openai/v1"
|
||||
)
|
||||
|
||||
# set API KEY
|
||||
api_key = (
|
||||
api_key
|
||||
or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there
|
||||
or litellm.groq_key
|
||||
or get_secret("GROQ_API_KEY")
|
||||
)
|
||||
|
||||
headers = headers or litellm.headers
|
||||
|
||||
## LOAD CONFIG - if set
|
||||
config: Final = litellm.GroqChatConfig.get_config()
|
||||
for k, v in provider_config_items(config):
|
||||
if (
|
||||
k not in optional_params
|
||||
): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in
|
||||
optional_params[k] = v
|
||||
|
||||
return base_llm_http_handler.completion(
|
||||
model=model,
|
||||
stream=stream,
|
||||
messages=messages,
|
||||
acompletion=acompletion,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
timeout=timeout,
|
||||
headers=headers,
|
||||
encoding=get_encoding(),
|
||||
api_key=api_key,
|
||||
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
|
||||
client=client,
|
||||
)
|
||||
|
||||
|
||||
def complete_opencode_go(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
return base_llm_http_handler.completion( # pyright: ignore[reportUnknownMemberType] # shared legacy handler has untyped request mappings
|
||||
model=ctx.model,
|
||||
stream=ctx.stream,
|
||||
messages=ctx.messages, # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # existing dispatch context carries validated messages
|
||||
acompletion=ctx.acompletion,
|
||||
api_base=ctx.api_base,
|
||||
model_response=ctx.model_response,
|
||||
optional_params=ctx.optional_params, # pyright: ignore[reportUnknownMemberType] # existing dispatch context carries the mapped provider parameters
|
||||
litellm_params=ctx.litellm_params,
|
||||
shared_session=ctx.shared_session,
|
||||
custom_llm_provider=ctx.custom_llm_provider,
|
||||
timeout=ctx.timeout, # pyright: ignore[reportArgumentType] # completion normalizes timeout before constructing this context
|
||||
headers=ctx.headers, # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # legacy context retains caller headers
|
||||
encoding=get_encoding(),
|
||||
api_key=ctx.api_key,
|
||||
logging_obj=ctx.logging,
|
||||
client=dispatch_client_http(ctx),
|
||||
)
|
||||
|
||||
|
||||
def complete_minimax(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
provider_config: Final = ctx.provider_config
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
api_key = api_key or get_secret_str("MINIMAX_API_KEY") or litellm.api_key
|
||||
|
||||
api_base = api_base or litellm.api_base or get_secret_str("MINIMAX_API_BASE") or "https://api.minimax.io/v1"
|
||||
|
||||
response: Final = base_llm_http_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
api_base=api_base,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
model_response=model_response,
|
||||
encoding=get_encoding(),
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
timeout=timeout,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
acompletion=acompletion,
|
||||
stream=stream,
|
||||
api_key=api_key,
|
||||
headers=headers,
|
||||
client=client,
|
||||
provider_config=provider_config,
|
||||
)
|
||||
logging.post_call(input=messages, api_key=api_key, original_response=response)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
def complete_mistral(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
provider_config: Final = ctx.provider_config
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
api_key = api_key or litellm.api_key or get_secret("MISTRAL_API_KEY")
|
||||
api_base = api_base or litellm.api_base or get_secret("MISTRAL_API_BASE") or "https://api.mistral.ai/v1"
|
||||
|
||||
return base_llm_http_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
api_base=api_base,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
model_response=model_response,
|
||||
encoding=get_encoding(),
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
timeout=timeout,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
acompletion=acompletion,
|
||||
stream=stream,
|
||||
api_key=api_key,
|
||||
headers=headers,
|
||||
client=client,
|
||||
provider_config=provider_config,
|
||||
)
|
||||
|
||||
|
||||
def complete_replicate(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key: Final = ctx.api_key
|
||||
custom_prompt_dict = ctx.custom_prompt_dict
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logger_fn: Final = ctx.logger_fn
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
|
||||
replicate_key: Final = (
|
||||
api_key
|
||||
or litellm.replicate_key
|
||||
or litellm.api_key
|
||||
or get_secret("REPLICATE_API_KEY")
|
||||
or get_secret("REPLICATE_API_TOKEN")
|
||||
)
|
||||
|
||||
api_base = api_base or litellm.api_base or get_secret("REPLICATE_API_BASE") or "https://api.replicate.com/v1"
|
||||
|
||||
custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict
|
||||
|
||||
model_response = replicate_chat_completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
encoding=get_encoding(), # for calculating input/output tokens
|
||||
api_key=replicate_key,
|
||||
logging_obj=logging,
|
||||
custom_prompt_dict=custom_prompt_dict,
|
||||
acompletion=acompletion,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
if optional_params.get("stream", False) is True:
|
||||
## LOGGING
|
||||
logging.post_call(
|
||||
input=messages,
|
||||
api_key=replicate_key,
|
||||
original_response=model_response,
|
||||
)
|
||||
|
||||
return model_response
|
||||
|
||||
|
||||
def complete_nlp_cloud(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key: Final = ctx.api_key
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logger_fn: Final = ctx.logger_fn
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
|
||||
nlp_cloud_key: Final = api_key or litellm.nlp_cloud_key or get_secret("NLP_CLOUD_API_KEY") or litellm.api_key
|
||||
|
||||
api_base = api_base or litellm.api_base or get_secret("NLP_CLOUD_API_BASE") or "https://api.nlpcloud.io/v1/gpu/"
|
||||
|
||||
response = nlp_cloud_chat_completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
encoding=get_encoding(),
|
||||
api_key=nlp_cloud_key,
|
||||
logging_obj=logging,
|
||||
)
|
||||
|
||||
if "stream" in optional_params and optional_params["stream"] is True:
|
||||
# don't try to access stream object,
|
||||
response = CustomStreamWrapper(
|
||||
response,
|
||||
model,
|
||||
custom_llm_provider="nlp_cloud",
|
||||
logging_obj=logging,
|
||||
)
|
||||
|
||||
if optional_params.get("stream", False) or acompletion is True:
|
||||
## LOGGING
|
||||
logging.post_call(
|
||||
input=messages,
|
||||
api_key=api_key,
|
||||
original_response=response,
|
||||
)
|
||||
|
||||
return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
|
||||
|
||||
|
||||
def complete_aleph_alpha(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
api_base = ctx.api_base
|
||||
api_key: Final = ctx.api_key
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logger_fn: Final = ctx.logger_fn
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
|
||||
aleph_alpha_key: Final = (
|
||||
api_key
|
||||
or litellm.aleph_alpha_key
|
||||
or get_secret("ALEPH_ALPHA_API_KEY")
|
||||
or get_secret("ALEPHALPHA_API_KEY")
|
||||
or litellm.api_key
|
||||
)
|
||||
|
||||
api_base = (
|
||||
api_base or litellm.api_base or get_secret("ALEPH_ALPHA_API_BASE") or "https://api.aleph-alpha.com/complete"
|
||||
)
|
||||
|
||||
model_response = aleph_alpha.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
encoding=get_encoding(),
|
||||
default_max_tokens_to_sample=litellm.max_tokens,
|
||||
api_key=aleph_alpha_key,
|
||||
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
|
||||
)
|
||||
|
||||
if "stream" in optional_params and optional_params["stream"] is True:
|
||||
# don't try to access stream object,
|
||||
return CustomStreamWrapper(
|
||||
model_response,
|
||||
model,
|
||||
custom_llm_provider="aleph_alpha",
|
||||
logging_obj=logging,
|
||||
)
|
||||
return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
|
||||
|
||||
|
||||
def complete_cohere_chat(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key: Final = ctx.api_key
|
||||
extra_headers: Final = ctx.extra_headers
|
||||
headers = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
provider_config: Final = ctx.provider_config
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
cohere_key: Final = (
|
||||
api_key
|
||||
or litellm.cohere_key
|
||||
or get_secret_str("COHERE_API_KEY")
|
||||
or get_secret_str("CO_API_KEY")
|
||||
or litellm.api_key
|
||||
)
|
||||
|
||||
cohere_route: Final = CohereModelInfo.get_cohere_route(model)
|
||||
verbose_logger.debug("Cohere route: %s", cohere_route)
|
||||
# Set API base based on route
|
||||
if cohere_route == "v2":
|
||||
api_base = api_base or litellm.api_base or get_secret_str("COHERE_API_BASE") or "https://api.cohere.com/v2/chat"
|
||||
# Remove v2/ prefix from model name for the actual API call
|
||||
if "v2/" in model:
|
||||
model = model.replace("v2/", "")
|
||||
else:
|
||||
api_base = api_base or litellm.api_base or get_secret_str("COHERE_API_BASE") or "https://api.cohere.ai/v1/chat"
|
||||
|
||||
headers = headers or litellm.headers or {}
|
||||
if headers is None:
|
||||
headers = {}
|
||||
|
||||
if extra_headers is not None:
|
||||
headers.update(extra_headers)
|
||||
|
||||
verbose_logger.debug("Model: %s, API Base: %s", model, api_base)
|
||||
verbose_logger.debug("Provider Config: %s", provider_config)
|
||||
return base_llm_http_handler.completion(
|
||||
model=model,
|
||||
stream=stream,
|
||||
messages=messages,
|
||||
acompletion=acompletion,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
custom_llm_provider="cohere_chat",
|
||||
timeout=timeout,
|
||||
headers=headers,
|
||||
encoding=get_encoding(),
|
||||
api_key=cohere_key,
|
||||
provider_config=provider_config,
|
||||
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
|
||||
)
|
||||
|
||||
|
||||
def complete_maritalk(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
api_base = ctx.api_base
|
||||
api_key: Final = ctx.api_key
|
||||
custom_prompt_dict: Final = ctx.custom_prompt_dict
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logger_fn: Final = ctx.logger_fn
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
|
||||
maritalk_key: Final = api_key or litellm.maritalk_key or get_secret("MARITALK_API_KEY") or litellm.api_key
|
||||
|
||||
api_base = api_base or litellm.api_base or get_secret("MARITALK_API_BASE") or "https://chat.maritaca.ai/api"
|
||||
|
||||
return openai_like_chat_completion.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
encoding=get_encoding(),
|
||||
api_key=maritalk_key,
|
||||
logging_obj=logging,
|
||||
custom_llm_provider="maritalk",
|
||||
custom_prompt_dict=custom_prompt_dict,
|
||||
)
|
||||
|
||||
|
||||
def complete_huggingface(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base: Final = ctx.api_base
|
||||
api_key: Final = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
huggingface_key: Final = (
|
||||
api_key
|
||||
or litellm.huggingface_key
|
||||
or os.environ.get("HF_TOKEN")
|
||||
or os.environ.get("HUGGINGFACE_API_KEY")
|
||||
or litellm.api_key
|
||||
)
|
||||
hf_headers: Final = headers or litellm.headers
|
||||
return base_llm_http_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=hf_headers,
|
||||
model_response=model_response,
|
||||
api_key=huggingface_key,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
encoding=get_encoding(),
|
||||
stream=stream,
|
||||
)
|
||||
|
||||
|
||||
def complete_compactifai(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
provider_config: Final = ctx.provider_config
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
api_key = api_key or get_secret_str("COMPACTIFAI_API_KEY") or litellm.api_key
|
||||
|
||||
api_base = api_base or "https://api.compactif.ai/v1"
|
||||
|
||||
## COMPLETION CALL
|
||||
return base_llm_http_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
model_response=model_response,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
encoding=get_encoding(),
|
||||
stream=stream,
|
||||
provider_config=provider_config,
|
||||
)
|
||||
|
||||
|
||||
def complete_predibase(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
custom_prompt_dict: Final = ctx.custom_prompt_dict
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logger_fn: Final = ctx.logger_fn
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
tenant_id: Final = (
|
||||
optional_params.pop("tenant_id", None)
|
||||
or optional_params.pop("predibase_tenant_id", None)
|
||||
or litellm.predibase_tenant_id
|
||||
or get_secret("PREDIBASE_TENANT_ID")
|
||||
)
|
||||
|
||||
if tenant_id is None:
|
||||
raise ValueError(
|
||||
"Missing Predibase Tenant ID - Required for making the request. Set dynamically (e.g. `completion(..tenant_id=<MY-ID>)`) or in env - `PREDIBASE_TENANT_ID`."
|
||||
)
|
||||
|
||||
api_base = (
|
||||
api_base
|
||||
or optional_params.pop("api_base", None)
|
||||
or optional_params.pop("base_url", None)
|
||||
or litellm.api_base
|
||||
or get_secret("PREDIBASE_API_BASE")
|
||||
)
|
||||
|
||||
api_key = api_key or litellm.api_key or litellm.predibase_key or get_secret("PREDIBASE_API_KEY")
|
||||
|
||||
_model_response: Final = predibase_chat_completions.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
encoding=get_encoding(),
|
||||
logging_obj=logging,
|
||||
acompletion=acompletion,
|
||||
api_base=api_base,
|
||||
custom_prompt_dict=custom_prompt_dict,
|
||||
api_key=api_key,
|
||||
tenant_id=tenant_id,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
if "stream" in optional_params and optional_params["stream"] is True and acompletion is False:
|
||||
return _model_response
|
||||
return _model_response
|
||||
|
||||
|
||||
def complete_text_completion_codestral(
|
||||
ctx: CompletionDispatchContext,
|
||||
) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
custom_prompt_dict: Final = ctx.custom_prompt_dict
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logger_fn: Final = ctx.logger_fn
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
optional_params: Final = ctx.optional_params
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
api_base = (
|
||||
api_base
|
||||
or optional_params.pop("api_base", None)
|
||||
or optional_params.pop("base_url", None)
|
||||
or litellm.api_base
|
||||
or "https://codestral.mistral.ai/v1/fim/completions"
|
||||
)
|
||||
|
||||
api_key = api_key or litellm.api_key or get_secret("CODESTRAL_API_KEY")
|
||||
|
||||
text_completion_model_response: Final = litellm.TextCompletionResponse(stream=stream)
|
||||
|
||||
_model_response: Final = codestral_text_completions.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
model_response=text_completion_model_response,
|
||||
print_verbose=print_verbose,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
encoding=get_encoding(),
|
||||
logging_obj=logging,
|
||||
acompletion=acompletion,
|
||||
api_base=api_base,
|
||||
custom_prompt_dict=custom_prompt_dict,
|
||||
api_key=api_key,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
if "stream" in optional_params and optional_params["stream"] is True and acompletion is False:
|
||||
return _model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
|
||||
return _model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
|
||||
|
||||
|
||||
def complete_bytez(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base: Final = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
api_key = api_key or litellm.bytez_key or get_secret_str("BYTEZ_API_KEY") or litellm.api_key
|
||||
|
||||
response: Final = base_llm_http_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
model_response=model_response,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
encoding=get_encoding(),
|
||||
stream=stream,
|
||||
provider_config=bytez_transformation,
|
||||
)
|
||||
|
||||
return response
|
||||
390
litellm/native/chat_providers/openai.py
Normal file
390
litellm/native/chat_providers/openai.py
Normal file
|
|
@ -0,0 +1,390 @@
|
|||
from typing import Final, cast # noqa: TID251 # casts moved verbatim from litellm/main.py
|
||||
|
||||
import openai
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route
|
||||
from litellm.llms.base_llm.chat.transformation import with_attribution_headers
|
||||
from litellm.llms.custom_httpx.http_handler import http2_enabled
|
||||
from litellm.native.chat_providers.handlers import (
|
||||
base_llm_aiohttp_handler,
|
||||
base_llm_http_handler,
|
||||
openai_chat_completions,
|
||||
openai_text_completions,
|
||||
)
|
||||
from litellm.native.chat_providers.support import (
|
||||
dispatch_client_http,
|
||||
dispatch_client_openai,
|
||||
dispatch_metadata,
|
||||
get_encoding,
|
||||
print_verbose,
|
||||
provider_config_items,
|
||||
)
|
||||
from litellm.responses.bridge_check import resolve_openai_api_base
|
||||
from litellm.secret_managers.main import get_secret_bool, get_secret_str
|
||||
from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult
|
||||
from litellm.utils import get_requester_metadata, get_secret
|
||||
|
||||
|
||||
def complete_text_completion_openai(
|
||||
ctx: CompletionDispatchContext,
|
||||
) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
client: Final = dispatch_client_openai(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logger_fn: Final = ctx.logger_fn
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
text_completion: Final = ctx.text_completion
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
openai.api_type = "openai"
|
||||
|
||||
api_base = (
|
||||
api_base
|
||||
or litellm.api_base
|
||||
or get_secret("OPENAI_BASE_URL")
|
||||
or get_secret("OPENAI_API_BASE")
|
||||
or "https://api.openai.com/v1"
|
||||
)
|
||||
|
||||
openai.api_version = None
|
||||
# set API KEY
|
||||
|
||||
api_key = api_key or litellm.api_key or litellm.openai_key or get_secret("OPENAI_API_KEY")
|
||||
|
||||
headers = headers or litellm.headers
|
||||
|
||||
## LOAD CONFIG - if set
|
||||
config: Final = litellm.OpenAITextCompletionConfig.get_config()
|
||||
for k, v in provider_config_items(config):
|
||||
if (
|
||||
k not in optional_params
|
||||
): # completion(top_k=3) > openai_text_config(top_k=3) <- allows for dynamic variables to be passed in
|
||||
optional_params[k] = v
|
||||
if litellm.organization:
|
||||
openai.organization = litellm.organization
|
||||
|
||||
## COMPLETION CALL
|
||||
_response = openai_text_completions.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
api_key=api_key,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
client=client, # pass AsyncOpenAI, OpenAI client
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
if optional_params.get("stream", False) is False and acompletion is False and text_completion is False:
|
||||
# convert to chat completion response
|
||||
_response = litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object(
|
||||
response_object=_response, model_response_object=model_response
|
||||
)
|
||||
|
||||
if optional_params.get("stream", False) or acompletion is True:
|
||||
## LOGGING
|
||||
logging.post_call(
|
||||
input=messages,
|
||||
api_key=api_key,
|
||||
original_response=_response,
|
||||
additional_args={"headers": headers},
|
||||
)
|
||||
return _response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
|
||||
|
||||
|
||||
def complete_aiohttp_openai(
|
||||
ctx: CompletionDispatchContext,
|
||||
) -> CompletionDispatchResult:
|
||||
if http2_enabled():
|
||||
verbose_logger.warning(
|
||||
"litellm.http2 is enabled but aiohttp_openai/ always uses aiohttp, which has no HTTP/2 client; this request stays on HTTP/1.1"
|
||||
)
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
extra_headers: Final = ctx.extra_headers
|
||||
headers = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
api_base = (
|
||||
api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there
|
||||
or litellm.api_base
|
||||
or get_secret("OPENAI_BASE_URL")
|
||||
or get_secret("OPENAI_API_BASE")
|
||||
or "https://api.openai.com/v1"
|
||||
)
|
||||
# set API KEY
|
||||
api_key = (
|
||||
api_key
|
||||
or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there
|
||||
or litellm.openai_key
|
||||
or get_secret("OPENAI_API_KEY")
|
||||
)
|
||||
|
||||
headers = headers or litellm.headers
|
||||
|
||||
if extra_headers is not None:
|
||||
optional_params["extra_headers"] = extra_headers
|
||||
return base_llm_aiohttp_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
model_response=model_response,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
encoding=get_encoding(),
|
||||
stream=stream,
|
||||
)
|
||||
|
||||
|
||||
def complete_custom_openai(
|
||||
ctx: CompletionDispatchContext,
|
||||
) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
custom_prompt_dict: Final = ctx.custom_prompt_dict
|
||||
extra_headers = ctx.extra_headers
|
||||
headers = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logger_fn: Final = ctx.logger_fn
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
metadata: Final = dispatch_metadata(ctx)
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
organization = ctx.organization
|
||||
provider_config: Final = ctx.provider_config
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
# for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there
|
||||
api_base = resolve_openai_api_base(api_base)
|
||||
if custom_llm_provider == "bigmodel" and not api_key:
|
||||
raise litellm.AuthenticationError(
|
||||
message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model
|
||||
)
|
||||
organization = (
|
||||
organization
|
||||
or (None if custom_llm_provider == "bigmodel" else litellm.organization or get_secret("OPENAI_ORGANIZATION"))
|
||||
or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105
|
||||
)
|
||||
openai.organization = organization
|
||||
# set API KEY
|
||||
api_key = (
|
||||
api_key
|
||||
or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there
|
||||
or litellm.openai_key
|
||||
or get_secret("OPENAI_API_KEY")
|
||||
)
|
||||
|
||||
headers = headers or litellm.headers
|
||||
outbound_headers: Final = (
|
||||
headers
|
||||
if provider_config is None
|
||||
else with_attribution_headers(provider_config.get_attribution_headers(), headers)
|
||||
)
|
||||
|
||||
# Add GitHub Copilot headers (same as /responses endpoint does)
|
||||
if custom_llm_provider == "github_copilot":
|
||||
from litellm.llms.github_copilot.authenticator import Authenticator
|
||||
from litellm.llms.github_copilot.common_utils import (
|
||||
get_copilot_default_headers,
|
||||
pin_session_authorization,
|
||||
)
|
||||
from litellm.llms.github_copilot.per_user_auth import (
|
||||
require_github_copilot_user_session,
|
||||
)
|
||||
|
||||
user_session: Final = require_github_copilot_user_session(litellm_params)
|
||||
copilot_headers: Final = get_copilot_default_headers(
|
||||
user_session.token if user_session is not None else Authenticator().get_api_key()
|
||||
)
|
||||
if extra_headers:
|
||||
copilot_headers.update(
|
||||
cast("dict[str, str]", extra_headers) # cast-ok: extra_headers is a str-valued request dict
|
||||
)
|
||||
if user_session is not None:
|
||||
pin_session_authorization(copilot_headers, user_session.token)
|
||||
extra_headers = copilot_headers
|
||||
|
||||
use_base_llm_http_handler: Final = get_secret_bool("EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER")
|
||||
|
||||
if extra_headers is not None and not use_base_llm_http_handler:
|
||||
optional_params["extra_headers"] = extra_headers
|
||||
|
||||
if litellm.enable_preview_features and metadata is not None: # [PREVIEW] allow metadata to be passed to OPENAI
|
||||
openai_metadata: Final = get_requester_metadata(metadata)
|
||||
if openai_metadata is not None:
|
||||
optional_params["metadata"] = openai_metadata
|
||||
|
||||
## LOAD CONFIG - if set
|
||||
config: Final = litellm.OpenAIConfig.get_config()
|
||||
for k, v in provider_config_items(config):
|
||||
if (
|
||||
k not in optional_params
|
||||
): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in
|
||||
optional_params[k] = v
|
||||
|
||||
## COMPLETION CALL
|
||||
try:
|
||||
if use_base_llm_http_handler:
|
||||
response = base_llm_http_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
api_base=api_base,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
model_response=model_response,
|
||||
encoding=get_encoding(),
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
timeout=timeout,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
acompletion=acompletion,
|
||||
stream=stream,
|
||||
api_key=api_key,
|
||||
headers=outbound_headers,
|
||||
client=client,
|
||||
provider_config=provider_config,
|
||||
)
|
||||
else:
|
||||
response = openai_chat_completions.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=outbound_headers,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
timeout=timeout,
|
||||
custom_prompt_dict=custom_prompt_dict,
|
||||
client=client, # pass AsyncOpenAI, OpenAI client
|
||||
organization=organization,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
shared_session=shared_session,
|
||||
)
|
||||
except Exception as e:
|
||||
## LOGGING - log the original exception returned
|
||||
logging.post_call(
|
||||
input=messages,
|
||||
api_key=api_key,
|
||||
original_response=str(e),
|
||||
additional_args={"headers": outbound_headers},
|
||||
)
|
||||
raise e
|
||||
|
||||
if optional_params.get("stream", False):
|
||||
## LOGGING
|
||||
logging.post_call(
|
||||
input=messages,
|
||||
api_key=api_key,
|
||||
original_response=response,
|
||||
additional_args={"headers": outbound_headers},
|
||||
)
|
||||
|
||||
return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
|
||||
|
||||
|
||||
def complete_text_completion_inception(
|
||||
ctx: CompletionDispatchContext,
|
||||
) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
client: Final = dispatch_client_openai(ctx)
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logger_fn: Final = ctx.logger_fn
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
text_completion: Final = ctx.text_completion
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
passed_api_base: Final = api_base or optional_params.pop("api_base", None) or optional_params.pop("base_url", None)
|
||||
api_base = passed_api_base or get_secret_str("INCEPTION_API_BASE") or "https://api.inceptionlabs.ai/v1"
|
||||
# FIM is served at `/v1/fim/completions`; the OpenAI client appends
|
||||
# `/completions`, so point it at the `/v1/fim` base.
|
||||
api_base = ensure_api_route(api_base, "fim")
|
||||
|
||||
# Don't forward the server-managed Inception key to a caller-supplied
|
||||
# api_base; only resolve it for the default/server base, or when the
|
||||
# caller passes their own key.
|
||||
if passed_api_base is None or api_key:
|
||||
api_key = api_key or litellm.inception_key or get_secret_str("INCEPTION_API_KEY")
|
||||
|
||||
_response = openai_text_completions.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
api_key=api_key,
|
||||
custom_llm_provider="text-completion-inception",
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
client=client,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
if optional_params.get("stream", False) is False and acompletion is False and text_completion is False:
|
||||
_response = litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object(
|
||||
response_object=_response, model_response_object=model_response
|
||||
)
|
||||
|
||||
if optional_params.get("stream", False) or acompletion is True:
|
||||
logging.post_call(
|
||||
input=messages,
|
||||
api_key=api_key,
|
||||
original_response=_response,
|
||||
additional_args={"headers": headers},
|
||||
)
|
||||
return _response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
|
||||
623
litellm/native/chat_providers/platforms.py
Normal file
623
litellm/native/chat_providers/platforms.py
Normal file
|
|
@ -0,0 +1,623 @@
|
|||
from typing import Final
|
||||
|
||||
import litellm
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
from litellm.llms.databricks.common_utils import DatabricksBase
|
||||
from litellm.native.chat_providers.handlers import (
|
||||
base_llm_http_handler,
|
||||
ovhcloud_transformation,
|
||||
sap_gen_ai_hub_chat_completions,
|
||||
watsonx_chat_completion,
|
||||
)
|
||||
from litellm.native.chat_providers.support import (
|
||||
dispatch_client_http,
|
||||
get_encoding,
|
||||
print_verbose,
|
||||
provider_config_items,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult
|
||||
from litellm.utils import get_secret
|
||||
|
||||
|
||||
def complete_heroku(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base: Final = ctx.api_base
|
||||
api_key: Final = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
provider_config: Final = ctx.provider_config
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
try:
|
||||
response: Final = base_llm_http_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
model_response=model_response,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
encoding=get_encoding(),
|
||||
stream=stream,
|
||||
provider_config=provider_config,
|
||||
)
|
||||
except Exception as e:
|
||||
logging.post_call(
|
||||
input=messages,
|
||||
api_key=api_key,
|
||||
original_response=str(e),
|
||||
additional_args={"headers": headers},
|
||||
)
|
||||
raise e
|
||||
|
||||
return response
|
||||
|
||||
|
||||
def complete_gigachat(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base: Final = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
provider_config: Final = ctx.provider_config
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
api_key = (
|
||||
api_key
|
||||
or litellm.api_key
|
||||
or litellm.gigachat_key
|
||||
or get_secret("GIGACHAT_API_KEY")
|
||||
or get_secret("GIGACHAT_CREDENTIALS")
|
||||
)
|
||||
|
||||
headers = headers or litellm.headers or {}
|
||||
|
||||
## COMPLETION CALL
|
||||
try:
|
||||
response: Final = base_llm_http_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
model_response=model_response,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
encoding=get_encoding(),
|
||||
stream=stream,
|
||||
provider_config=provider_config,
|
||||
)
|
||||
except Exception as e:
|
||||
## LOGGING - log the original exception returned
|
||||
logging.post_call(
|
||||
input=messages,
|
||||
api_key=api_key,
|
||||
original_response=str(e),
|
||||
additional_args={"headers": headers},
|
||||
)
|
||||
raise e
|
||||
|
||||
return response
|
||||
|
||||
|
||||
def complete_sap(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base: Final = ctx.api_base
|
||||
api_key: Final = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
headers = headers or litellm.headers
|
||||
## LOAD CONFIG - if set
|
||||
config: Final = litellm.GenAIHubOrchestrationConfig.get_config()
|
||||
for k, v in provider_config_items(config):
|
||||
if (
|
||||
k not in optional_params
|
||||
): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in
|
||||
optional_params[k] = v
|
||||
|
||||
return sap_gen_ai_hub_chat_completions.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
model_response=model_response,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
timeout=timeout,
|
||||
shared_session=shared_session,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
encoding=get_encoding(),
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
stream=stream,
|
||||
)
|
||||
|
||||
|
||||
def complete_oci(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base: Final = ctx.api_base
|
||||
api_key: Final = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
return base_llm_http_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
model_response=model_response,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
encoding=get_encoding(),
|
||||
stream=stream,
|
||||
)
|
||||
|
||||
|
||||
def complete_databricks(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base: Final = DatabricksBase.resolve_api_base(ctx.api_base)
|
||||
api_key: Final = DatabricksBase.resolve_api_key(ctx.api_key)
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
headers = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
headers = headers or litellm.headers
|
||||
|
||||
## COMPLETION CALL
|
||||
try:
|
||||
response: Final = base_llm_http_handler.completion(
|
||||
model=model,
|
||||
stream=stream,
|
||||
messages=messages,
|
||||
acompletion=acompletion,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
custom_llm_provider="databricks",
|
||||
timeout=timeout,
|
||||
headers=headers,
|
||||
encoding=get_encoding(),
|
||||
api_key=api_key,
|
||||
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
|
||||
client=client,
|
||||
)
|
||||
except Exception as e:
|
||||
## LOGGING - log the original exception returned
|
||||
logging.post_call(
|
||||
input=messages,
|
||||
api_key=api_key,
|
||||
original_response=str(e),
|
||||
additional_args={"headers": headers},
|
||||
)
|
||||
raise e
|
||||
|
||||
if optional_params.get("stream", False):
|
||||
## LOGGING
|
||||
logging.post_call(
|
||||
input=messages,
|
||||
api_key=api_key,
|
||||
original_response=response,
|
||||
additional_args={"headers": headers},
|
||||
)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
def complete_databricks_agent(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
from litellm.llms.databricks.agent.transformation import DatabricksAgentConfig
|
||||
|
||||
api_base, api_key = DatabricksAgentConfig.resolve_api_base_and_key(
|
||||
api_base=ctx.api_base or litellm.api_base,
|
||||
api_key=ctx.api_key or litellm.api_key,
|
||||
)
|
||||
return base_llm_http_handler.completion(
|
||||
model=ctx.model,
|
||||
stream=ctx.stream,
|
||||
messages=ctx.messages,
|
||||
acompletion=ctx.acompletion,
|
||||
api_base=api_base,
|
||||
model_response=ctx.model_response,
|
||||
optional_params=ctx.optional_params,
|
||||
litellm_params={**ctx.litellm_params, "api_base": api_base},
|
||||
shared_session=ctx.shared_session,
|
||||
custom_llm_provider=ctx.custom_llm_provider,
|
||||
timeout=ctx.timeout,
|
||||
headers=ctx.headers or litellm.headers,
|
||||
encoding=get_encoding(),
|
||||
api_key=api_key,
|
||||
logging_obj=ctx.logging,
|
||||
client=dispatch_client_http(ctx),
|
||||
)
|
||||
|
||||
|
||||
def complete_datarobot(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base: Final = ctx.api_base
|
||||
api_key: Final = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
provider_config: Final = ctx.provider_config
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
return base_llm_http_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
model_response=model_response,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
encoding=get_encoding(),
|
||||
stream=stream,
|
||||
provider_config=provider_config,
|
||||
)
|
||||
|
||||
|
||||
def complete_watsonx(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base: Final = ctx.api_base
|
||||
api_key: Final = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_prompt_dict: Final = ctx.custom_prompt_dict
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logger_fn: Final = ctx.logger_fn
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
return watsonx_chat_completion.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
timeout=timeout,
|
||||
custom_prompt_dict=custom_prompt_dict,
|
||||
client=client, # pass AsyncOpenAI, OpenAI client
|
||||
encoding=get_encoding(),
|
||||
custom_llm_provider="watsonx",
|
||||
)
|
||||
|
||||
|
||||
def complete_watsonx_text(
|
||||
ctx: CompletionDispatchContext,
|
||||
) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
api_key = (
|
||||
api_key
|
||||
or optional_params.pop("apikey", None)
|
||||
or get_secret_str("WATSONX_APIKEY")
|
||||
or get_secret_str("WATSONX_API_KEY")
|
||||
or get_secret_str("WX_API_KEY")
|
||||
)
|
||||
|
||||
api_base = (
|
||||
api_base
|
||||
or optional_params.pop(
|
||||
"url",
|
||||
optional_params.pop("api_base", optional_params.pop("base_url", None)),
|
||||
)
|
||||
or get_secret_str("WATSONX_API_BASE")
|
||||
or get_secret_str("WATSONX_URL")
|
||||
or get_secret_str("WX_URL")
|
||||
or get_secret_str("WML_URL")
|
||||
)
|
||||
|
||||
wx_credentials: Final = optional_params.pop(
|
||||
"wx_credentials",
|
||||
optional_params.pop("watsonx_credentials", None), # follow {provider}_credentials, same as vertex ai
|
||||
)
|
||||
|
||||
token: str | None = None
|
||||
if wx_credentials is not None:
|
||||
api_base = wx_credentials.get("url", api_base)
|
||||
api_key = wx_credentials.get("apikey", wx_credentials.get("api_key", api_key))
|
||||
token = wx_credentials.get(
|
||||
"token",
|
||||
wx_credentials.get(
|
||||
"watsonx_token", None
|
||||
), # follow format of {provider}_token, same as azure - e.g. 'azure_ad_token=..'
|
||||
)
|
||||
|
||||
if token is not None:
|
||||
optional_params["token"] = token
|
||||
|
||||
return base_llm_http_handler.completion(
|
||||
model=model,
|
||||
stream=stream,
|
||||
messages=messages,
|
||||
acompletion=acompletion,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
custom_llm_provider="watsonx_text",
|
||||
timeout=timeout,
|
||||
headers=headers,
|
||||
encoding=get_encoding(),
|
||||
api_key=api_key,
|
||||
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
|
||||
client=client,
|
||||
)
|
||||
|
||||
|
||||
def complete_cloudflare(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
custom_prompt_dict = ctx.custom_prompt_dict
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
api_key = api_key or litellm.cloudflare_api_key or litellm.api_key or get_secret("CLOUDFLARE_API_KEY")
|
||||
api_base = api_base or litellm.api_base or get_secret("CLOUDFLARE_API_BASE")
|
||||
|
||||
custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict
|
||||
return base_llm_http_handler.completion(
|
||||
model=model,
|
||||
stream=stream,
|
||||
messages=messages,
|
||||
acompletion=acompletion,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
custom_llm_provider="cloudflare",
|
||||
timeout=timeout,
|
||||
headers=headers,
|
||||
encoding=get_encoding(),
|
||||
api_key=api_key,
|
||||
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
|
||||
)
|
||||
|
||||
|
||||
def complete_snowflake(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base: Final = ctx.api_base
|
||||
api_key: Final = ctx.api_key
|
||||
injected_client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
client: Final = (
|
||||
injected_client if injected_client is not None else (HTTPHandler(timeout=timeout) if stream is False else None)
|
||||
) # Keep this here, otherwise, the httpx.client closes and streaming is impossible
|
||||
|
||||
try:
|
||||
response: Final = base_llm_http_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
model_response=model_response,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
encoding=get_encoding(),
|
||||
stream=stream,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
## LOGGING - log the original exception returned
|
||||
logging.post_call(
|
||||
input=messages,
|
||||
api_key=api_key,
|
||||
original_response=str(e),
|
||||
additional_args={"headers": headers},
|
||||
)
|
||||
raise e
|
||||
|
||||
return response
|
||||
|
||||
|
||||
def complete_gradient_ai(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key: Final = ctx.api_key
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
api_base = litellm.api_base or api_base
|
||||
return base_llm_http_handler.completion(
|
||||
model=model,
|
||||
stream=stream,
|
||||
messages=messages,
|
||||
acompletion=acompletion,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
custom_llm_provider="gradient_ai",
|
||||
timeout=timeout,
|
||||
headers=headers,
|
||||
encoding=get_encoding(),
|
||||
api_key=api_key,
|
||||
logging_obj=logging,
|
||||
)
|
||||
|
||||
|
||||
def complete_ovhcloud(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
api_key = api_key or litellm.ovhcloud_key or get_secret_str("OVHCLOUD_API_KEY") or litellm.api_key
|
||||
|
||||
api_base = (
|
||||
api_base
|
||||
or litellm.api_base
|
||||
or get_secret_str("OVHCLOUD_API_BASE")
|
||||
or "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1"
|
||||
)
|
||||
|
||||
response: Final = base_llm_http_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
model_response=model_response,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
encoding=get_encoding(),
|
||||
stream=stream,
|
||||
provider_config=ovhcloud_transformation,
|
||||
)
|
||||
|
||||
return response
|
||||
329
litellm/native/chat_providers/self_hosted.py
Normal file
329
litellm/native/chat_providers/self_hosted.py
Normal file
|
|
@ -0,0 +1,329 @@
|
|||
import os
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.llms.oobabooga.chat import oobabooga
|
||||
from litellm.llms.petals.completion import handler as petals_handler
|
||||
from litellm.llms.vllm.completion import handler as vllm_handler
|
||||
from litellm.native.chat_providers.handlers import base_llm_http_handler, lemonade_transformation
|
||||
from litellm.native.chat_providers.support import dispatch_client_http, get_encoding, print_verbose
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult
|
||||
from litellm.utils import CustomStreamWrapper, get_secret
|
||||
|
||||
_OPAQUE_VALUE_ADAPTER: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
def complete_hosted_vllm(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key: Final = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
provider_config: Final = ctx.provider_config
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
api_base = api_base or litellm.api_base or get_secret_str("HOSTED_VLLM_API_BASE")
|
||||
|
||||
response: Final = base_llm_http_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
api_base=api_base,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
model_response=model_response,
|
||||
encoding=get_encoding(),
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
timeout=timeout,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
acompletion=acompletion,
|
||||
stream=stream,
|
||||
api_key=api_key,
|
||||
headers=headers,
|
||||
client=client,
|
||||
provider_config=provider_config,
|
||||
)
|
||||
logging.post_call(input=messages, api_key=api_key, original_response=response)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
def complete_oobabooga(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
api_base: Final = ctx.api_base
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logger_fn: Final = ctx.logger_fn
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
|
||||
model_response = oobabooga.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
model_response=model_response,
|
||||
api_base=api_base,
|
||||
print_verbose=print_verbose,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
api_key=None,
|
||||
logger_fn=logger_fn,
|
||||
encoding=get_encoding(),
|
||||
logging_obj=logging,
|
||||
)
|
||||
if "stream" in optional_params and optional_params["stream"] is True:
|
||||
# don't try to access stream object,
|
||||
return CustomStreamWrapper(
|
||||
model_response,
|
||||
model,
|
||||
custom_llm_provider="oobabooga",
|
||||
logging_obj=logging,
|
||||
)
|
||||
return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract
|
||||
|
||||
|
||||
def complete_vllm(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
custom_prompt_dict = ctx.custom_prompt_dict
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logger_fn: Final = ctx.logger_fn
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
|
||||
custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict
|
||||
model_response = vllm_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
custom_prompt_dict=custom_prompt_dict,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
encoding=get_encoding(),
|
||||
logging_obj=logging,
|
||||
)
|
||||
|
||||
if "stream" in optional_params and optional_params["stream"] is True: ## [BETA]
|
||||
# don't try to access stream object,
|
||||
return CustomStreamWrapper(
|
||||
model_response,
|
||||
model,
|
||||
custom_llm_provider="vllm",
|
||||
logging_obj=logging,
|
||||
)
|
||||
|
||||
## RESPONSE OBJECT
|
||||
return model_response
|
||||
|
||||
|
||||
def complete_ollama(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key: Final = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
api_base = litellm.api_base or api_base or get_secret("OLLAMA_API_BASE") or "http://localhost:11434"
|
||||
if api_key is not None and "Authorization" not in headers:
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
|
||||
return base_llm_http_handler.completion(
|
||||
model=model,
|
||||
stream=stream,
|
||||
messages=messages,
|
||||
acompletion=acompletion,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
custom_llm_provider="ollama",
|
||||
timeout=timeout,
|
||||
headers=headers,
|
||||
encoding=get_encoding(),
|
||||
api_key=api_key,
|
||||
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
|
||||
client=client,
|
||||
)
|
||||
|
||||
|
||||
def complete_ollama_chat(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
api_base = litellm.api_base or api_base or get_secret("OLLAMA_API_BASE") or "http://localhost:11434"
|
||||
|
||||
api_key = api_key or litellm.ollama_key or os.environ.get("OLLAMA_API_KEY") or litellm.api_key
|
||||
if api_key is not None and "Authorization" not in headers:
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
|
||||
return base_llm_http_handler.completion(
|
||||
model=model,
|
||||
stream=stream,
|
||||
messages=messages,
|
||||
acompletion=acompletion,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
custom_llm_provider="ollama_chat",
|
||||
timeout=timeout,
|
||||
headers=headers,
|
||||
encoding=get_encoding(),
|
||||
api_key=api_key,
|
||||
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
|
||||
client=client,
|
||||
)
|
||||
|
||||
|
||||
def complete_triton(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base = ctx.api_base
|
||||
api_key: Final = ctx.api_key
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
api_base = litellm.api_base or api_base
|
||||
return base_llm_http_handler.completion(
|
||||
model=model,
|
||||
stream=stream,
|
||||
messages=messages,
|
||||
acompletion=acompletion,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
timeout=timeout,
|
||||
headers=headers,
|
||||
encoding=get_encoding(),
|
||||
api_key=api_key,
|
||||
logging_obj=logging,
|
||||
)
|
||||
|
||||
|
||||
def complete_petals(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
api_base = ctx.api_base
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logger_fn: Final = ctx.logger_fn
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
stream = ctx.stream
|
||||
|
||||
api_base = api_base or litellm.api_base
|
||||
|
||||
stream = optional_params.pop("stream", False)
|
||||
model_response = petals_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
encoding=get_encoding(),
|
||||
logging_obj=logging,
|
||||
client=client,
|
||||
)
|
||||
if stream is True: ## [BETA]
|
||||
# Fake streaming for petals
|
||||
resp_string: Final = _OPAQUE_VALUE_ADAPTER.validate_python(model_response["choices"][0]["message"]["content"])
|
||||
return CustomStreamWrapper(
|
||||
resp_string,
|
||||
model,
|
||||
custom_llm_provider="petals",
|
||||
logging_obj=logging,
|
||||
)
|
||||
return model_response
|
||||
|
||||
|
||||
def complete_lemonade(ctx: CompletionDispatchContext) -> CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
api_base: Final = ctx.api_base
|
||||
api_key = ctx.api_key
|
||||
client: Final = dispatch_client_http(ctx)
|
||||
custom_llm_provider: Final = ctx.custom_llm_provider
|
||||
headers: Final = ctx.headers
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
api_key = api_key or litellm.lemonade_key or get_secret_str("LEMONADE_API_KEY") or litellm.api_key
|
||||
|
||||
response: Final = base_llm_http_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
model_response=model_response,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
encoding=get_encoding(),
|
||||
stream=stream,
|
||||
provider_config=lemonade_transformation,
|
||||
)
|
||||
|
||||
return response
|
||||
47
litellm/native/chat_providers/support.py
Normal file
47
litellm/native/chat_providers/support.py
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
from collections.abc import Iterable, Mapping
|
||||
from functools import cache
|
||||
|
||||
import openai
|
||||
|
||||
import litellm
|
||||
from litellm._lazy_imports import get_default_encoding
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.types.completion import CompletionDispatchContext
|
||||
|
||||
|
||||
def dispatch_metadata(ctx: CompletionDispatchContext) -> Mapping[str, object] | None:
|
||||
return ctx.metadata
|
||||
|
||||
|
||||
def dispatch_client_http(ctx: CompletionDispatchContext) -> HTTPHandler | AsyncHTTPHandler | None:
|
||||
return ctx.client
|
||||
|
||||
|
||||
def dispatch_client_azure(
|
||||
ctx: CompletionDispatchContext,
|
||||
) -> openai.AzureOpenAI | openai.AsyncAzureOpenAI | HTTPHandler | AsyncHTTPHandler | None:
|
||||
return ctx.client
|
||||
|
||||
|
||||
def dispatch_client_openai(ctx: CompletionDispatchContext) -> openai.OpenAI | openai.AsyncOpenAI | None:
|
||||
return ctx.client
|
||||
|
||||
|
||||
def provider_config_items(config: Mapping[str, object]) -> Iterable[tuple[str, object]]:
|
||||
return config.items()
|
||||
|
||||
|
||||
def print_verbose(print_statement: object) -> None:
|
||||
try:
|
||||
verbose_logger.debug(print_statement)
|
||||
if litellm.set_verbose:
|
||||
print(print_statement) # noqa: T201
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@cache
|
||||
def get_encoding() -> Tokenizer:
|
||||
return get_default_encoding()
|
||||
120
litellm/native/messages.py
Normal file
120
litellm/native/messages.py
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
"""Sends an Anthropic Messages request to a provider that speaks Messages natively."""
|
||||
|
||||
from collections.abc import AsyncIterator, Coroutine, Mapping
|
||||
from typing import Final
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.anthropic.messages.native.utils import AnthropicMessagesRequestUtils
|
||||
from litellm.llms.base_llm.anthropic_messages.transformation import BaseAnthropicMessagesConfig
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from litellm.router_utils.reasoning_effort_capability import (
|
||||
RequestedReasoningEffort,
|
||||
deployment_reasoning_effort_levels,
|
||||
keeps_effort,
|
||||
model_group_from_metadata,
|
||||
resolve_deployment_reasoning_effort,
|
||||
with_adaptive_thinking_as_effort,
|
||||
)
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
__all__ = ("execute_messages",)
|
||||
|
||||
|
||||
def _with_output_config(params: Mapping[str, object], output_config: RequestedReasoningEffort) -> Mapping[str, object]:
|
||||
rest: Final = {key: value for key, value in params.items() if key != "output_config"}
|
||||
return rest if output_config is None else {**rest, "output_config": output_config}
|
||||
|
||||
|
||||
def _with_deployment_output_effort(
|
||||
params: Mapping[str, object],
|
||||
*,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
kwargs: Mapping[str, object],
|
||||
drop_params: bool,
|
||||
) -> Mapping[str, object]:
|
||||
deployment_model_info: Final = kwargs.get("model_info")
|
||||
|
||||
def resolve(requested: RequestedReasoningEffort) -> RequestedReasoningEffort:
|
||||
return resolve_deployment_reasoning_effort(
|
||||
requested,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
deployment_model_info=deployment_model_info,
|
||||
model_group=model_group_from_metadata(kwargs.get("litellm_metadata"), kwargs.get("metadata")),
|
||||
drop_params=drop_params,
|
||||
).value
|
||||
|
||||
def translate(source: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return with_adaptive_thinking_as_effort(
|
||||
source, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params
|
||||
)
|
||||
|
||||
requested_output: Final = as_str_mapping(params.get("output_config"))
|
||||
caller_output: Final = None if requested_output is None else resolve(requested_output)
|
||||
caller_params: Final = params if requested_output is None else _with_output_config(params, caller_output)
|
||||
if deployment_reasoning_effort_levels(deployment_model_info) is None:
|
||||
return caller_params
|
||||
if requested_output is not None and keeps_effort(requested_output, caller_output):
|
||||
return _with_output_config(translate(params), caller_output)
|
||||
translated: Final = translate(caller_params)
|
||||
budget_output: Final = as_str_mapping(translated.get("output_config"))
|
||||
return translated if budget_output is None else _with_output_config(translated, resolve(budget_output))
|
||||
|
||||
|
||||
def execute_messages(
|
||||
*,
|
||||
http_handler: BaseLLMHTTPHandler,
|
||||
model: str,
|
||||
messages: list[dict[str, object]], # mutable-ok: the HTTP handler takes list[dict] messages
|
||||
provider_config: BaseAnthropicMessagesConfig,
|
||||
request_params: Mapping[str, object],
|
||||
custom_llm_provider: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
api_key: str | None,
|
||||
api_base: str | None,
|
||||
dynamic_api_base: str | None,
|
||||
client: AsyncHTTPHandler | None,
|
||||
is_async: bool,
|
||||
stream: bool | None,
|
||||
request_kwargs: dict[str, object], # mutable-ok: forwarded as the HTTP handler's kwargs dict
|
||||
) -> AnthropicMessagesResponse | Coroutine[object, object, AnthropicMessagesResponse | AsyncIterator[bytes]]:
|
||||
optional_params: Final = AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param(
|
||||
params=request_params,
|
||||
model=model,
|
||||
drop_params=litellm_params.get("drop_params") is True,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
resolved_api_base: Final = (
|
||||
dynamic_api_base
|
||||
if dynamic_api_base is not None and provider_config.uses_get_llm_provider_api_base()
|
||||
else api_base
|
||||
)
|
||||
return http_handler.anthropic_messages_handler(
|
||||
model=model,
|
||||
messages=messages,
|
||||
anthropic_messages_provider_config=provider_config,
|
||||
anthropic_messages_optional_request_params=dict(
|
||||
_with_deployment_output_effort(
|
||||
optional_params,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
kwargs=request_kwargs,
|
||||
drop_params=litellm.drop_params is True or litellm_params.get("drop_params") is True,
|
||||
)
|
||||
),
|
||||
_is_async=is_async,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params,
|
||||
logging_obj=logging_obj,
|
||||
api_key=api_key,
|
||||
api_base=resolved_api_base,
|
||||
stream=stream,
|
||||
kwargs=request_kwargs,
|
||||
)
|
||||
149
litellm/native/request_support.py
Normal file
149
litellm/native/request_support.py
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
from collections.abc import Callable, Mapping
|
||||
from typing import Final, Protocol
|
||||
|
||||
from typing_extensions import assert_never
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
|
||||
from litellm.litellm_core_utils.get_litellm_params import InvalidControlOption, parse_control_options
|
||||
from litellm.responses.bridge_check import keep_registered_mode
|
||||
from litellm.types.litellm_params import ControlOptions
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.utils import CustomPricingLiteLLMParams
|
||||
|
||||
|
||||
def resolve_control_options(kwargs: Mapping[str, object], model: str) -> ControlOptions:
|
||||
control: Final = parse_control_options(kwargs)
|
||||
match control:
|
||||
case ControlOptions():
|
||||
return control
|
||||
case InvalidControlOption(param=param, message=message):
|
||||
if litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True:
|
||||
return ControlOptions()
|
||||
raise litellm.BadRequestError(message=message, model=model, llm_provider=None, body={"param": param})
|
||||
case _:
|
||||
return assert_never(control)
|
||||
|
||||
|
||||
class _AdmissionReservation(LiteLLMBaseModel):
|
||||
input_tokens: int | None = None
|
||||
|
||||
|
||||
class _AdmissionMetadata(LiteLLMBaseModel):
|
||||
user_api_key_budget_reservation: _AdmissionReservation | None = None
|
||||
|
||||
|
||||
def admission_input_tokens(kwargs: Mapping[str, object]) -> int | None:
|
||||
reservations: Final = (
|
||||
_AdmissionMetadata.model_validate(kwargs.get(key) or {}).user_api_key_budget_reservation
|
||||
for key in ("litellm_metadata", "metadata")
|
||||
)
|
||||
return next(
|
||||
(
|
||||
reservation.input_tokens
|
||||
for reservation in reservations
|
||||
if reservation and reservation.input_tokens is not None
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def admitted_prompt_token_counter(prompt_tokens: int | None) -> Callable[[], int] | None:
|
||||
if prompt_tokens is None:
|
||||
return None
|
||||
return lambda: prompt_tokens
|
||||
|
||||
|
||||
class _ProxyAuthHeadersProvider(Protocol):
|
||||
def get_auth_headers(self) -> Mapping[str, str]: ...
|
||||
|
||||
|
||||
def proxy_auth_headers(proxy_auth: _ProxyAuthHeadersProvider) -> Mapping[str, str]:
|
||||
return proxy_auth.get_auth_headers()
|
||||
|
||||
|
||||
def _build_custom_pricing_entry(
|
||||
custom_llm_provider: str,
|
||||
kwargs: dict,
|
||||
model_info: dict | None = None,
|
||||
) -> dict:
|
||||
"""Build a complete model cost entry from kwargs and model_info.
|
||||
|
||||
Collects all CustomPricingLiteLLMParams fields present in kwargs and
|
||||
merges metadata from model_info (mode, supports_prompt_caching, max_tokens)
|
||||
so that register_model() receives the full pricing configuration.
|
||||
"""
|
||||
entry: Final[dict] = {"litellm_provider": custom_llm_provider}
|
||||
|
||||
for field_name in CustomPricingLiteLLMParams.model_fields:
|
||||
value = kwargs.get(field_name)
|
||||
if value is not None:
|
||||
entry[field_name] = value
|
||||
|
||||
if model_info and isinstance(model_info, dict):
|
||||
for key in ("mode", "supports_prompt_caching", "max_tokens"):
|
||||
if key in model_info and model_info[key] is not None:
|
||||
entry.setdefault(key, model_info[key])
|
||||
|
||||
return entry
|
||||
|
||||
|
||||
def _get_router_deployment_id(kwargs: dict) -> str | None:
|
||||
for metadata_key in ("litellm_metadata", "metadata"):
|
||||
metadata = kwargs.get(metadata_key) or {}
|
||||
if not isinstance(metadata, dict):
|
||||
continue
|
||||
deployment_model_info = metadata.get("model_info") or {}
|
||||
if not isinstance(deployment_model_info, dict):
|
||||
continue
|
||||
deployment_id = deployment_model_info.get("id")
|
||||
if deployment_id is not None:
|
||||
return str(deployment_id)
|
||||
return None
|
||||
|
||||
|
||||
def register_custom_pricing_for_request(
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
kwargs: dict,
|
||||
model_info: dict | None,
|
||||
) -> None:
|
||||
"""Register per-request custom pricing in litellm.model_cost.
|
||||
|
||||
Router-originated requests (identified by the deployment id the router puts
|
||||
in metadata) get their full pricing registered under that unique id only;
|
||||
the shared ``{provider}/{model}`` key receives the entry with pricing fields
|
||||
stripped, mirroring Router._create_deployment. This keeps one deployment's
|
||||
pricing overrides (e.g. a zero-cost wildcard) from clobbering built-in
|
||||
pricing used by sibling deployments of the same backend model. Direct SDK
|
||||
calls keep the legacy behavior of registering the shared key with pricing.
|
||||
Either way the shared key's ``mode`` follows ``keep_registered_mode``, as on
|
||||
deployment registration, since a call routes by its own ``model_info``.
|
||||
"""
|
||||
entry: Final = _build_custom_pricing_entry(
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
kwargs=kwargs,
|
||||
model_info=model_info,
|
||||
)
|
||||
shared_key: Final = f"{custom_llm_provider}/{model}"
|
||||
deployment_id: Final = _get_router_deployment_id(kwargs)
|
||||
if deployment_id is None:
|
||||
litellm.register_model({shared_key: keep_registered_mode(shared_key, entry)}, persist_across_reloads=False)
|
||||
return
|
||||
shared_entry: Final = keep_registered_mode(
|
||||
shared_key, CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry)
|
||||
)
|
||||
litellm.register_model(
|
||||
{deployment_id: entry},
|
||||
persist_across_reloads=False,
|
||||
warning_display_name=shared_key,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
add_to_provider_model_sets=False,
|
||||
)
|
||||
litellm.register_model(
|
||||
{shared_key: shared_entry},
|
||||
persist_across_reloads=False,
|
||||
warning_display_name=shared_key,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
add_to_provider_model_sets=False,
|
||||
)
|
||||
185
litellm/native/responses.py
Normal file
185
litellm/native/responses.py
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
"""Sends a Responses API request to a provider that speaks Responses natively."""
|
||||
|
||||
from collections.abc import Collection, Coroutine, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Annotated, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
|
||||
|
||||
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind
|
||||
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from litellm.llms.openai.data_residency import infer_openai_data_residency
|
||||
from litellm.responses.litellm_completion_transformation.compaction_items import (
|
||||
with_compaction_items_as_summaries,
|
||||
)
|
||||
from litellm.responses.litellm_completion_transformation.transformation import (
|
||||
with_compaction_items_as_params,
|
||||
)
|
||||
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import CostPerToken
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from aiohttp import ClientSession
|
||||
|
||||
__all__ = ("execute_responses",)
|
||||
|
||||
|
||||
class _DeploymentModelInfo(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
id: object = None
|
||||
|
||||
|
||||
_DEPLOYMENT_MODEL_INFO: Final[TypeAdapter[_DeploymentModelInfo | object]] = TypeAdapter(
|
||||
Annotated[_DeploymentModelInfo | object, Field(union_mode="left_to_right")]
|
||||
)
|
||||
|
||||
|
||||
def _is_router_deployment(deployment_model_info: object) -> bool:
|
||||
"""A Router stamps the deployment's model_info (with its id) onto every call it routes."""
|
||||
parsed: Final = _DEPLOYMENT_MODEL_INFO.validate_python(deployment_model_info)
|
||||
return isinstance(parsed, _DeploymentModelInfo) and parsed.id is not None
|
||||
|
||||
|
||||
_COST_PER_TOKEN: Final = TypeAdapter(CostPerToken)
|
||||
|
||||
|
||||
def _per_call_cost_per_token(litellm_params: GenericLiteLLMParams) -> CostPerToken | None:
|
||||
if litellm_params.input_cost_per_token is None or litellm_params.output_cost_per_token is None:
|
||||
return None
|
||||
return _COST_PER_TOKEN.validate_python(
|
||||
litellm_params.model_dump(
|
||||
include={
|
||||
"input_cost_per_token",
|
||||
"output_cost_per_token",
|
||||
"cache_read_input_token_cost",
|
||||
"cache_creation_input_token_cost",
|
||||
},
|
||||
exclude_none=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def execute_responses(
|
||||
*,
|
||||
http_handler: BaseLLMHTTPHandler,
|
||||
model: str,
|
||||
input: str | ResponseInputParam,
|
||||
provider_config: BaseResponsesAPIConfig,
|
||||
requested_params: ResponsesAPIOptionalRequestParams,
|
||||
allowed_openai_params: Sequence[str] | None,
|
||||
declared_reasoning_efforts: Collection[str],
|
||||
deployment_model_info: object,
|
||||
custom_llm_provider: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
user: str | None,
|
||||
extra_headers: dict[str, object] | None,
|
||||
extra_body: Mapping[str, object] | None,
|
||||
timeout: float | httpx.Timeout | None,
|
||||
is_async: bool,
|
||||
stream: bool | None,
|
||||
client: HTTPHandler | AsyncHTTPHandler | None,
|
||||
shared_session: "ClientSession | None",
|
||||
litellm_metadata: dict[str, object] | None,
|
||||
request_kwargs: dict[str, object],
|
||||
) -> (
|
||||
ResponsesAPIResponse
|
||||
| BaseResponsesAPIStreamingIterator
|
||||
| Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator]
|
||||
):
|
||||
request_params: Final = ResponsesAPIRequestUtils.get_optional_params_responses_api(
|
||||
model=model,
|
||||
responses_api_provider_config=provider_config,
|
||||
response_api_optional_params=requested_params,
|
||||
allowed_openai_params=allowed_openai_params,
|
||||
drop_params=normalize_drop_params(request_kwargs.get("drop_params")),
|
||||
declared_reasoning_efforts=declared_reasoning_efforts,
|
||||
)
|
||||
|
||||
logging_obj.update_from_kwargs(
|
||||
kwargs=request_kwargs,
|
||||
model=model,
|
||||
user=user,
|
||||
optional_params=dict(request_params),
|
||||
litellm_params={
|
||||
**request_params,
|
||||
**litellm_params.custom_pricing_overrides(),
|
||||
"aresponses": is_async,
|
||||
"litellm_call_id": request_kwargs.get("litellm_call_id"),
|
||||
"model_info": request_kwargs.get("model_info"),
|
||||
"data_residency": infer_openai_data_residency(custom_llm_provider, litellm_params.api_base),
|
||||
"metadata": (
|
||||
request_kwargs["litellm_metadata"]
|
||||
if "litellm_metadata" in request_kwargs
|
||||
else request_kwargs.get("metadata")
|
||||
),
|
||||
"provider_affinity_header": litellm_params.provider_affinity_header,
|
||||
},
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
if not _is_router_deployment(deployment_model_info):
|
||||
logging_obj.custom_cost_per_token = _per_call_cost_per_token(litellm_params) # rebind-ok: prices the call
|
||||
logging_obj.custom_cost_per_second = litellm_params.cost_per_second # rebind-ok: prices the call
|
||||
|
||||
fake_stream: Final = provider_config.should_fake_stream(
|
||||
model=model, stream=stream, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
applied_timeout: Final = CompletionTimeout.for_stream(
|
||||
(StreamKind.FAKED if fake_stream else StreamKind.LIVE) if stream is True else StreamKind.NOT_STREAMED,
|
||||
CompletionTimeout.as_limit(timeout),
|
||||
None if stream is not True or litellm_params.stream_timeout is None else float(litellm_params.stream_timeout),
|
||||
)
|
||||
logging_obj.record_applied_timeout(applied_timeout)
|
||||
response: Final = http_handler.response_api_handler(
|
||||
model=model,
|
||||
input=with_compaction_items_as_summaries(
|
||||
with_compaction_items_as_params(ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(input))
|
||||
),
|
||||
responses_api_provider_config=provider_config,
|
||||
response_api_optional_request_params=request_params,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params,
|
||||
logging_obj=logging_obj,
|
||||
extra_headers=extra_headers,
|
||||
extra_body=extra_body,
|
||||
timeout=applied_timeout,
|
||||
_is_async=is_async,
|
||||
client=client,
|
||||
fake_stream=fake_stream,
|
||||
litellm_metadata=litellm_metadata,
|
||||
shared_session=shared_session,
|
||||
)
|
||||
if isinstance(response, Coroutine):
|
||||
return _stamped_when_done(response, litellm_metadata, custom_llm_provider)
|
||||
return _stamped(response, litellm_metadata, custom_llm_provider)
|
||||
|
||||
|
||||
def _stamped(
|
||||
response: ResponsesAPIResponse | BaseResponsesAPIStreamingIterator,
|
||||
litellm_metadata: Mapping[str, object] | None,
|
||||
custom_llm_provider: str,
|
||||
) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator:
|
||||
if not isinstance(response, ResponsesAPIResponse):
|
||||
return response
|
||||
stamped: Final = ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id(
|
||||
responses_api_response=response,
|
||||
litellm_metadata=litellm_metadata,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
stamped.hidden_params["custom_llm_provider"] = custom_llm_provider
|
||||
return stamped
|
||||
|
||||
|
||||
async def _stamped_when_done(
|
||||
response: Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator],
|
||||
litellm_metadata: Mapping[str, object] | None,
|
||||
custom_llm_provider: str,
|
||||
) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator:
|
||||
return _stamped(await response, litellm_metadata, custom_llm_provider)
|
||||
165
litellm/native/route_plan.py
Normal file
165
litellm/native/route_plan.py
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
"""Which native API format a request reaches its provider in, decided once per request by the client-facing surface."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
from litellm.llms.base_llm.anthropic_messages.transformation import BaseAnthropicMessagesConfig
|
||||
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
__all__ = (
|
||||
"NativeChat",
|
||||
"NativeMessages",
|
||||
"NativeResponses",
|
||||
"RoutePlan",
|
||||
"native_messages_config",
|
||||
"plan_chat_route",
|
||||
"plan_messages_route",
|
||||
"plan_responses_route",
|
||||
)
|
||||
|
||||
_MESSAGES_TO_RESPONSES_PROVIDERS: Final = frozenset({"openai"})
|
||||
_PROVIDER_VALUES: Final = frozenset(provider.value for provider in LlmProviders)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class NativeMessages:
|
||||
config: BaseAnthropicMessagesConfig
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class NativeResponses:
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class NativeChat:
|
||||
pass
|
||||
|
||||
|
||||
RoutePlan: TypeAlias = NativeMessages | NativeResponses | NativeChat
|
||||
|
||||
|
||||
def _deployment_passes_through_anthropic_messages(model_info: object) -> bool:
|
||||
"""Whether the deployment opted into forwarding /v1/messages untranslated.
|
||||
|
||||
The opt-in is ``model_info.supported_endpoints`` containing ``"/v1/messages"``,
|
||||
declared per deployment in config.yaml and plumbed here as ``kwargs["model_info"]``
|
||||
by the router.
|
||||
"""
|
||||
if not isinstance(model_info, dict):
|
||||
return False
|
||||
supported_endpoints: Final = model_info.get("supported_endpoints")
|
||||
return isinstance(supported_endpoints, (list, tuple)) and "/v1/messages" in supported_endpoints
|
||||
|
||||
|
||||
def _deployment_supports_cache_control_ttl(model_info: object) -> bool:
|
||||
return isinstance(model_info, dict) and model_info.get("cache_control_ttl") is True
|
||||
|
||||
|
||||
def native_messages_config(
|
||||
model: str, custom_llm_provider: str | None, model_info: object, api_base: str | None
|
||||
) -> BaseAnthropicMessagesConfig | None:
|
||||
provider_config: Final = (
|
||||
ProviderConfigManager.get_provider_anthropic_messages_config(
|
||||
model=model,
|
||||
provider=LlmProviders(custom_llm_provider),
|
||||
)
|
||||
if custom_llm_provider is not None and custom_llm_provider in _PROVIDER_VALUES
|
||||
else None
|
||||
)
|
||||
native_config: Final = (
|
||||
provider_config if provider_config is not None and provider_config.serves_api_base(model, api_base) else None
|
||||
)
|
||||
if native_config is not None or not _deployment_passes_through_anthropic_messages(model_info):
|
||||
return native_config
|
||||
from litellm.llms.openai_like.messages.transformation import (
|
||||
OpenAILikeAnthropicMessagesConfig,
|
||||
)
|
||||
|
||||
return OpenAILikeAnthropicMessagesConfig(cache_control_ttl=_deployment_supports_cache_control_ttl(model_info))
|
||||
|
||||
|
||||
def _bridges_to_responses_api(model: str, custom_llm_provider: str, deployment_model_info: object) -> bool:
|
||||
from litellm.responses.bridge_check import responses_api_bridge_check, str_keyed_mapping
|
||||
|
||||
model_info, _ = responses_api_bridge_check(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
deployment_model_info=str_keyed_mapping(deployment_model_info),
|
||||
)
|
||||
return model_info.get("mode") == "responses"
|
||||
|
||||
|
||||
def _responses_mode_is_lost_by_prefix_strip(
|
||||
requested_model: str, resolved_model: str, custom_llm_provider: str, deployment_model_info: object
|
||||
) -> bool:
|
||||
"""Whether a Responses-only deployment stops looking like one once its provider prefix is stripped.
|
||||
|
||||
``litellm.completion`` re-derives the Responses bridge from the stripped id alone, so a
|
||||
deployment id such as ``perplexity/perplexity/sonar`` (mode ``responses``) is shadowed by the
|
||||
chat entry ``perplexity/sonar`` and would otherwise be sent to chat/completions.
|
||||
"""
|
||||
if requested_model == resolved_model:
|
||||
return False
|
||||
return _bridges_to_responses_api(
|
||||
requested_model, custom_llm_provider, deployment_model_info
|
||||
) and not _bridges_to_responses_api(resolved_model, custom_llm_provider, deployment_model_info)
|
||||
|
||||
|
||||
def _messages_bridge_to_responses(
|
||||
custom_llm_provider: str | None,
|
||||
requested_model: str,
|
||||
resolved_model: str,
|
||||
deployment_model_info: object,
|
||||
force_chat_completions: bool,
|
||||
stop_requested: bool,
|
||||
) -> bool:
|
||||
if force_chat_completions or stop_requested:
|
||||
return False
|
||||
if custom_llm_provider in _MESSAGES_TO_RESPONSES_PROVIDERS:
|
||||
return True
|
||||
if custom_llm_provider is None:
|
||||
return False
|
||||
return _responses_mode_is_lost_by_prefix_strip(
|
||||
requested_model, resolved_model, custom_llm_provider, deployment_model_info
|
||||
)
|
||||
|
||||
|
||||
def plan_messages_route(
|
||||
*,
|
||||
requested_model: str,
|
||||
resolved_model: str,
|
||||
custom_llm_provider: str | None,
|
||||
model_info: object,
|
||||
api_base: str | None,
|
||||
compaction_requested: bool,
|
||||
force_chat_completions: bool,
|
||||
stop_requested: bool = False,
|
||||
) -> RoutePlan:
|
||||
"""``force_chat_completions`` is ``litellm.use_chat_completions_url_for_anthropic_messages``.
|
||||
|
||||
A request with ``stop_sequences`` stays on chat completions.
|
||||
"""
|
||||
config: Final = native_messages_config(resolved_model, custom_llm_provider, model_info, api_base)
|
||||
if config is not None:
|
||||
return NativeMessages(config)
|
||||
if not compaction_requested and _messages_bridge_to_responses(
|
||||
custom_llm_provider, requested_model, resolved_model, model_info, force_chat_completions, stop_requested
|
||||
):
|
||||
return NativeResponses()
|
||||
return NativeChat()
|
||||
|
||||
|
||||
def plan_responses_route(
|
||||
responses_api_provider_config: BaseResponsesAPIConfig | None, use_chat_completions_api: bool
|
||||
) -> NativeResponses | NativeChat:
|
||||
if responses_api_provider_config is None or use_chat_completions_api:
|
||||
return NativeChat()
|
||||
return NativeResponses()
|
||||
|
||||
|
||||
def plan_chat_route(responses_api_provider_config: BaseResponsesAPIConfig | None) -> NativeChat | NativeResponses:
|
||||
"""For a deployment ``responses_api_bridge_check`` resolved to mode ``responses``."""
|
||||
return NativeChat() if responses_api_provider_config is None else NativeResponses()
|
||||
|
|
@ -48,7 +48,7 @@ def s3_object_reader(bucket_name: str) -> SyncBucketObjectReader:
|
|||
import boto3
|
||||
from botocore.credentials import Credentials
|
||||
|
||||
from litellm.main import bedrock_converse_chat_completion
|
||||
from litellm.native.chat_providers.handlers import bedrock_converse_chat_completion
|
||||
|
||||
credentials: Final[Credentials] = bedrock_converse_chat_completion.get_credentials()
|
||||
s3_client: Final = boto3.client(
|
||||
|
|
|
|||
|
|
@ -69,9 +69,9 @@ def _is_openai_backed_api_base(api_base: str) -> bool:
|
|||
return hostname is not None and (hostname == _OPENAI_API_HOST or hostname.endswith(f".{_OPENAI_API_HOST}"))
|
||||
|
||||
|
||||
def _resolve_openai_api_base(api_base: str | None) -> str:
|
||||
def resolve_openai_api_base(api_base: str | None) -> str:
|
||||
"""Effective OpenAI base a chat request will hit: arg > global > env > default. The bridge gate
|
||||
and the ``_complete_custom_openai`` chat handler MUST resolve this identically, or a custom base
|
||||
and the ``complete_custom_openai`` chat handler MUST resolve this identically, or a custom base
|
||||
set via ``litellm.api_base`` or ``OPENAI_BASE_URL``/``OPENAI_API_BASE`` is invisible to the gate,
|
||||
which then misreads it as the default OpenAI endpoint and bridges a request the backend can't serve."""
|
||||
return (
|
||||
|
|
@ -243,7 +243,7 @@ def responses_api_bridge_check(
|
|||
# exactly as the chat handler does, so a custom base set via litellm.api_base or
|
||||
# OPENAI_BASE_URL/OPENAI_API_BASE isn't misread as the default and bridged to a /responses route it
|
||||
# lacks. A whitespace-only base collapses to the default too.
|
||||
resolved_api_base: Final = _resolve_openai_api_base(api_base).strip()
|
||||
resolved_api_base: Final = resolve_openai_api_base(api_base).strip()
|
||||
on_foundry_openai_endpoint: Final = custom_llm_provider == "azure_ai" and azure_ai_supports_native_responses(
|
||||
model, api_base
|
||||
)
|
||||
|
|
|
|||
259
litellm/responses/bridge_executor.py
Normal file
259
litellm/responses/bridge_executor.py
Normal file
|
|
@ -0,0 +1,259 @@
|
|||
"""Hands a bridged Responses request to the native Responses executor, so the bridge never re-enters `aresponses`."""
|
||||
|
||||
from collections.abc import Coroutine, Mapping, Sequence
|
||||
from typing import Final, NoReturn
|
||||
|
||||
import httpx
|
||||
from aiohttp import ClientSession
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.constants import request_timeout
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.native.responses import execute_responses
|
||||
from litellm.responses.main import (
|
||||
apply_managed_file_id_mapping,
|
||||
base_llm_http_handler,
|
||||
reasoning_for_deployment,
|
||||
requested_reasoning,
|
||||
resolve_model_provider_for_responses,
|
||||
resolve_responses_api_provider_config,
|
||||
with_provider_affinity_header,
|
||||
with_provider_specific_headers,
|
||||
)
|
||||
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.router_utils.reasoning_effort_capability import deployment_reasoning_effort_levels
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import AgenticLoopParams
|
||||
|
||||
__all__ = ("anative_responses", "has_native_responses_config", "native_logging_obj", "native_responses")
|
||||
|
||||
_STRING_KEYED_DICT: Final = TypeAdapter(dict[str, object])
|
||||
_CLIENT_HEADERS: Final = TypeAdapter(dict[str, str])
|
||||
_STRINGS: Final = TypeAdapter(tuple[str, ...])
|
||||
|
||||
|
||||
def _optional_str(value: object) -> str | None:
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
def _string_keyed_dict(value: object) -> dict[str, object] | None:
|
||||
try:
|
||||
return _STRING_KEYED_DICT.validate_python(value) if value is not None else None
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _client_headers(value: object) -> Mapping[str, str] | None:
|
||||
try:
|
||||
return _CLIENT_HEADERS.validate_python(value) if isinstance(value, dict) else None
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _shared_session(value: object) -> ClientSession | None:
|
||||
return value if isinstance(value, ClientSession) else None
|
||||
|
||||
|
||||
def _http_client(value: object) -> HTTPHandler | AsyncHTTPHandler | None:
|
||||
return value if isinstance(value, (HTTPHandler, AsyncHTTPHandler)) else None
|
||||
|
||||
|
||||
def _timeout(value: object) -> float | httpx.Timeout:
|
||||
return value if isinstance(value, (float, int, httpx.Timeout)) and value else request_timeout
|
||||
|
||||
|
||||
def _allowed_openai_params(value: object) -> Sequence[str] | None:
|
||||
try:
|
||||
return _STRINGS.validate_python(value) if value is not None else None
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _responses_target(
|
||||
model: str, forwarded_kwargs: Mapping[str, object], litellm_params: GenericLiteLLMParams
|
||||
) -> tuple[str, str, BaseResponsesAPIConfig] | None:
|
||||
provider_model, custom_llm_provider = resolve_model_provider_for_responses(
|
||||
model=model,
|
||||
custom_llm_provider=_optional_str(forwarded_kwargs.get("custom_llm_provider")),
|
||||
litellm_params=litellm_params,
|
||||
local_vars={},
|
||||
)
|
||||
if custom_llm_provider is None:
|
||||
return None
|
||||
provider_config: Final = resolve_responses_api_provider_config(
|
||||
provider_model, custom_llm_provider, forwarded_kwargs.get("model_info"), litellm_params.api_base
|
||||
)
|
||||
return None if provider_config is None else (provider_model, custom_llm_provider, provider_config)
|
||||
|
||||
|
||||
def has_native_responses_config(model: str, forwarded_kwargs: Mapping[str, object]) -> bool:
|
||||
return _responses_target(model, forwarded_kwargs, GenericLiteLLMParams(**forwarded_kwargs)) is not None
|
||||
|
||||
|
||||
def _native_responses_call(
|
||||
request: Mapping[str, object],
|
||||
model: str,
|
||||
forwarded_kwargs: Mapping[str, object],
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
is_async: bool,
|
||||
) -> (
|
||||
ResponsesAPIResponse
|
||||
| BaseResponsesAPIStreamingIterator
|
||||
| Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator]
|
||||
):
|
||||
"""Resolve the deployment and hand the translated request to the native Responses executor."""
|
||||
logging_obj.record_agentic_loop_params(AgenticLoopParams(model=model))
|
||||
litellm_params: Final = GenericLiteLLMParams(**forwarded_kwargs)
|
||||
target: Final = _responses_target(model, forwarded_kwargs, litellm_params)
|
||||
if target is None:
|
||||
raise ValueError(f"No Responses API config for {model}")
|
||||
provider_model, custom_llm_provider, provider_config = target
|
||||
request_input, request_tools = apply_managed_file_id_mapping(
|
||||
input=request["input"],
|
||||
tools=request.get("tools"),
|
||||
kwargs=dict(forwarded_kwargs),
|
||||
local_vars={},
|
||||
)
|
||||
params: Final = {
|
||||
**forwarded_kwargs,
|
||||
**request,
|
||||
"input": request_input,
|
||||
"tools": request_tools,
|
||||
"custom_llm_provider": custom_llm_provider,
|
||||
}
|
||||
deployment_model_info: Final = forwarded_kwargs.get("model_info")
|
||||
reasoning: Final = reasoning_for_deployment(
|
||||
requested_reasoning(params, provider_model, custom_llm_provider),
|
||||
provider_model,
|
||||
custom_llm_provider,
|
||||
deployment_model_info,
|
||||
forwarded_kwargs,
|
||||
)
|
||||
return execute_responses(
|
||||
http_handler=base_llm_http_handler,
|
||||
model=provider_model,
|
||||
input=request_input,
|
||||
provider_config=provider_config,
|
||||
requested_params=ResponsesAPIRequestUtils.get_requested_response_api_optional_param(
|
||||
{key: value for key, value in {**params, "reasoning": reasoning}.items() if key != "reasoning_effort"}
|
||||
),
|
||||
allowed_openai_params=_allowed_openai_params(forwarded_kwargs.get("allowed_openai_params")),
|
||||
declared_reasoning_efforts=deployment_reasoning_effort_levels(deployment_model_info) or (),
|
||||
deployment_model_info=deployment_model_info,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params,
|
||||
logging_obj=logging_obj,
|
||||
user=_optional_str(request.get("user")),
|
||||
extra_headers=with_provider_affinity_header(
|
||||
with_provider_specific_headers(
|
||||
ResponsesAPIRequestUtils.merge_client_forwarded_headers(
|
||||
extra_headers=_string_keyed_dict(forwarded_kwargs.get("extra_headers")),
|
||||
client_headers=_client_headers(forwarded_kwargs.get("headers")),
|
||||
),
|
||||
litellm_params,
|
||||
custom_llm_provider,
|
||||
),
|
||||
litellm_params,
|
||||
_string_keyed_dict(request.get("metadata")),
|
||||
forwarded_kwargs,
|
||||
provider_model,
|
||||
custom_llm_provider,
|
||||
),
|
||||
extra_body=_string_keyed_dict(forwarded_kwargs.get("extra_body")),
|
||||
timeout=_timeout(forwarded_kwargs.get("timeout")),
|
||||
is_async=is_async,
|
||||
stream=request.get("stream") is True,
|
||||
client=_http_client(forwarded_kwargs.get("client")),
|
||||
shared_session=_shared_session(forwarded_kwargs.get("shared_session")),
|
||||
litellm_metadata=_string_keyed_dict(forwarded_kwargs.get("litellm_metadata")) or {},
|
||||
request_kwargs=dict(forwarded_kwargs),
|
||||
)
|
||||
|
||||
|
||||
_OPENAI_CHAT_COMPLETIONS_MODEL_PREFIX: Final = "openai/chat_completions/"
|
||||
|
||||
|
||||
def _requests_chat_completions_api(params: Mapping[str, object]) -> bool:
|
||||
model: Final = params.get("model")
|
||||
return bool(params.get("use_chat_completions_api")) or (
|
||||
isinstance(model, str) and model.startswith(_OPENAI_CHAT_COMPLETIONS_MODEL_PREFIX)
|
||||
)
|
||||
|
||||
|
||||
def _reads_responses_cache(params: Mapping[str, object]) -> bool:
|
||||
return litellm.cache is not None or params.get("caching") is True
|
||||
|
||||
|
||||
def native_logging_obj(
|
||||
logging_obj: LiteLLMLoggingObj | None, forwarded_kwargs: Mapping[str, object], request: Mapping[str, object]
|
||||
) -> LiteLLMLoggingObj | None:
|
||||
"""The logging object when the native executor can serve the call, None when only `aresponses` can.
|
||||
|
||||
The executor skips the prompt management hooks `aresponses` runs, so a call that needs them re-enters it.
|
||||
So does a deployment that asks for the chat completions API, which `aresponses` bridges to chat, and a call
|
||||
that reads the response cache, which `aresponses` keys and fills.
|
||||
"""
|
||||
if logging_obj is None:
|
||||
return None
|
||||
params: Final = {**forwarded_kwargs, **request}
|
||||
if _requests_chat_completions_api(params) or _reads_responses_cache(params):
|
||||
return None
|
||||
if logging_obj.should_run_prompt_management_hooks(
|
||||
non_default_params=params, prompt_id=_optional_str(params.get("prompt_id"))
|
||||
):
|
||||
return None
|
||||
return logging_obj
|
||||
|
||||
|
||||
def _raise_mapped(error: Exception, model: str, forwarded_kwargs: Mapping[str, object]) -> NoReturn:
|
||||
raise litellm.exception_type(
|
||||
model=model,
|
||||
custom_llm_provider=_optional_str(forwarded_kwargs.get("custom_llm_provider")),
|
||||
original_exception=error,
|
||||
extra_kwargs=dict(forwarded_kwargs),
|
||||
) from error
|
||||
|
||||
|
||||
def _priced(
|
||||
response: ResponsesAPIResponse | BaseResponsesAPIStreamingIterator, logging_obj: LiteLLMLoggingObj
|
||||
) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator:
|
||||
"""The response, priced as `aresponses` would have priced it before the bridge translates it."""
|
||||
if isinstance(response, ResponsesAPIResponse):
|
||||
logging_obj.price_bridged_provider_response(response)
|
||||
else:
|
||||
logging_obj.price_bridged_provider_stream(response)
|
||||
return response
|
||||
|
||||
|
||||
def native_responses(
|
||||
request: Mapping[str, object],
|
||||
model: str,
|
||||
forwarded_kwargs: Mapping[str, object],
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator:
|
||||
try:
|
||||
response: Final = _native_responses_call(request, model, forwarded_kwargs, logging_obj, is_async=False)
|
||||
except Exception as e: # noqa: BLE001 # every provider failure maps to the litellm exception aresponses would raise
|
||||
_raise_mapped(e, model, forwarded_kwargs)
|
||||
if isinstance(response, Coroutine):
|
||||
raise TypeError("The sync Responses executor returned a coroutine")
|
||||
return _priced(response, logging_obj)
|
||||
|
||||
|
||||
async def anative_responses(
|
||||
request: Mapping[str, object],
|
||||
model: str,
|
||||
forwarded_kwargs: Mapping[str, object],
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator:
|
||||
try:
|
||||
response: Final = _native_responses_call(request, model, forwarded_kwargs, logging_obj, is_async=True)
|
||||
settled: Final = await response if isinstance(response, Coroutine) else response
|
||||
except Exception as e: # noqa: BLE001 # every provider failure maps to the litellm exception aresponses would raise
|
||||
_raise_mapped(e, model, forwarded_kwargs)
|
||||
return _priced(settled, logging_obj)
|
||||
|
|
@ -4,7 +4,7 @@ Handler for transforming responses api requests to litellm.completion requests
|
|||
|
||||
from collections.abc import Coroutine, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
|
|
@ -30,6 +30,9 @@ from litellm.types.llms.openai import (
|
|||
)
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.native.chat_plan import PlannedChat
|
||||
|
||||
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
|
|
@ -55,6 +58,29 @@ def _bridged_request(
|
|||
)
|
||||
|
||||
|
||||
def _plan_bridged_chat(request: Mapping[str, object]) -> "PlannedChat | None":
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.native.chat_plan import plan_chat
|
||||
|
||||
logging_obj: Final = request.get("litellm_logging_obj")
|
||||
return plan_chat(request, logging_obj) if isinstance(logging_obj, Logging) else None
|
||||
|
||||
|
||||
def _price_planned_chat_response(
|
||||
request: Mapping[str, object], response: ModelResponse | litellm.CustomStreamWrapper
|
||||
) -> None:
|
||||
"""Prices the chat response as `acompletion` would have, before it becomes a Responses object."""
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
|
||||
logging_obj: Final = request.get("litellm_logging_obj")
|
||||
if not isinstance(logging_obj, Logging):
|
||||
return
|
||||
if isinstance(response, ModelResponse):
|
||||
logging_obj.price_bridged_provider_response(response)
|
||||
else:
|
||||
logging_obj.price_bridged_provider_stream(response)
|
||||
|
||||
|
||||
class LiteLLMCompletionTransformationHandler:
|
||||
def response_api_handler(
|
||||
self,
|
||||
|
|
@ -116,10 +142,18 @@ class LiteLLMCompletionTransformationHandler:
|
|||
**kwargs,
|
||||
)
|
||||
|
||||
litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = litellm.completion(
|
||||
**{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True},
|
||||
completion_args: Final = {**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True}
|
||||
|
||||
from litellm.native.chat_plan import execute_planned_chat
|
||||
|
||||
planned_chat: Final = _plan_bridged_chat(_REQUEST_KWARGS_ADAPTER.validate_python(completion_args))
|
||||
litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = (
|
||||
litellm.completion(**completion_args) if planned_chat is None else execute_planned_chat(planned_chat)
|
||||
)
|
||||
|
||||
if planned_chat is not None:
|
||||
_price_planned_chat_response(completion_args, litellm_completion_response)
|
||||
|
||||
if isinstance(litellm_completion_response, ModelResponse):
|
||||
responses_api_response: Final[ResponsesAPIResponse] = (
|
||||
LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response(
|
||||
|
|
@ -139,6 +173,7 @@ class LiteLLMCompletionTransformationHandler:
|
|||
responses_api_request=bridged_request,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_metadata=kwargs.get("litellm_metadata", {}),
|
||||
logs_completion=planned_chat is not None,
|
||||
)
|
||||
raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}")
|
||||
|
||||
|
|
@ -208,9 +243,20 @@ class LiteLLMCompletionTransformationHandler:
|
|||
input_replaces_history=holds_compaction_summary(with_compaction_items_as_params(request_input)),
|
||||
)
|
||||
|
||||
litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = await litellm.acompletion(
|
||||
**{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True},
|
||||
acompletion_args: Final = {**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True}
|
||||
|
||||
from litellm.native.chat_plan import aexecute_planned_chat
|
||||
|
||||
planned_chat: Final = _plan_bridged_chat(_REQUEST_KWARGS_ADAPTER.validate_python(acompletion_args))
|
||||
litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = (
|
||||
await litellm.acompletion(**acompletion_args)
|
||||
if planned_chat is None
|
||||
else await aexecute_planned_chat(planned_chat)
|
||||
)
|
||||
|
||||
if planned_chat is not None:
|
||||
_price_planned_chat_response(acompletion_args, litellm_completion_response)
|
||||
|
||||
if isinstance(litellm_completion_response, ModelResponse):
|
||||
responses_api_response: Final[ResponsesAPIResponse] = (
|
||||
LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response(
|
||||
|
|
@ -230,5 +276,6 @@ class LiteLLMCompletionTransformationHandler:
|
|||
responses_api_request=responses_api_request,
|
||||
custom_llm_provider=litellm_completion_request.get("custom_llm_provider"),
|
||||
litellm_metadata=kwargs.get("litellm_metadata", {}),
|
||||
logs_completion=planned_chat is not None,
|
||||
)
|
||||
raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}")
|
||||
|
|
|
|||
|
|
@ -2,13 +2,16 @@ import asyncio
|
|||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain
|
||||
from itertools import chain, groupby
|
||||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER
|
||||
from litellm.litellm_core_utils.cold_storage_handler import ColdStorageHandler
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.responses.litellm_completion_transformation.client_tool_search import (
|
||||
bridge_client_tool_search,
|
||||
client_tool_search_loaded_tools,
|
||||
|
|
@ -41,6 +44,7 @@ else:
|
|||
# Cold Storage Handler
|
||||
########################################################
|
||||
COLD_STORAGE_HANDLER: Final = ColdStorageHandler()
|
||||
_STORED_REQUEST: Final = TypeAdapter(dict[str, object])
|
||||
########################################################
|
||||
|
||||
|
||||
|
|
@ -181,6 +185,82 @@ def _instruction_items_before(
|
|||
)
|
||||
|
||||
|
||||
_OBJECT_LIST: Final = TypeAdapter(list[object])
|
||||
|
||||
|
||||
def _stored_web_search_results(stored_response: object) -> Sequence[Mapping[str, object]]:
|
||||
"""The Anthropic ``web_search_tool_result`` blocks a bridged response kept in its ``provider_specific_fields``."""
|
||||
response: Final = as_str_mapping(stored_response)
|
||||
fields: Final = None if response is None else as_str_mapping(response.get("provider_specific_fields"))
|
||||
results: Final = None if fields is None else fields.get("web_search_results")
|
||||
if not isinstance(results, list):
|
||||
return ()
|
||||
return tuple(result for result in map(as_str_mapping, _OBJECT_LIST.validate_python(results)) if result is not None)
|
||||
|
||||
|
||||
def _server_web_search_call_id(item: object, web_search_results: Sequence[Mapping[str, object]]) -> str | None:
|
||||
"""The ``server_tool_use`` id behind a stored ``web_search_call`` item, when its result block was kept."""
|
||||
if not isinstance(item, Mapping) or item.get("type") != "web_search_call":
|
||||
return None
|
||||
item_id: Final = item.get("id")
|
||||
call_id: Final = item_id.removeprefix("ws_") if isinstance(item_id, str) else None
|
||||
if call_id and any(result.get("tool_use_id") == call_id for result in web_search_results):
|
||||
return call_id
|
||||
return None
|
||||
|
||||
|
||||
def _server_web_search_message(items: Sequence[object], web_search_results: Sequence[Mapping[str, object]]) -> Message:
|
||||
"""Replay stored web search calls as the assistant tool calls the chat completion carried,
|
||||
so an Anthropic target rebuilds its ``server_tool_use`` / ``web_search_tool_result`` blocks."""
|
||||
calls: Final = tuple(
|
||||
(call_id, item)
|
||||
for item in map(as_str_mapping, items)
|
||||
if item is not None and (call_id := _server_web_search_call_id(item, web_search_results)) is not None
|
||||
)
|
||||
call_ids: Final = frozenset(call_id for call_id, _ in calls)
|
||||
return Message(
|
||||
role="assistant",
|
||||
content=None,
|
||||
tool_calls=[
|
||||
{
|
||||
"id": call_id,
|
||||
"type": "function",
|
||||
"function": {"name": "web_search", "arguments": json.dumps({"query": _web_search_query(item)})},
|
||||
}
|
||||
for call_id, item in calls
|
||||
],
|
||||
provider_specific_fields={
|
||||
"web_search_results": [result for result in web_search_results if result.get("tool_use_id") in call_ids]
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _web_search_query(item: Mapping[str, object]) -> str:
|
||||
action: Final = item.get("action")
|
||||
query: Final = action.get("query") if isinstance(action, Mapping) else None
|
||||
return query if isinstance(query, str) else ""
|
||||
|
||||
|
||||
_CHAT_MESSAGE_CONTENT_ITEM_TYPES: Final = ("reasoning", "message")
|
||||
|
||||
|
||||
def _in_chat_message_order(
|
||||
output: ResponseInputParam, web_search_results: Sequence[Mapping[str, object]]
|
||||
) -> ResponseInputParam:
|
||||
"""A turn with server web searches in the order its chat completion message replayed it: reasoning, then text,
|
||||
then the tool calls as they happened. Any other turn keeps its output order."""
|
||||
if not any(_server_web_search_call_id(item, web_search_results) is not None for item in output):
|
||||
return output
|
||||
return sorted(
|
||||
output,
|
||||
key=lambda item: (
|
||||
_CHAT_MESSAGE_CONTENT_ITEM_TYPES.index(item_type)
|
||||
if (item_type := item.get("type")) in _CHAT_MESSAGE_CONTENT_ITEM_TYPES
|
||||
else len(_CHAT_MESSAGE_CONTENT_ITEM_TYPES)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _stored_instructions(proxy_server_request: Mapping[str, object] | None) -> str | None:
|
||||
instructions: Final = None if proxy_server_request is None else proxy_server_request.get("instructions")
|
||||
return instructions if isinstance(instructions, str) and instructions else None
|
||||
|
|
@ -371,15 +451,23 @@ class ResponsesSessionHandler:
|
|||
############################################################
|
||||
_response_output: Final[object] = spend_log.get("response", "{}")
|
||||
if (responses_api_output := _stored_responses_api_output(_response_output)) is not None:
|
||||
chat_completion_message_history.extend(
|
||||
LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
|
||||
input=bridge_client_tool_search(responses_api_output, None).input,
|
||||
responses_api_request={},
|
||||
replay_reasoning=True,
|
||||
item_cache_control=item_cache_control,
|
||||
only_since_compaction=True,
|
||||
web_search_results: Final = _stored_web_search_results(_response_output)
|
||||
for is_server_web_search, items in groupby(
|
||||
_in_chat_message_order(responses_api_output, web_search_results),
|
||||
key=lambda item: _server_web_search_call_id(item, web_search_results) is not None,
|
||||
):
|
||||
if is_server_web_search:
|
||||
chat_completion_message_history.append(_server_web_search_message(tuple(items), web_search_results))
|
||||
continue
|
||||
chat_completion_message_history.extend(
|
||||
LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
|
||||
input=bridge_client_tool_search(list(items), None).input,
|
||||
responses_api_request={},
|
||||
replay_reasoning=True,
|
||||
item_cache_control=item_cache_control,
|
||||
only_since_compaction=True,
|
||||
)
|
||||
)
|
||||
)
|
||||
elif isinstance(_response_output, dict) and _response_output and _response_output != {}:
|
||||
# transform `ChatCompletion Response` to `ResponsesAPIResponse`
|
||||
model_response: Final = ModelResponse(**_response_output)
|
||||
|
|
@ -397,30 +485,23 @@ class ResponsesSessionHandler:
|
|||
Get the parsed proxy server request from the spend log
|
||||
"""
|
||||
proxy_server_request: Final[str | dict] = spend_log.get("proxy_server_request") or "{}"
|
||||
proxy_server_request_dict: dict[str, object] | None = None
|
||||
if isinstance(proxy_server_request, dict):
|
||||
proxy_server_request_dict = proxy_server_request
|
||||
else:
|
||||
proxy_server_request_dict = json.loads(proxy_server_request)
|
||||
proxy_server_request_dict: Final = _STORED_REQUEST.validate_python(
|
||||
proxy_server_request if isinstance(proxy_server_request, dict) else json.loads(proxy_server_request)
|
||||
)
|
||||
|
||||
############################################################
|
||||
# Check if user has setup cold storage for session handling
|
||||
############################################################
|
||||
if ResponsesSessionHandler._should_check_cold_storage_for_full_payload(proxy_server_request_dict):
|
||||
# Try to get cold storage object key from spend log metadata
|
||||
_proxy_server_request_dict: dict | None = None
|
||||
cold_storage_object_key = ResponsesSessionHandler._get_cold_storage_object_key_from_spend_log(spend_log)
|
||||
if cold_storage_object_key:
|
||||
# Use the object key directly from metadata
|
||||
_proxy_server_request_dict = (
|
||||
await ResponsesSessionHandler.get_proxy_server_request_from_cold_storage_with_object_key(
|
||||
object_key=cold_storage_object_key,
|
||||
)
|
||||
)
|
||||
if _proxy_server_request_dict:
|
||||
proxy_server_request_dict = _proxy_server_request_dict
|
||||
|
||||
return proxy_server_request_dict
|
||||
if not ResponsesSessionHandler._should_check_cold_storage_for_full_payload(proxy_server_request_dict):
|
||||
return proxy_server_request_dict
|
||||
cold_storage_object_key: Final = ResponsesSessionHandler._get_cold_storage_object_key_from_spend_log(spend_log)
|
||||
if not cold_storage_object_key:
|
||||
return proxy_server_request_dict
|
||||
return (
|
||||
await ResponsesSessionHandler.get_proxy_server_request_from_cold_storage_with_object_key(
|
||||
object_key=cold_storage_object_key,
|
||||
)
|
||||
) or proxy_server_request_dict
|
||||
|
||||
@staticmethod
|
||||
def _get_cold_storage_object_key_from_spend_log(
|
||||
|
|
@ -462,14 +543,10 @@ class ResponsesSessionHandler:
|
|||
"""
|
||||
verbose_proxy_logger.debug("inside get_proxy_server_request_from_cold_storage_with_object_key...")
|
||||
|
||||
proxy_server_request_dict: Final = (
|
||||
await COLD_STORAGE_HANDLER.get_proxy_server_request_from_cold_storage_with_object_key(
|
||||
object_key=object_key,
|
||||
)
|
||||
return await COLD_STORAGE_HANDLER.get_proxy_server_request_from_cold_storage_with_object_key(
|
||||
object_key=object_key,
|
||||
)
|
||||
|
||||
return proxy_server_request_dict
|
||||
|
||||
@staticmethod
|
||||
def _should_check_cold_storage_for_full_payload(
|
||||
proxy_server_request_dict: Mapping[str, object] | None,
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import litellm
|
|||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items
|
||||
from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params, served_from_cache
|
||||
from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.main import stream_chunk_builder
|
||||
from litellm.responses.litellm_completion_transformation.compaction_items import compaction_output_item
|
||||
from litellm.responses.litellm_completion_transformation.custom_tools import (
|
||||
|
|
@ -26,7 +27,11 @@ from litellm.responses.litellm_completion_transformation.tool_search import (
|
|||
from litellm.responses.litellm_completion_transformation.transformation import (
|
||||
LiteLLMCompletionResponsesConfig,
|
||||
)
|
||||
from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator, stamp_responses_usage_cost
|
||||
from litellm.responses.streaming_iterator import (
|
||||
ResponsesAPIStreamingIterator,
|
||||
dispatch_stream_success,
|
||||
stamp_responses_usage_cost,
|
||||
)
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.types.llms.openai import (
|
||||
PART_UNION_TYPES,
|
||||
|
|
@ -119,6 +124,10 @@ def _delta_carries_reasoning_and_answer(delta: ChatCompletionDelta) -> bool:
|
|||
return has_reasoning and bool(delta.content or delta.tool_calls)
|
||||
|
||||
|
||||
def _logged_by_this_stream(_assembled: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
||||
"""
|
||||
Async iterator for processing streaming responses from the Responses API.
|
||||
|
|
@ -132,9 +141,14 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
responses_api_request: ResponsesAPIOptionalRequestParams,
|
||||
custom_llm_provider: str | None = None,
|
||||
litellm_metadata: dict | None = None,
|
||||
logs_completion: bool = False,
|
||||
):
|
||||
"""``logs_completion`` makes this stream, not the chat stream it wraps, log the request as a Responses row."""
|
||||
self.model: str = model
|
||||
self.litellm_custom_stream_wrapper: litellm.CustomStreamWrapper = litellm_custom_stream_wrapper
|
||||
self._logs_completion = logs_completion
|
||||
if logs_completion:
|
||||
litellm_custom_stream_wrapper.hand_completion_to(_logged_by_this_stream)
|
||||
self.request_input: str | ResponseInputParam = request_input
|
||||
self.responses_api_request: ResponsesAPIOptionalRequestParams = responses_api_request
|
||||
self.custom_llm_provider: str | None = custom_llm_provider
|
||||
|
|
@ -143,7 +157,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
# repeated Pydantic attribute access in end-of-stream assembly.
|
||||
self.collected_chat_completion_chunks: list[dict[str, object]] = []
|
||||
self.finished: bool = False
|
||||
self.litellm_logging_obj = litellm_custom_stream_wrapper.logging_obj
|
||||
self.litellm_logging_obj: LiteLLMLoggingObj = litellm_custom_stream_wrapper.logging_obj
|
||||
self.sent_response_created_event: bool = False
|
||||
self.sent_response_in_progress_event: bool = False
|
||||
self.sent_output_item_added_event: bool = False
|
||||
|
|
@ -1120,6 +1134,15 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
# Latch so wrappers (FallbackResponsesStreamWrapper) + proxy
|
||||
# container-ownership hook can read completed_response.
|
||||
self.completed_response = response_completed_event
|
||||
if self._logs_completion:
|
||||
dispatch_stream_success(
|
||||
self.litellm_logging_obj,
|
||||
response_completed_event,
|
||||
start_time=None,
|
||||
end_time=None,
|
||||
cache_hit=None,
|
||||
is_async=not sync_mode,
|
||||
)
|
||||
return response_completed_event
|
||||
else:
|
||||
if sync_mode:
|
||||
|
|
@ -1493,6 +1516,12 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
def bill_failure_after_earlier_rounds(self, earlier: BilledRoundsSpend) -> None:
|
||||
self.litellm_custom_stream_wrapper.bill_failure_after_earlier_rounds(earlier)
|
||||
|
||||
def log_rounds_as_chat(self) -> None:
|
||||
"""An MCP auto-execute bills its rounds from each chat stream's assembled response, so the chat stream logs."""
|
||||
if self._logs_completion:
|
||||
self._logs_completion = False
|
||||
self.litellm_custom_stream_wrapper.take_completion_back()
|
||||
|
||||
def partial_response_on_disconnect(self) -> ResponseIncompleteEvent | None:
|
||||
"""
|
||||
The response.incomplete event to bill when the client went away mid-stream.
|
||||
|
|
|
|||
|
|
@ -6,10 +6,10 @@ from contextlib import contextmanager
|
|||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast, get_type_hints
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast, get_type_hints
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
|
||||
from pydantic import BaseModel, TypeAdapter, ValidationError
|
||||
from typing_extensions import (
|
||||
TypeIs, # noqa: TID251 # narrows a client input item before its mark is read
|
||||
assert_never,
|
||||
|
|
@ -29,7 +29,6 @@ from litellm.integrations.anthropic_cache_control_hook import (
|
|||
configured_injection_points,
|
||||
)
|
||||
from litellm.litellm_core_utils.asyncify import run_async_function
|
||||
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
|
@ -43,6 +42,8 @@ from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfi
|
|||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from litellm.llms.openai.common_utils import get_openai_credentials
|
||||
from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig
|
||||
from litellm.native.responses import execute_responses
|
||||
from litellm.native.route_plan import NativeChat, plan_responses_route
|
||||
from litellm.responses.litellm_completion_transformation.compaction_items import (
|
||||
with_compaction_items_as_summaries,
|
||||
)
|
||||
|
|
@ -92,7 +93,6 @@ from litellm.types.utils import (
|
|||
ADDRESSED_RESPONSE_ID_FIELD,
|
||||
AgenticLoopParams,
|
||||
CallTypes,
|
||||
CostPerToken,
|
||||
all_litellm_params,
|
||||
)
|
||||
from litellm.utils import (
|
||||
|
|
@ -565,48 +565,6 @@ async def aresponses_api_with_mcp(
|
|||
return response
|
||||
|
||||
|
||||
class _DeploymentModelInfo(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
id: object = None
|
||||
|
||||
|
||||
_DEPLOYMENT_MODEL_INFO: Final[TypeAdapter[_DeploymentModelInfo | object]] = TypeAdapter(
|
||||
Annotated[_DeploymentModelInfo | object, Field(union_mode="left_to_right")]
|
||||
)
|
||||
|
||||
|
||||
def _is_router_deployment(deployment_model_info: object) -> bool:
|
||||
"""A Router stamps the deployment's model_info (with its id) onto every call it routes."""
|
||||
parsed: Final = _DEPLOYMENT_MODEL_INFO.validate_python(deployment_model_info)
|
||||
return isinstance(parsed, _DeploymentModelInfo) and parsed.id is not None
|
||||
|
||||
|
||||
_COST_PER_TOKEN: Final = TypeAdapter(CostPerToken)
|
||||
|
||||
|
||||
def _per_call_cost_per_token(litellm_params: GenericLiteLLMParams) -> CostPerToken | None:
|
||||
if litellm_params.input_cost_per_token is None or litellm_params.output_cost_per_token is None:
|
||||
return None
|
||||
return _COST_PER_TOKEN.validate_python(
|
||||
litellm_params.model_dump(
|
||||
include={
|
||||
"input_cost_per_token",
|
||||
"output_cost_per_token",
|
||||
"cache_read_input_token_cost",
|
||||
"cache_creation_input_token_cost",
|
||||
},
|
||||
exclude_none=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _bridges_to_chat_completions(
|
||||
responses_api_provider_config: BaseResponsesAPIConfig | None, use_chat_completions_api: bool
|
||||
) -> bool:
|
||||
"""Whether the request reaches its provider as a chat completion, not a Responses call."""
|
||||
return responses_api_provider_config is None or use_chat_completions_api is True
|
||||
|
||||
|
||||
_RESPONSES_ONLY_REQUEST_FIELDS_NEVER_BRIDGED: Final = frozenset({"client_metadata"})
|
||||
|
||||
|
||||
|
|
@ -693,7 +651,7 @@ def _encrypted_task_support_failure(
|
|||
) -> _ResponsesCompatibilityFailure | None:
|
||||
if (
|
||||
responses_api_provider_config is None
|
||||
or _bridges_to_chat_completions(responses_api_provider_config, use_chat_completions_api)
|
||||
or isinstance(plan_responses_route(responses_api_provider_config, use_chat_completions_api), NativeChat)
|
||||
or not responses_api_provider_config.supports_encrypted_agent_messages()
|
||||
):
|
||||
return "encrypted_task_unsupported"
|
||||
|
|
@ -737,7 +695,7 @@ class _AsyncPromptManagementOutcome:
|
|||
deployment_model_info: object
|
||||
|
||||
|
||||
def _resolve_responses_api_provider_config(
|
||||
def resolve_responses_api_provider_config(
|
||||
model: str, custom_llm_provider: str, model_info: object, api_base: str | None
|
||||
) -> BaseResponsesAPIConfig | None:
|
||||
provider_config: Final = ProviderConfigManager.get_provider_responses_api_config(
|
||||
|
|
@ -770,7 +728,7 @@ def _will_bridge_to_chat_completions(
|
|||
api_base: str | None,
|
||||
litellm_params: GenericLiteLLMParams | None = None,
|
||||
) -> bool:
|
||||
"""``_bridges_to_chat_completions`` for callers running before the provider config is resolved.
|
||||
"""``plan_responses_route`` for callers running before the provider config is resolved.
|
||||
|
||||
Resolving the config is a pure lookup, so this asks the same question the dispatch
|
||||
asks rather than restating its condition, with the model name the dispatch hands the
|
||||
|
|
@ -783,14 +741,17 @@ def _will_bridge_to_chat_completions(
|
|||
normalized_model: Final = _normalize_openai_chat_completions_responses_model(model)
|
||||
if custom_llm_provider is None:
|
||||
return True
|
||||
return _bridges_to_chat_completions(
|
||||
_resolve_responses_api_provider_config(
|
||||
_dispatched_model_name(normalized_model[0], custom_llm_provider, api_base, litellm_params),
|
||||
custom_llm_provider,
|
||||
model_info,
|
||||
api_base,
|
||||
return isinstance(
|
||||
plan_responses_route(
|
||||
resolve_responses_api_provider_config(
|
||||
_dispatched_model_name(normalized_model[0], custom_llm_provider, api_base, litellm_params),
|
||||
custom_llm_provider,
|
||||
model_info,
|
||||
api_base,
|
||||
),
|
||||
use_chat_completions_api or normalized_model[1],
|
||||
),
|
||||
use_chat_completions_api or normalized_model[1],
|
||||
NativeChat,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1251,7 +1212,88 @@ def _strip_responses_routing_prefix(model: str) -> str:
|
|||
return model[len(_RESPONSES_ROUTING_PREFIX) :]
|
||||
|
||||
|
||||
def _resolve_model_provider_for_responses(
|
||||
def with_provider_specific_headers(
|
||||
extra_headers: dict[str, object] | None, litellm_params: GenericLiteLLMParams, custom_llm_provider: str | None
|
||||
) -> dict[str, object] | None:
|
||||
from litellm.litellm_core_utils.core_helpers import attribute_or_none
|
||||
|
||||
return ResponsesAPIRequestUtils.merge_provider_specific_headers(
|
||||
extra_headers=extra_headers,
|
||||
provider_specific_header=attribute_or_none(litellm_params, "provider_specific_header"),
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
|
||||
def with_provider_affinity_header(
|
||||
extra_headers: dict[str, object] | None,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
metadata: Mapping[str, object] | None,
|
||||
kwargs: Mapping[str, object],
|
||||
model: str,
|
||||
custom_llm_provider: str | None,
|
||||
) -> dict[str, object] | None:
|
||||
affinity_api_base: Final = (
|
||||
get_openai_credentials(api_base=litellm_params.api_base).api_base
|
||||
if custom_llm_provider == "openai"
|
||||
else litellm_params.api_base
|
||||
)
|
||||
default_affinity_header: Final = get_provider_affinity_header(affinity_api_base)
|
||||
if litellm_params.provider_affinity_header is None and default_affinity_header is None:
|
||||
return extra_headers
|
||||
try:
|
||||
return add_provider_affinity_header(
|
||||
headers=extra_headers or MappingProxyType({}),
|
||||
litellm_params=MappingProxyType(
|
||||
{
|
||||
"provider_affinity_header": litellm_params.provider_affinity_header,
|
||||
"litellm_session_id": kwargs.get("litellm_session_id"),
|
||||
"session_id": kwargs.get("session_id"),
|
||||
"metadata": metadata,
|
||||
"litellm_metadata": kwargs.get("litellm_metadata"),
|
||||
}
|
||||
),
|
||||
default_header=default_affinity_header,
|
||||
)
|
||||
except ValueError as affinity_error:
|
||||
raise litellm.BadRequestError(
|
||||
message=str(affinity_error), model=model, llm_provider=custom_llm_provider
|
||||
) from affinity_error
|
||||
|
||||
|
||||
def requested_reasoning(params: Mapping[str, object], model: str, custom_llm_provider: str | None) -> Reasoning | None:
|
||||
current_reasoning: Final = cast( # cast-ok: prompt-managed reasoning arrives as a plain dict
|
||||
Reasoning | None, params.get("reasoning")
|
||||
)
|
||||
reasoning_effort: Final = params.get("reasoning_effort")
|
||||
if current_reasoning is None and reasoning_effort is not None:
|
||||
return LiteLLMResponsesTransformationHandler().map_reasoning_effort(
|
||||
reasoning_effort, model, custom_llm_provider
|
||||
)
|
||||
return current_reasoning
|
||||
|
||||
|
||||
def reasoning_for_deployment(
|
||||
reasoning: Reasoning | None,
|
||||
model: str,
|
||||
custom_llm_provider: str | None,
|
||||
deployment_model_info: object,
|
||||
kwargs: Mapping[str, object],
|
||||
) -> Reasoning | None:
|
||||
"""The requested reasoning with its effort resolved onto the deployment's declared reasoning_effort_levels."""
|
||||
return cast( # cast-ok: the resolver hands a Reasoning back in the shape it was given
|
||||
Reasoning | None,
|
||||
resolve_deployment_reasoning_effort(
|
||||
reasoning,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
deployment_model_info=deployment_model_info,
|
||||
model_group=model_group_from_metadata(kwargs.get("litellm_metadata")),
|
||||
drop_params=litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True,
|
||||
).value,
|
||||
)
|
||||
|
||||
|
||||
def resolve_model_provider_for_responses(
|
||||
model: str,
|
||||
custom_llm_provider: str | None,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
|
|
@ -1276,7 +1318,7 @@ def _resolve_model_provider_for_responses(
|
|||
return _strip_responses_routing_prefix(provider_model), resolved_provider
|
||||
|
||||
|
||||
def _apply_managed_file_id_mapping(
|
||||
def apply_managed_file_id_mapping(
|
||||
input: str | ResponseInputParam,
|
||||
tools: Iterable[ToolParam] | None,
|
||||
kwargs: dict[str, Any],
|
||||
|
|
@ -1523,7 +1565,6 @@ def responses(
|
|||
|
||||
try:
|
||||
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
|
||||
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
|
||||
_is_async: Final = kwargs.pop("aresponses", False) is True
|
||||
skip_mcp_handler: Final = kwargs.pop("_skip_mcp_handler", False)
|
||||
require_encrypted_task_support: Final = kwargs.pop("_require_encrypted_task_support", False) is True
|
||||
|
|
@ -1576,44 +1617,12 @@ def responses(
|
|||
)
|
||||
litellm_logging_obj.record_agentic_loop_params(AgenticLoopParams(model=model))
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import attribute_or_none
|
||||
|
||||
litellm_params: Final = GenericLiteLLMParams(extra_headers=extra_headers, **kwargs)
|
||||
provider_headers: Final = ResponsesAPIRequestUtils.merge_provider_specific_headers(
|
||||
extra_headers=extra_headers,
|
||||
provider_specific_header=attribute_or_none(litellm_params, "provider_specific_header"),
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
provider_headers: Final = with_provider_specific_headers(extra_headers, litellm_params, custom_llm_provider)
|
||||
local_vars["extra_headers"] = provider_headers
|
||||
|
||||
affinity_api_base: Final = (
|
||||
get_openai_credentials(api_base=litellm_params.api_base).api_base
|
||||
if custom_llm_provider == "openai"
|
||||
else litellm_params.api_base
|
||||
effective_extra_headers: Final = with_provider_affinity_header(
|
||||
provider_headers, litellm_params, metadata, kwargs, model, custom_llm_provider
|
||||
)
|
||||
default_affinity_header: Final = get_provider_affinity_header(affinity_api_base)
|
||||
try:
|
||||
effective_extra_headers: Final = (
|
||||
add_provider_affinity_header(
|
||||
headers=provider_headers or MappingProxyType({}),
|
||||
litellm_params=MappingProxyType(
|
||||
{
|
||||
"provider_affinity_header": litellm_params.provider_affinity_header,
|
||||
"litellm_session_id": kwargs.get("litellm_session_id"),
|
||||
"session_id": kwargs.get("session_id"),
|
||||
"metadata": metadata,
|
||||
"litellm_metadata": kwargs.get("litellm_metadata"),
|
||||
}
|
||||
),
|
||||
default_header=default_affinity_header,
|
||||
)
|
||||
if litellm_params.provider_affinity_header is not None or default_affinity_header is not None
|
||||
else provider_headers
|
||||
)
|
||||
except ValueError as affinity_error:
|
||||
raise litellm.BadRequestError(
|
||||
message=str(affinity_error), model=model, llm_provider=custom_llm_provider
|
||||
) from affinity_error
|
||||
|
||||
#########################################################
|
||||
# MOCK RESPONSE LOGIC
|
||||
|
|
@ -1621,7 +1630,7 @@ def responses(
|
|||
if litellm_params.mock_response and isinstance(litellm_params.mock_response, str):
|
||||
return mock_responses_api_response(mock_response=litellm_params.mock_response)
|
||||
|
||||
model, custom_llm_provider = _resolve_model_provider_for_responses(
|
||||
model, custom_llm_provider = resolve_model_provider_for_responses(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params,
|
||||
|
|
@ -1631,7 +1640,7 @@ def responses(
|
|||
#########################################################
|
||||
# Update input and tools with provider-specific file IDs if managed files are used
|
||||
#########################################################
|
||||
input, tools = _apply_managed_file_id_mapping(input=input, tools=tools, kwargs=kwargs, local_vars=local_vars)
|
||||
input, tools = apply_managed_file_id_mapping(input=input, tools=tools, kwargs=kwargs, local_vars=local_vars)
|
||||
|
||||
#########################################################
|
||||
# Native MCP Responses API
|
||||
|
|
@ -1674,7 +1683,7 @@ def responses(
|
|||
if custom_llm_provider is None:
|
||||
responses_api_provider_config = None
|
||||
else:
|
||||
responses_api_provider_config = _resolve_responses_api_provider_config(
|
||||
responses_api_provider_config = resolve_responses_api_provider_config(
|
||||
model, custom_llm_provider, deployment_model_info, litellm_params.api_base
|
||||
)
|
||||
|
||||
|
|
@ -1690,27 +1699,9 @@ def responses(
|
|||
_raise_responses_compatibility_failure(compatibility_failure, model, custom_llm_provider)
|
||||
|
||||
local_vars.update(kwargs)
|
||||
current_reasoning: Final = cast( # cast-ok: prompt-managed reasoning arrives as a plain dict
|
||||
Reasoning | None, local_vars.get("reasoning")
|
||||
)
|
||||
reasoning_effort: Final[object] = local_vars.get("reasoning_effort")
|
||||
request_reasoning: Final = (
|
||||
LiteLLMResponsesTransformationHandler().map_reasoning_effort(reasoning_effort, model, custom_llm_provider)
|
||||
if current_reasoning is None and reasoning_effort is not None
|
||||
else current_reasoning
|
||||
)
|
||||
routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict
|
||||
deployment_reasoning: Final = cast( # cast-ok: the resolver hands a Reasoning back in the shape it was given
|
||||
Reasoning | None,
|
||||
resolve_deployment_reasoning_effort(
|
||||
request_reasoning,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
deployment_model_info=deployment_model_info,
|
||||
model_group=model_group_from_metadata(routing_kwargs.get("litellm_metadata")),
|
||||
drop_params=litellm.drop_params is True
|
||||
or normalize_drop_params(routing_kwargs.get("drop_params")) is True,
|
||||
).value,
|
||||
request_reasoning: Final = requested_reasoning(local_vars, model, custom_llm_provider)
|
||||
deployment_reasoning: Final = reasoning_for_deployment(
|
||||
request_reasoning, model, custom_llm_provider, deployment_model_info, kwargs
|
||||
)
|
||||
response_api_optional_params: Final[ResponsesAPIOptionalRequestParams] = (
|
||||
ResponsesAPIRequestUtils.get_requested_response_api_optional_param(
|
||||
|
|
@ -1756,7 +1747,7 @@ def responses(
|
|||
if _file_search_dispatch is not None:
|
||||
return _file_search_dispatch
|
||||
|
||||
if _bridges_to_chat_completions(responses_api_provider_config, use_chat_completions_api):
|
||||
if isinstance(plan_responses_route(responses_api_provider_config, use_chat_completions_api), NativeChat):
|
||||
bridge_kwargs: Final = _bridge_kwargs(kwargs, responses_api_provider_config, allowed_openai_params)
|
||||
if isinstance(bridge_kwargs, _UntranslatableResponsesFields):
|
||||
_raise_untranslatable_responses_fields(bridge_kwargs, model, custom_llm_provider)
|
||||
|
|
@ -1775,88 +1766,32 @@ def responses(
|
|||
**bridge_kwargs,
|
||||
)
|
||||
|
||||
# Get optional parameters for the responses API
|
||||
request_drop_params: Final = kwargs.get("drop_params")
|
||||
responses_api_request_params: Final[dict] = ResponsesAPIRequestUtils.get_optional_params_responses_api(
|
||||
model=model,
|
||||
responses_api_provider_config=responses_api_provider_config,
|
||||
response_api_optional_params=response_api_optional_params,
|
||||
allowed_openai_params=allowed_openai_params,
|
||||
drop_params=normalize_drop_params(request_drop_params),
|
||||
declared_reasoning_efforts=deployment_reasoning_effort_levels(deployment_model_info) or (),
|
||||
)
|
||||
|
||||
litellm_logging_obj.update_from_kwargs(
|
||||
kwargs=kwargs,
|
||||
model=model,
|
||||
user=user,
|
||||
optional_params=dict(responses_api_request_params),
|
||||
litellm_params={
|
||||
**responses_api_request_params,
|
||||
**litellm_params.custom_pricing_overrides(),
|
||||
"aresponses": _is_async,
|
||||
"litellm_call_id": litellm_call_id,
|
||||
"model_info": kwargs.get("model_info"),
|
||||
"data_residency": infer_openai_data_residency(custom_llm_provider, litellm_params.api_base),
|
||||
"metadata": (kwargs["litellm_metadata"] if "litellm_metadata" in kwargs else kwargs.get("metadata")),
|
||||
"provider_affinity_header": litellm_params.provider_affinity_header,
|
||||
},
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
if not _is_router_deployment(deployment_model_info):
|
||||
litellm_logging_obj.custom_cost_per_token = _per_call_cost_per_token(litellm_params)
|
||||
litellm_logging_obj.custom_cost_per_second = litellm_params.cost_per_second
|
||||
# Decode any litellm-encoded encrypted-content item IDs back to their original IDs
|
||||
input = with_compaction_items_as_summaries(
|
||||
with_compaction_items_as_params(ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(input))
|
||||
)
|
||||
|
||||
# Call the handler with _is_async flag instead of directly calling the async handler
|
||||
if custom_llm_provider is None:
|
||||
if custom_llm_provider is None or responses_api_provider_config is None:
|
||||
raise ValueError("custom_llm_provider is required but passed as None")
|
||||
|
||||
fake_stream: Final = responses_api_provider_config.should_fake_stream(
|
||||
model=model, stream=stream, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
applied_timeout: Final = CompletionTimeout.for_stream(
|
||||
(StreamKind.FAKED if fake_stream else StreamKind.LIVE) if stream is True else StreamKind.NOT_STREAMED,
|
||||
CompletionTimeout.as_limit(timeout or request_timeout),
|
||||
None
|
||||
if stream is not True or litellm_params.stream_timeout is None
|
||||
else float(litellm_params.stream_timeout),
|
||||
)
|
||||
litellm_logging_obj.record_applied_timeout(applied_timeout)
|
||||
response = base_llm_http_handler.response_api_handler(
|
||||
return execute_responses(
|
||||
http_handler=base_llm_http_handler,
|
||||
model=model,
|
||||
input=input,
|
||||
responses_api_provider_config=responses_api_provider_config,
|
||||
response_api_optional_request_params=responses_api_request_params,
|
||||
provider_config=responses_api_provider_config,
|
||||
requested_params=response_api_optional_params,
|
||||
allowed_openai_params=allowed_openai_params,
|
||||
declared_reasoning_efforts=deployment_reasoning_effort_levels(deployment_model_info) or (),
|
||||
deployment_model_info=deployment_model_info,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params,
|
||||
logging_obj=litellm_logging_obj,
|
||||
user=user,
|
||||
extra_headers=effective_extra_headers,
|
||||
extra_body=extra_body,
|
||||
timeout=applied_timeout,
|
||||
_is_async=_is_async,
|
||||
timeout=timeout or request_timeout,
|
||||
is_async=_is_async,
|
||||
stream=stream,
|
||||
client=kwargs.get("client"),
|
||||
fake_stream=fake_stream,
|
||||
litellm_metadata=kwargs.get("litellm_metadata", {}),
|
||||
shared_session=kwargs.get("shared_session"),
|
||||
litellm_metadata=kwargs.get("litellm_metadata", {}),
|
||||
request_kwargs=kwargs,
|
||||
)
|
||||
|
||||
# Update the responses_api_response_id with the model_id
|
||||
if isinstance(response, ResponsesAPIResponse):
|
||||
response = ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id(
|
||||
responses_api_response=response,
|
||||
litellm_metadata=kwargs.get("litellm_metadata", {}),
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
# Stamp custom_llm_provider so callbacks can identify the provider
|
||||
# (mirrors litellm/main.py:1371 for chat completions)
|
||||
response.hidden_params["custom_llm_provider"] = custom_llm_provider
|
||||
|
||||
return response
|
||||
except Exception as e:
|
||||
raise litellm.exception_type(
|
||||
model=model,
|
||||
|
|
@ -2694,7 +2629,7 @@ def compact_responses(
|
|||
# get llm provider logic
|
||||
litellm_params: Final = GenericLiteLLMParams(**kwargs)
|
||||
|
||||
model, custom_llm_provider = _resolve_model_provider_for_responses(
|
||||
model, custom_llm_provider = resolve_model_provider_for_responses(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params,
|
||||
|
|
|
|||
|
|
@ -475,6 +475,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator):
|
|||
if not self.should_auto_execute:
|
||||
return None
|
||||
if isinstance(self.base_iterator, LiteLLMCompletionStreamingIterator):
|
||||
self.base_iterator.log_rounds_as_chat()
|
||||
return StreamedRoundBilling(self.base_iterator.litellm_logging_obj)
|
||||
if isinstance(self.base_iterator, BaseResponsesAPIStreamingIterator):
|
||||
return StreamedRoundBilling(self.base_iterator.logging_obj)
|
||||
|
|
@ -1046,6 +1047,8 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator):
|
|||
self._cached_response_id = None
|
||||
self._round_index += 1
|
||||
if self._round_billing is not None:
|
||||
if isinstance(follow_up_response, LiteLLMCompletionStreamingIterator):
|
||||
follow_up_response.log_rounds_as_chat()
|
||||
self._round_billing.record_billed_rounds(follow_up_response)
|
||||
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ from litellm.litellm_core_utils.llm_response_utils.get_headers import provider_r
|
|||
from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
|
||||
update_response_metadata,
|
||||
)
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.litellm_core_utils.thread_pool_executor import executor
|
||||
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
|
||||
from litellm.responses.litellm_completion_transformation.transformation import (
|
||||
|
|
@ -366,6 +367,50 @@ def _mid_stream_fallback_eligible(mapped_exception: Exception) -> bool:
|
|||
PRE_OUTPUT_LIFECYCLE_EVENT_TYPES: Final = frozenset({"response.created", "response.in_progress", "response.queued"})
|
||||
|
||||
|
||||
def dispatch_stream_success(
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
logging_response: object,
|
||||
*,
|
||||
start_time: datetime | None,
|
||||
end_time: datetime | None,
|
||||
cache_hit: bool | None,
|
||||
is_async: bool,
|
||||
client_detached: bool = False,
|
||||
) -> None:
|
||||
if not is_async:
|
||||
run_async_function(
|
||||
async_function=logging_obj.async_success_handler,
|
||||
result=logging_response,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
cache_hit=cache_hit,
|
||||
)
|
||||
executor.submit(
|
||||
logging_obj.success_handler,
|
||||
result=logging_response,
|
||||
cache_hit=cache_hit,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
)
|
||||
return
|
||||
logging_coroutine: Final = logging_obj.dispatch_success_handlers(
|
||||
logging_response,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
cache_hit=cache_hit,
|
||||
prefer_async_handlers=True,
|
||||
)
|
||||
if client_detached:
|
||||
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=logging_coroutine)
|
||||
return
|
||||
if getattr(logging_obj, "_on_deferred_stream_complete", None) is not None:
|
||||
# End-of-stream guardrail scans write guardrail_information after
|
||||
# the terminal event; dispatching now would snapshot metadata early.
|
||||
logging_obj.hold_stream_completion((logging_coroutine,))
|
||||
return
|
||||
spawn_logging_task(asyncio.get_running_loop(), logging_coroutine, task_name="Responses stream success logging")
|
||||
|
||||
|
||||
class BaseResponsesAPIStreamingIterator:
|
||||
"""
|
||||
Base class for streaming iterators that process responses from the Responses API.
|
||||
|
|
@ -377,6 +422,7 @@ class BaseResponsesAPIStreamingIterator:
|
|||
_started_tool_call: bool = False
|
||||
_earlier_rounds_spend: BilledRoundsSpend | None = None
|
||||
_replays_stored_stream: bool = False
|
||||
_on_complete: Callable[[object], None] | None = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -430,6 +476,10 @@ class BaseResponsesAPIStreamingIterator:
|
|||
) # GUARANTEE OPENAI HEADERS IN RESPONSE
|
||||
self._raw_response_headers: Mapping[str, str] = MappingProxyType(dict(self.response.headers or {}))
|
||||
|
||||
def hand_completion_to(self, on_complete: Callable[[object], None]) -> None:
|
||||
"""A bridge whose own client stream logs the request takes the completed response instead of the logger."""
|
||||
self._on_complete = on_complete
|
||||
|
||||
def _check_max_streaming_duration(self) -> None:
|
||||
"""Raise litellm.Timeout if the stream has exceeded LITELLM_MAX_STREAMING_DURATION_SECONDS."""
|
||||
if LITELLM_MAX_STREAMING_DURATION_SECONDS is None:
|
||||
|
|
@ -634,37 +684,16 @@ class BaseResponsesAPIStreamingIterator:
|
|||
self._restore_provider_response_headers(logging_response)
|
||||
|
||||
end_time: Final = datetime.now()
|
||||
if is_async:
|
||||
logging_coroutine: Final = self.logging_obj.dispatch_success_handlers(
|
||||
if self._on_complete is not None:
|
||||
self._on_complete(logging_response)
|
||||
else:
|
||||
dispatch_stream_success(
|
||||
self.logging_obj,
|
||||
logging_response,
|
||||
start_time=self.start_time,
|
||||
end_time=end_time,
|
||||
cache_hit=self._completed_response_cache_hit,
|
||||
prefer_async_handlers=True,
|
||||
)
|
||||
deferred_dispatch_armed: Final = getattr(self.logging_obj, "_on_deferred_stream_complete", None) is not None
|
||||
if deferred_dispatch_armed:
|
||||
# End-of-stream guardrail scans write guardrail_information after
|
||||
# the terminal event; dispatching now would snapshot metadata early.
|
||||
self.logging_obj._deferred_stream_complete_args = (logging_coroutine,)
|
||||
else:
|
||||
spawn_logging_task(
|
||||
asyncio.get_running_loop(), logging_coroutine, task_name="Responses stream success logging"
|
||||
)
|
||||
else:
|
||||
run_async_function(
|
||||
async_function=self.logging_obj.async_success_handler,
|
||||
result=logging_response,
|
||||
start_time=self.start_time,
|
||||
end_time=end_time,
|
||||
cache_hit=self._completed_response_cache_hit,
|
||||
)
|
||||
executor.submit(
|
||||
self.logging_obj.success_handler,
|
||||
result=logging_response,
|
||||
cache_hit=self._completed_response_cache_hit,
|
||||
start_time=self.start_time,
|
||||
end_time=end_time,
|
||||
is_async=is_async,
|
||||
)
|
||||
self._run_post_success_hooks(end_time=end_time)
|
||||
|
||||
|
|
|
|||
|
|
@ -49,7 +49,9 @@ class _UsageExtraFields(ExtraItemsTypedDict, total=False, extra_items=object):
|
|||
pass
|
||||
|
||||
|
||||
def _apply_nested_drop_params(params: dict[str, object], additional_drop_params: list[str] | None) -> dict[str, object]:
|
||||
def _apply_nested_drop_params(
|
||||
params: dict[str, object], additional_drop_params: Sequence[str] | None
|
||||
) -> dict[str, object]:
|
||||
nested_paths: Final = tuple(path for path in additional_drop_params or () if is_nested_path(path))
|
||||
return reduce(lambda acc, path: delete_nested_value(acc, path), nested_paths, params)
|
||||
|
||||
|
|
@ -224,7 +226,7 @@ class ResponsesAPIRequestUtils:
|
|||
@staticmethod
|
||||
def merge_client_forwarded_headers(
|
||||
extra_headers: dict[str, object] | None,
|
||||
client_headers: dict[str, str] | None,
|
||||
client_headers: Mapping[str, str] | None,
|
||||
) -> dict[str, object] | None:
|
||||
"""
|
||||
Merge headers forwarded by the proxy (`headers` kwarg, set when
|
||||
|
|
@ -259,8 +261,8 @@ class ResponsesAPIRequestUtils:
|
|||
|
||||
@staticmethod
|
||||
def _check_valid_arg(
|
||||
supported_params: list[str] | None,
|
||||
non_default_params: dict,
|
||||
supported_params: Sequence[str] | None,
|
||||
non_default_params: Mapping[str, object],
|
||||
drop_params: bool | None,
|
||||
custom_llm_provider: str | None,
|
||||
model: str,
|
||||
|
|
@ -285,7 +287,7 @@ class ResponsesAPIRequestUtils:
|
|||
model: str,
|
||||
responses_api_provider_config: BaseResponsesAPIConfig,
|
||||
response_api_optional_params: ResponsesAPIOptionalRequestParams,
|
||||
allowed_openai_params: list[str] | None = None,
|
||||
allowed_openai_params: Sequence[str] | None = None,
|
||||
drop_params: bool | None = None,
|
||||
declared_reasoning_efforts: Collection[str] | None = None,
|
||||
) -> dict:
|
||||
|
|
@ -311,7 +313,7 @@ class ResponsesAPIRequestUtils:
|
|||
non_default_params: Final = cast(dict, response_api_optional_params)
|
||||
# Check for unsupported parameters
|
||||
ResponsesAPIRequestUtils._check_valid_arg(
|
||||
supported_params=supported_params + (allowed_openai_params or []),
|
||||
supported_params=[*supported_params, *(allowed_openai_params or ())],
|
||||
non_default_params=non_default_params,
|
||||
drop_params=should_drop_params,
|
||||
custom_llm_provider=responses_api_provider_config.custom_llm_provider,
|
||||
|
|
@ -340,7 +342,7 @@ class ResponsesAPIRequestUtils:
|
|||
return apply_openai_param_overrides(
|
||||
optional_params=params_with_normalized_stream_options,
|
||||
non_default_params=non_default_params,
|
||||
allowed_openai_params=allowed_openai_params or [],
|
||||
allowed_openai_params=list(allowed_openai_params or ()),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -1311,7 +1313,7 @@ class ResponseAPILoggingUtils:
|
|||
output_tokens: Final = usage_input.get("output_tokens")
|
||||
if isinstance(input_tokens, int) and isinstance(output_tokens, int):
|
||||
usage_input["total_tokens"] = input_tokens + output_tokens
|
||||
response_api_usage = ResponseAPIUsage(**usage_input)
|
||||
response_api_usage = ResponseAPIUsage.model_validate(usage_input)
|
||||
else:
|
||||
response_api_usage = usage_input
|
||||
prompt_tokens: Final[int] = response_api_usage.input_tokens or 0
|
||||
|
|
|
|||
|
|
@ -245,7 +245,7 @@ class CompletionDispatchContext:
|
|||
_CompletionDispatchContext = CompletionDispatchContext
|
||||
|
||||
|
||||
_CompletionDispatchResult = Union[
|
||||
CompletionDispatchResult = Union[
|
||||
Coroutine[object, object, Union["ModelResponse", "CustomStreamWrapper"]],
|
||||
"ModelResponse",
|
||||
"CustomStreamWrapper",
|
||||
|
|
|
|||
|
|
@ -9251,11 +9251,11 @@ def validate_and_fix_openai_messages(messages: list):
|
|||
return validate_chat_completion_user_messages(messages=new_messages)
|
||||
|
||||
|
||||
def validate_and_fix_openai_tools(tools: list | None) -> list[dict] | None:
|
||||
def validate_and_fix_openai_tools(tools: list | None) -> list[dict[str, object]] | None:
|
||||
"""
|
||||
Ensure tools is List[dict] and not List[BaseModel]
|
||||
"""
|
||||
new_tools: Final = []
|
||||
new_tools: Final[list[dict[str, object]]] = []
|
||||
if tools is None:
|
||||
return tools
|
||||
for tool in tools:
|
||||
|
|
@ -9343,9 +9343,9 @@ def validate_chat_completion_user_messages(messages: list[AllMessageValues]):
|
|||
|
||||
|
||||
def validate_chat_completion_tool_choice(
|
||||
tool_choice: dict | str | None,
|
||||
tool_choice: dict[str, object] | str | None,
|
||||
model: str = "",
|
||||
) -> dict | str | None:
|
||||
) -> dict[str, object] | str | None:
|
||||
"""
|
||||
Confirm the tool choice is passed in the OpenAI format.
|
||||
|
||||
|
|
@ -9376,7 +9376,7 @@ def validate_chat_completion_tool_choice(
|
|||
)
|
||||
|
||||
|
||||
def validate_openai_optional_params(stop: str | list[str] | None = None, **kwargs) -> str | list[str] | None:
|
||||
def validate_openai_optional_params(stop: str | list[str] | None = None, **kwargs: object) -> str | list[str] | None:
|
||||
"""
|
||||
Validates and fixes OpenAI optional parameters.
|
||||
|
||||
|
|
@ -11426,8 +11426,8 @@ def reasoning_effort_with_summary_alias(
|
|||
|
||||
|
||||
def strip_reasoning_summary_aliases_from_optional_params(
|
||||
optional_params: dict,
|
||||
) -> tuple[dict, object | None]:
|
||||
optional_params: Mapping[str, object],
|
||||
) -> tuple[dict[str, object], object | None]:
|
||||
"""Copy optional_params; remove reasoningSummary aliases from top-level and extra_body."""
|
||||
op: Final = dict(optional_params)
|
||||
rs_val = op.pop("reasoningSummary", None)
|
||||
|
|
|
|||
58
scripts/check_native_imports.py
Normal file
58
scripts/check_native_imports.py
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
import ast
|
||||
import sys
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
PACKAGE: Final = Path("litellm/native")
|
||||
FORBIDDEN: Final = "litellm.main"
|
||||
|
||||
|
||||
def _is_forbidden(module: str) -> bool:
|
||||
return module == FORBIDDEN or module.startswith(f"{FORBIDDEN}.")
|
||||
|
||||
|
||||
def _from_import_targets(package: tuple[str, ...], node: ast.ImportFrom) -> Iterator[str]:
|
||||
base: Final = package[: len(package) - node.level + 1] if node.level else ()
|
||||
module: Final = ".".join((*base, node.module) if node.module else base)
|
||||
yield module
|
||||
yield from (f"{module}.{alias.name}" for alias in node.names)
|
||||
|
||||
|
||||
def _imported(package: tuple[str, ...], node: ast.AST) -> Iterator[str]:
|
||||
match node:
|
||||
case ast.Import(names=names):
|
||||
yield from (alias.name for alias in names)
|
||||
case ast.ImportFrom():
|
||||
yield from _from_import_targets(package, node)
|
||||
case ast.Attribute(value=ast.Name(id="litellm"), attr=attr):
|
||||
yield f"litellm.{attr}"
|
||||
case _:
|
||||
return
|
||||
|
||||
|
||||
def violations(path: Path, source: str) -> tuple[str, ...]:
|
||||
package: Final = (
|
||||
("litellm", "native", *path.relative_to(PACKAGE).parent.parts) if path.is_relative_to(PACKAGE) else ()
|
||||
)
|
||||
tree: Final = ast.parse(source, filename=str(path))
|
||||
return tuple(
|
||||
f"{path}:{node.lineno}: litellm.native must not depend on litellm.main"
|
||||
for node in ast.walk(tree)
|
||||
if any(_is_forbidden(target) for target in _imported(package, node))
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
findings: Final = tuple(
|
||||
finding for path in sorted(PACKAGE.rglob("*.py")) for finding in violations(path, path.read_text())
|
||||
)
|
||||
if findings:
|
||||
sys.stderr.write("\n".join(findings) + "\n")
|
||||
return 1
|
||||
sys.stdout.write("litellm.native imports: passed\n")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
|
@ -107,6 +107,9 @@ litellm_py_files=$(scope_match "$litellm_py_pattern")
|
|||
if [ -n "$(scope_match '^(litellm/proxy/mcp_server/|scripts/check_mcp_operation_boundary\.py)')" ]; then
|
||||
uv run --no-sync python scripts/check_mcp_operation_boundary.py || exit 1
|
||||
fi
|
||||
if [ -n "$(scope_match '^(litellm/native/|scripts/check_native_imports\.py)')" ]; then
|
||||
uv run --no-sync python scripts/check_native_imports.py || exit 1
|
||||
fi
|
||||
e2e_py_files=$(scope_match "$e2e_py_pattern")
|
||||
test_tree_files=$(scope_match "$test_tree_pattern")
|
||||
# ruff format (and CI's format step) skip enterprise; the rest of make lint covers it.
|
||||
|
|
|
|||
|
|
@ -1,15 +1,26 @@
|
|||
import asyncio
|
||||
import json
|
||||
import math
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import datetime
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
import respx
|
||||
from openai.types.chat import ChatCompletion
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.caching.caching import Cache
|
||||
from litellm.completion_extras.litellm_responses_transformation.handler import (
|
||||
ResponsesToCompletionBridgeHandler,
|
||||
)
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.responses import bridge_executor
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
|
|
@ -41,6 +52,11 @@ def test_is_preformatted_cached_chat_stream_false_wrong_type():
|
|||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _cache_on(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "cache", MagicMock(spec=Cache))
|
||||
|
||||
|
||||
def _bridge_kwargs(stream: bool):
|
||||
logging_obj = LiteLLMLogging(
|
||||
litellm_call_id="test-call",
|
||||
|
|
@ -63,7 +79,7 @@ def _bridge_kwargs(stream: bool):
|
|||
}
|
||||
|
||||
|
||||
def test_completion_returns_cached_model_response_directly():
|
||||
def test_completion_returns_cached_model_response_directly(_cache_on):
|
||||
"""Non-streaming bridge cache hit: responses() returns a ModelResponse -> bridge returns it as-is."""
|
||||
cached = ModelResponse(id="chatcmpl-cached-nonstream", model="gpt-5.4")
|
||||
bridge = ResponsesToCompletionBridgeHandler()
|
||||
|
|
@ -82,7 +98,7 @@ def test_completion_returns_cached_model_response_directly():
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_acompletion_returns_cached_model_response_directly():
|
||||
async def test_acompletion_returns_cached_model_response_directly(_cache_on):
|
||||
cached = ModelResponse(id="chatcmpl-cached-nonstream-async", model="gpt-5.4")
|
||||
bridge = ResponsesToCompletionBridgeHandler()
|
||||
|
||||
|
|
@ -99,7 +115,7 @@ async def test_acompletion_returns_cached_model_response_directly():
|
|||
assert result is cached
|
||||
|
||||
|
||||
def test_completion_skips_rewrapping_preformatted_cached_chat_stream():
|
||||
def test_completion_skips_rewrapping_preformatted_cached_chat_stream(_cache_on):
|
||||
"""Streaming bridge cache hit returning CustomStreamWrapper(cached_response) -> bridge skips re-wrapping."""
|
||||
stream = MagicMock(spec=CustomStreamWrapper)
|
||||
stream.custom_llm_provider = "cached_response"
|
||||
|
|
@ -124,7 +140,7 @@ def test_completion_skips_rewrapping_preformatted_cached_chat_stream():
|
|||
assert result is stream
|
||||
|
||||
|
||||
def test_completion_preserves_top_level_stream_flag_in_responses_request():
|
||||
def test_completion_preserves_top_level_stream_flag_in_responses_request(_cache_on):
|
||||
stream = MagicMock(spec=CustomStreamWrapper)
|
||||
stream.custom_llm_provider = "cached_response"
|
||||
bridge = ResponsesToCompletionBridgeHandler()
|
||||
|
|
@ -152,7 +168,7 @@ def test_completion_preserves_top_level_stream_flag_in_responses_request():
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_acompletion_skips_rewrapping_preformatted_cached_chat_stream():
|
||||
async def test_acompletion_skips_rewrapping_preformatted_cached_chat_stream(_cache_on):
|
||||
stream = MagicMock(spec=CustomStreamWrapper)
|
||||
stream.custom_llm_provider = "cached_response"
|
||||
bridge = ResponsesToCompletionBridgeHandler()
|
||||
|
|
@ -177,7 +193,7 @@ async def test_acompletion_skips_rewrapping_preformatted_cached_chat_stream():
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_acompletion_preserves_top_level_stream_flag_in_responses_request():
|
||||
async def test_acompletion_preserves_top_level_stream_flag_in_responses_request(_cache_on):
|
||||
stream = MagicMock(spec=CustomStreamWrapper)
|
||||
stream.custom_llm_provider = "cached_response"
|
||||
bridge = ResponsesToCompletionBridgeHandler()
|
||||
|
|
@ -219,7 +235,7 @@ def _completed_chat_response() -> ModelResponse:
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_acompletion_streams_completed_model_response():
|
||||
async def test_acompletion_streams_completed_model_response(_cache_on):
|
||||
"""A streaming request whose bridge call comes back already completed must still be
|
||||
handed back as an async-iterable stream. Returning the bare ModelResponse crashed the
|
||||
proxy's SSE generator with "'async for' requires an object with __aiter__ method".
|
||||
|
|
@ -245,7 +261,7 @@ async def test_acompletion_streams_completed_model_response():
|
|||
assert [c for c in chunks if c.choices[0].finish_reason], "stream never emitted a finish_reason"
|
||||
|
||||
|
||||
def test_completion_streams_completed_model_response():
|
||||
def test_completion_streams_completed_model_response(_cache_on):
|
||||
completed = _completed_chat_response()
|
||||
bridge = ResponsesToCompletionBridgeHandler()
|
||||
|
||||
|
|
@ -288,7 +304,7 @@ def _upstream_model_for(handed_model: str, custom_llm_provider: str) -> str:
|
|||
def test_completion_keeps_provider_native_model_id_through_responses(
|
||||
custom_llm_provider, bridge_model, expected_upstream_model
|
||||
):
|
||||
"""responses() resolves the provider itself, so the bridge must not hand it an already-stripped model."""
|
||||
"""The Responses executor resolves the provider itself, so the bridge must not hand it an already-stripped model."""
|
||||
cached = ModelResponse(id="chatcmpl-cached", model=bridge_model)
|
||||
bridge = ResponsesToCompletionBridgeHandler()
|
||||
kwargs = _bridge_kwargs(stream=False)
|
||||
|
|
@ -301,11 +317,11 @@ def test_completion_keeps_provider_native_model_id_through_responses(
|
|||
"transform_request",
|
||||
return_value={"model": bridge_model, "input": "hi"},
|
||||
),
|
||||
patch("litellm.responses", return_value=cached) as responses_call,
|
||||
patch.object(bridge_executor, "native_responses", return_value=cached) as responses_call,
|
||||
):
|
||||
bridge.completion(**kwargs)
|
||||
|
||||
handed_model = responses_call.call_args.kwargs["model"]
|
||||
handed_model = responses_call.call_args.args[1]
|
||||
assert _upstream_model_for(handed_model, custom_llm_provider) == expected_upstream_model
|
||||
|
||||
|
||||
|
|
@ -329,9 +345,306 @@ async def test_acompletion_keeps_provider_native_model_id_through_responses(
|
|||
"transform_request",
|
||||
return_value={"model": bridge_model, "input": "hi"},
|
||||
),
|
||||
patch("litellm.aresponses", new=AsyncMock(return_value=cached)) as responses_call,
|
||||
patch.object(bridge_executor, "anative_responses", new=AsyncMock(return_value=cached)) as responses_call,
|
||||
):
|
||||
await bridge.acompletion(**kwargs)
|
||||
|
||||
handed_model = responses_call.call_args.kwargs["model"]
|
||||
handed_model = responses_call.call_args.args[1]
|
||||
assert _upstream_model_for(handed_model, custom_llm_provider) == expected_upstream_model
|
||||
|
||||
|
||||
class _DeploymentHookCounter(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.calls = 0
|
||||
|
||||
async def async_pre_call_deployment_hook(self, kwargs, call_type) -> None:
|
||||
self.calls += 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridged_chat_call_runs_deployment_hooks_once(monkeypatch):
|
||||
counter = _DeploymentHookCounter()
|
||||
monkeypatch.setattr(litellm, "callbacks", [counter])
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
litellm.register_model(
|
||||
{"openai/bridge-only-model": {"litellm_provider": "openai", "mode": "responses", "max_tokens": 100}}
|
||||
)
|
||||
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
responses_route = upstream.post("https://example.invalid/v1/responses").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "resp_bridge",
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"status": "completed",
|
||||
"model": "bridge-only-model",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_bridge",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "from upstream", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
|
||||
},
|
||||
)
|
||||
)
|
||||
response = await litellm.acompletion(
|
||||
model="openai/bridge-only-model",
|
||||
api_base="https://example.invalid/v1",
|
||||
api_key="x",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
|
||||
assert response.choices[0].message.content == "from upstream"
|
||||
assert responses_route.call_count == 1
|
||||
assert counter.calls == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bridge_model, tools",
|
||||
[
|
||||
("gpt-5.4", [{"type": "mcp", "server_label": "docs", "server_url": "litellm_proxy"}]),
|
||||
("gpt-5.4", [{"type": "file_search", "vector_store_ids": ["vs_1"]}]),
|
||||
("chat_completions/gpt-5.4", None),
|
||||
],
|
||||
)
|
||||
def test_completion_keeps_responses_surface_for_what_only_it_serves(bridge_model, tools):
|
||||
cached = ModelResponse(id="chatcmpl-surface", model="gpt-5.4")
|
||||
bridge = ResponsesToCompletionBridgeHandler()
|
||||
kwargs = _bridge_kwargs(stream=False)
|
||||
kwargs["model"] = bridge_model
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
bridge.transformation_handler,
|
||||
"transform_request",
|
||||
return_value={"model": bridge_model, "input": "hi", "tools": tools},
|
||||
),
|
||||
patch("litellm.responses", return_value=cached) as responses_call,
|
||||
patch.object(bridge_executor, "native_responses") as executor_call,
|
||||
):
|
||||
result = bridge.completion(**kwargs)
|
||||
|
||||
assert result is cached
|
||||
assert responses_call.call_count == 1
|
||||
assert executor_call.call_count == 0
|
||||
|
||||
|
||||
_LOGGED_ROW: Final = TypeAdapter(dict[str, object])
|
||||
_LOGGED_CHAT: Final = TypeAdapter(ChatCompletion)
|
||||
|
||||
|
||||
class _SuccessRowRecorder(CustomLogger):
|
||||
rows: tuple[dict[str, object], ...] = ()
|
||||
|
||||
async def async_log_success_event(
|
||||
self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime
|
||||
) -> None:
|
||||
self.rows = (*self.rows, _LOGGED_ROW.validate_python(kwargs["standard_logging_object"]))
|
||||
|
||||
|
||||
def _responses_sse(events: Sequence[Mapping[str, object]]) -> bytes:
|
||||
return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events).encode()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridged_chat_stream_logs_one_chat_row_billed_from_provider_usage(monkeypatch: pytest.MonkeyPatch):
|
||||
recorder: Final = _SuccessRowRecorder()
|
||||
monkeypatch.setattr(litellm, "callbacks", [recorder])
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
prices: Final = {"input_cost_per_token": 1e-6, "output_cost_per_token": 4e-6, "cache_read_input_token_cost": 1e-7}
|
||||
litellm.register_model(
|
||||
{
|
||||
"openai/bridge-stream-model": {
|
||||
"litellm_provider": "openai",
|
||||
"mode": "responses",
|
||||
"max_tokens": 100,
|
||||
**prices,
|
||||
}
|
||||
}
|
||||
)
|
||||
usage: Final[dict[str, object]] = {
|
||||
"input_tokens": 100,
|
||||
"input_tokens_details": {"cached_tokens": 40},
|
||||
"output_tokens": 20,
|
||||
"output_tokens_details": {"reasoning_tokens": 5},
|
||||
"total_tokens": 120,
|
||||
}
|
||||
message: Final[dict[str, object]] = {
|
||||
"type": "message",
|
||||
"id": "msg_stream",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "streamed", "annotations": []}],
|
||||
}
|
||||
response: Final[dict[str, object]] = {
|
||||
"id": "resp_stream",
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"model": "bridge-stream-model",
|
||||
"output": [],
|
||||
"parallel_tool_calls": True,
|
||||
"tool_choice": "auto",
|
||||
"tools": [],
|
||||
}
|
||||
sse: Final = _responses_sse(
|
||||
[
|
||||
{"type": "response.created", "sequence_number": 0, "response": {**response, "status": "in_progress"}},
|
||||
{"type": "response.output_item.added", "sequence_number": 1, "output_index": 0, "item": message},
|
||||
{
|
||||
"type": "response.output_text.delta",
|
||||
"sequence_number": 2,
|
||||
"item_id": "msg_stream",
|
||||
"output_index": 0,
|
||||
"content_index": 0,
|
||||
"delta": "streamed",
|
||||
},
|
||||
{"type": "response.output_item.done", "sequence_number": 3, "output_index": 0, "item": message},
|
||||
{
|
||||
"type": "response.completed",
|
||||
"sequence_number": 4,
|
||||
"response": {**response, "status": "completed", "output": [message], "usage": usage},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
responses_route = upstream.post("https://example.invalid/v1/responses").mock(
|
||||
return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, content=sse)
|
||||
)
|
||||
stream = await litellm.acompletion(
|
||||
model="openai/bridge-stream-model",
|
||||
api_base="https://example.invalid/v1",
|
||||
api_key="x",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
)
|
||||
assert isinstance(stream, CustomStreamWrapper)
|
||||
text = "".join([chunk.choices[0].delta.content or "" async for chunk in stream if chunk.choices])
|
||||
for _ in range(100):
|
||||
if recorder.rows:
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
assert text == "streamed"
|
||||
assert responses_route.call_count == 1
|
||||
assert len(recorder.rows) == 1
|
||||
row: Final = recorder.rows[0]
|
||||
logged: Final = _LOGGED_CHAT.validate_python(row["response"])
|
||||
assert logged.object == "chat.completion"
|
||||
assert logged.choices[0].message.content == "streamed"
|
||||
assert row["prompt_tokens"] == 100
|
||||
assert row["completion_tokens"] == 20
|
||||
cost: Final = row["response_cost"]
|
||||
assert isinstance(cost, float)
|
||||
assert math.isclose(
|
||||
cost,
|
||||
60 * prices["input_cost_per_token"]
|
||||
+ 40 * prices["cache_read_input_token_cost"]
|
||||
+ 20 * prices["output_cost_per_token"],
|
||||
)
|
||||
|
||||
|
||||
_BUILT_IN_TOOL_CALLS: Final[Mapping[str, tuple[Sequence[Mapping[str, object]], Sequence[Mapping[str, object]]]]] = {
|
||||
"web_search": (
|
||||
tuple(
|
||||
{
|
||||
"type": "web_search_call",
|
||||
"id": f"ws_{i}",
|
||||
"status": "completed",
|
||||
"action": {"type": "search", "query": "q"},
|
||||
}
|
||||
for i in range(3)
|
||||
),
|
||||
(),
|
||||
),
|
||||
"image_generation": (
|
||||
(
|
||||
{
|
||||
"type": "image_generation_call",
|
||||
"id": "ig_1",
|
||||
"status": "completed",
|
||||
"result": "aGk=",
|
||||
"quality": "low",
|
||||
"size": "1024x1024",
|
||||
},
|
||||
),
|
||||
({"type": "image_generation", "model": "gpt-image-1"},),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _responses_body_with(tool_call: str) -> dict[str, object]:
|
||||
output_items, tools = _BUILT_IN_TOOL_CALLS[tool_call]
|
||||
message: Final[dict[str, object]] = {
|
||||
"type": "message",
|
||||
"id": "msg_tool",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "done", "annotations": []}],
|
||||
}
|
||||
return {
|
||||
"id": "resp_tool",
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"status": "completed",
|
||||
"model": "gpt-4.1",
|
||||
"output": [*output_items, message],
|
||||
"parallel_tool_calls": True,
|
||||
"tool_choice": "auto",
|
||||
"tools": list(tools),
|
||||
"usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
|
||||
}
|
||||
|
||||
|
||||
async def _rows_for(recorder: _SuccessRowRecorder, count: int) -> tuple[dict[str, object], ...]:
|
||||
for _ in range(200):
|
||||
if len(recorder.rows) >= count:
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
return recorder.rows
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("tool_call", sorted(_BUILT_IN_TOOL_CALLS))
|
||||
async def test_bridged_chat_call_is_billed_for_the_built_in_tool_calls_the_provider_made(
|
||||
monkeypatch: pytest.MonkeyPatch, tool_call: str
|
||||
):
|
||||
recorder: Final = _SuccessRowRecorder()
|
||||
monkeypatch.setattr(litellm, "callbacks", [recorder])
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
upstream.post("https://example.invalid/v1/responses").mock(
|
||||
return_value=httpx.Response(200, json=_responses_body_with(tool_call))
|
||||
)
|
||||
bridged = await litellm.acompletion(
|
||||
model="openai/responses/gpt-4.1",
|
||||
api_base="https://example.invalid/v1",
|
||||
api_key="x",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
await _rows_for(recorder, 1)
|
||||
await litellm.aresponses(model="openai/gpt-4.1", api_base="https://example.invalid/v1", api_key="x", input="hi")
|
||||
rows: Final = await _rows_for(recorder, 2)
|
||||
|
||||
bridged_row, native_row = rows
|
||||
assert bridged_row["call_type"] == "acompletion"
|
||||
assert native_row["call_type"] == "aresponses"
|
||||
native_cost: Final = native_row["response_cost"]
|
||||
assert isinstance(native_cost, float)
|
||||
assert bridged_row["response_cost"] == pytest.approx(native_cost)
|
||||
assert bridged._hidden_params["response_cost"] == pytest.approx(native_cost)
|
||||
breakdown: Final = _LOGGED_ROW.validate_python(bridged_row["cost_breakdown"])
|
||||
tool_usage_cost: Final = breakdown["tool_usage_cost"]
|
||||
assert isinstance(tool_usage_cost, float) and tool_usage_cost > 0
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from litellm.completion_extras.litellm_responses_transformation.handler import (
|
|||
ResponsesToCompletionBridgeHandler,
|
||||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.responses import bridge_executor
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
MODEL = "openai.gpt-5.5"
|
||||
|
|
@ -73,11 +74,11 @@ def test_completion_forwards_custom_llm_provider_to_responses():
|
|||
bridge = ResponsesToCompletionBridgeHandler()
|
||||
cached = ModelResponse(id="chatcmpl-cached", model="gpt-5.5")
|
||||
|
||||
with patch("litellm.responses", return_value=cached) as fake_responses:
|
||||
with patch.object(bridge_executor, "native_responses", return_value=cached) as fake_responses:
|
||||
result = bridge.completion(**_openai_kwargs())
|
||||
|
||||
assert result is cached
|
||||
assert fake_responses.call_args.kwargs["custom_llm_provider"] == "openai"
|
||||
assert fake_responses.call_args.args[2]["custom_llm_provider"] == "openai"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -85,13 +86,13 @@ async def test_acompletion_forwards_custom_llm_provider_to_aresponses():
|
|||
bridge = ResponsesToCompletionBridgeHandler()
|
||||
cached = ModelResponse(id="chatcmpl-cached", model="gpt-5.5")
|
||||
|
||||
async def _fake_aresponses(**kwargs):
|
||||
_fake_aresponses.kwargs = kwargs
|
||||
async def _fake_aresponses(request, model, forwarded_kwargs, logging_obj):
|
||||
_fake_aresponses.kwargs = forwarded_kwargs
|
||||
return cached
|
||||
|
||||
_fake_aresponses.kwargs = {}
|
||||
|
||||
with patch("litellm.aresponses", _fake_aresponses):
|
||||
with patch.object(bridge_executor, "anative_responses", _fake_aresponses):
|
||||
result = await bridge.acompletion(**_openai_kwargs())
|
||||
|
||||
assert result is cached
|
||||
|
|
@ -103,13 +104,13 @@ async def test_acompletion_forwards_aws_region_name_to_aresponses():
|
|||
bridge = ResponsesToCompletionBridgeHandler()
|
||||
cached = ModelResponse(id="chatcmpl-cached", model=MODEL)
|
||||
|
||||
async def _fake_aresponses(**kwargs):
|
||||
_fake_aresponses.kwargs = kwargs
|
||||
async def _fake_aresponses(request, model, forwarded_kwargs, logging_obj):
|
||||
_fake_aresponses.kwargs = forwarded_kwargs
|
||||
return cached
|
||||
|
||||
_fake_aresponses.kwargs = {}
|
||||
|
||||
with patch("litellm.aresponses", _fake_aresponses):
|
||||
with patch.object(bridge_executor, "anative_responses", _fake_aresponses):
|
||||
result = await bridge.acompletion(**_bedrock_mantle_kwargs())
|
||||
|
||||
assert result is cached
|
||||
|
|
|
|||
|
|
@ -31,11 +31,11 @@ def _funnel_kwargs_completion_forwards(monkeypatch, wif_kwargs: dict[str, object
|
|||
from unittest.mock import MagicMock
|
||||
|
||||
import litellm
|
||||
import litellm.main as litellm_main
|
||||
import litellm.native.chat as native_chat
|
||||
|
||||
spy = MagicMock(wraps=litellm_main.get_litellm_params)
|
||||
spy = MagicMock(wraps=native_chat.get_litellm_params)
|
||||
monkeypatch.setattr( # test-quality-ok: completion() has no injection seam for its kwargs funnel
|
||||
litellm_main, "get_litellm_params", spy
|
||||
native_chat, "get_litellm_params", spy
|
||||
)
|
||||
litellm.completion(
|
||||
model="anthropic/claude-sonnet-5",
|
||||
|
|
|
|||
|
|
@ -66,6 +66,14 @@ def test_translate_openai_response_to_anthropic_empty_choices() -> None:
|
|||
assert result["usage"]["input_tokens"] == 10
|
||||
|
||||
|
||||
def test_translate_openai_response_to_anthropic_without_usage_reports_zero_tokens() -> None:
|
||||
response: Final = ModelResponse(choices=[Choices(message=Message(content="x"), finish_reason="stop")])
|
||||
|
||||
result: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response)
|
||||
|
||||
assert (result["usage"]["input_tokens"], result["usage"]["output_tokens"]) == (0, 0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text,count,expected_stop", [
|
||||
("", 1, "compaction"), (None, 1, "compaction"), ("Answer", 1, "max_tokens"),
|
||||
(" ", 1, "max_tokens"), ("", 2, "max_tokens"), ("", 0, "max_tokens"),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,380 @@
|
|||
import asyncio
|
||||
import datetime
|
||||
from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeAlias, cast
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.custom_prompt_management import CustomPromptManagement
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.llms.anthropic.messages.adapters.handler import LiteLLMMessagesToCompletionTransformationHandler
|
||||
from litellm.llms.anthropic.messages.adapters.streaming_iterator import AnthropicSSEStream
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.prompts.init_prompts import PromptSpec
|
||||
from litellm.types.utils import StandardCallbackDynamicParams
|
||||
|
||||
API_BASE: Final = "http://chat-upstream.test/v1"
|
||||
CHAT_RESPONSE: Final = {
|
||||
"id": "chatcmpl-bridge",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "served-model",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "from upstream"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
|
||||
}
|
||||
|
||||
|
||||
class _DeploymentHookCounter(CustomLogger):
|
||||
pre_call: int = 0
|
||||
post_call: int = 0
|
||||
|
||||
async def async_pre_call_deployment_hook(self, kwargs: Mapping[str, object], call_type: object) -> None:
|
||||
self.pre_call += 1
|
||||
|
||||
async def async_post_call_success_deployment_hook(
|
||||
self, request_data: Mapping[str, object], response: object, call_type: object
|
||||
) -> None:
|
||||
self.post_call += 1
|
||||
|
||||
|
||||
_PromptResult: TypeAlias = tuple[str, list[AllMessageValues], dict[str, object]]
|
||||
|
||||
|
||||
class _TemplatePromptManager(CustomPromptManagement):
|
||||
async def async_get_chat_completion_prompt(
|
||||
self,
|
||||
model: str,
|
||||
messages: Sequence[AllMessageValues],
|
||||
non_default_params: Mapping[str, object],
|
||||
prompt_id: str | None,
|
||||
prompt_variables: Mapping[str, object] | None,
|
||||
dynamic_callback_params: StandardCallbackDynamicParams,
|
||||
litellm_logging_obj: Logging,
|
||||
prompt_spec: PromptSpec | None = None,
|
||||
tools: Sequence[Mapping[str, object]] | None = None,
|
||||
prompt_label: str | None = None,
|
||||
prompt_version: int | None = None,
|
||||
ignore_prompt_manager_model: bool | None = False,
|
||||
ignore_prompt_manager_optional_params: bool | None = False,
|
||||
) -> _PromptResult:
|
||||
return model, [{"role": "user", "content": "from the template"}], {**non_default_params}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def upstream(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch) -> respx.Route:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
return respx_mock.post(f"{API_BASE}/chat/completions").mock(return_value=httpx.Response(200, json=CHAT_RESPONSE))
|
||||
|
||||
|
||||
async def _ask(extra: Mapping[str, object] = MappingProxyType({})) -> object:
|
||||
return await litellm.anthropic.messages.acreate(
|
||||
model="hosted_vllm/served-model",
|
||||
api_base=API_BASE,
|
||||
api_key="sk-chat",
|
||||
max_tokens=10,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
**extra,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridge_runs_deployment_hooks_once_per_request(upstream: respx.Route, monkeypatch: pytest.MonkeyPatch):
|
||||
counter: Final = _DeploymentHookCounter()
|
||||
monkeypatch.setattr(litellm, "callbacks", [counter])
|
||||
|
||||
response: Final = await _ask()
|
||||
|
||||
assert isinstance(response, dict)
|
||||
assert response["content"] == [{"type": "text", "text": "from upstream"}]
|
||||
assert upstream.call_count == 1
|
||||
assert (counter.pre_call, counter.post_call) == (1, 1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridge_streams_the_upstream_reply(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
chunk: Final = '{"id":"c","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{"role":"assistant","content":"streamed"},"finish_reason":null}]}'
|
||||
done: Final = '{"id":"c","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}'
|
||||
respx_mock.post(f"{API_BASE}/chat/completions").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
content=f"data: {chunk}\n\ndata: {done}\n\ndata: [DONE]\n\n".encode(),
|
||||
headers={"content-type": "text/event-stream"},
|
||||
)
|
||||
)
|
||||
|
||||
stream: Final = await _ask({"stream": True})
|
||||
body: Final = b"".join([event async for event in stream]).decode()
|
||||
|
||||
assert "event: message_start" in body
|
||||
assert '"text":"streamed"' in body.replace(" ", "")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridge_applies_the_prompt_template_named_by_prompt_id(
|
||||
upstream: respx.Route, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
monkeypatch.setattr(litellm, "callbacks", [_TemplatePromptManager()])
|
||||
|
||||
await _ask({"prompt_id": "bridge_template"})
|
||||
|
||||
assert "from the template" in upstream.calls.last.request.content.decode()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridge_maps_an_upstream_429_to_rate_limit_error(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
respx_mock.post(f"{API_BASE}/chat/completions").mock(
|
||||
return_value=httpx.Response(429, json={"error": {"message": "slow down", "type": "rate_limit"}})
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.RateLimitError):
|
||||
await _ask()
|
||||
|
||||
|
||||
def _sync_bridge(logging_obj: Logging, *, stream: bool) -> object:
|
||||
return LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler(
|
||||
max_tokens=10,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
model="hosted_vllm/served-model",
|
||||
stream=stream,
|
||||
api_base=API_BASE,
|
||||
api_key="sk-chat",
|
||||
litellm_logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
|
||||
def test_sync_bridge_answers_from_one_upstream_call(upstream: respx.Route):
|
||||
logging_obj: Final = Logging(
|
||||
model="hosted_vllm/served-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=False,
|
||||
call_type="anthropic_messages",
|
||||
start_time=datetime.datetime.now(datetime.timezone.utc),
|
||||
litellm_call_id="6825beef-0000-4000-8000-000000000005",
|
||||
function_id="1234",
|
||||
)
|
||||
|
||||
response: Final = _sync_bridge(logging_obj, stream=False)
|
||||
|
||||
assert isinstance(response, dict)
|
||||
assert response["content"] == [{"type": "text", "text": "from upstream"}]
|
||||
assert upstream.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridge_runs_prompt_management_triggered_without_prompt_id(
|
||||
upstream: respx.Route, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
monkeypatch.setattr(litellm, "callbacks", [_TemplatePromptManager()])
|
||||
|
||||
await _ask({"vector_store_ids": ["vs_bridge"]})
|
||||
|
||||
assert "from the template" in upstream.calls.last.request.content.decode()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridge_keeps_the_litellm_system_prompt(upstream: respx.Route):
|
||||
await _ask({"litellm_system_prompt": "answer like a pirate"})
|
||||
|
||||
assert "answer like a pirate" in upstream.calls.last.request.content.decode()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridge_goes_through_a_wrapped_acompletion(upstream: respx.Route, monkeypatch: pytest.MonkeyPatch):
|
||||
unwrapped: Final = litellm.acompletion
|
||||
seen: Final[list[object]] = [] # mutable-ok: records the calls the wrapper sees
|
||||
|
||||
async def traced_acompletion(*args: object, **kwargs: object) -> object:
|
||||
seen.append(kwargs.get("model"))
|
||||
return await unwrapped(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", traced_acompletion)
|
||||
|
||||
response: Final = await _ask()
|
||||
|
||||
assert isinstance(response, dict)
|
||||
assert response["content"] == [{"type": "text", "text": "from upstream"}]
|
||||
assert seen == ["hosted_vllm/served-model"]
|
||||
assert upstream.call_count == 1
|
||||
|
||||
|
||||
class _SuccessRowRecorder(CustomLogger):
|
||||
call_types: tuple[str, ...] = ()
|
||||
|
||||
async def async_log_success_event(
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
self.call_types = (*self.call_types, str(kwargs.get("call_type")))
|
||||
|
||||
def log_success_event(
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
self.call_types = (*self.call_types, str(kwargs.get("call_type")))
|
||||
|
||||
|
||||
_PARKED_COMPLETION: Final = TypeAdapter(tuple[object])
|
||||
|
||||
|
||||
async def _release(parked: Coroutine[object, object, object]) -> None:
|
||||
await parked
|
||||
|
||||
|
||||
async def _settled_rows(recorder: _SuccessRowRecorder) -> tuple[str, ...]:
|
||||
for _ in range(200):
|
||||
if recorder.call_types:
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
await asyncio.sleep(0.05)
|
||||
return recorder.call_types
|
||||
|
||||
|
||||
STREAM_CHUNK: Final = '{"id":"c","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{"role":"assistant","content":"streamed"},"finish_reason":null}]}'
|
||||
STREAM_DONE: Final = '{"id":"c","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}'
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def stream_recorder(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch) -> _SuccessRowRecorder:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
respx_mock.post(f"{API_BASE}/chat/completions").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
content=f"data: {STREAM_CHUNK}\n\ndata: {STREAM_DONE}\n\ndata: [DONE]\n\n".encode(),
|
||||
headers={"content-type": "text/event-stream"},
|
||||
)
|
||||
)
|
||||
recorder: Final = _SuccessRowRecorder()
|
||||
monkeypatch.setattr(litellm, "callbacks", [recorder])
|
||||
return recorder
|
||||
|
||||
|
||||
async def _read_through_message_stop(stream: AsyncIterator[bytes]) -> None:
|
||||
async for frame in stream:
|
||||
if frame.startswith(b"event: message_stop"):
|
||||
return
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridged_stream_logs_one_row_when_the_client_stream_ends(stream_recorder: _SuccessRowRecorder):
|
||||
stream: Final = await _ask({"stream": True})
|
||||
assert isinstance(stream, AnthropicSSEStream)
|
||||
|
||||
await _read_through_message_stop(stream)
|
||||
rows_while_the_client_stream_is_open: Final = await _settled_rows(stream_recorder)
|
||||
remaining: Final[list[bytes]] = [frame async for frame in stream]
|
||||
|
||||
assert rows_while_the_client_stream_is_open == ()
|
||||
assert remaining == []
|
||||
assert await _settled_rows(stream_recorder) == ("anthropic_messages",)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridged_stream_still_logs_when_the_client_leaves_after_message_stop(
|
||||
stream_recorder: _SuccessRowRecorder,
|
||||
):
|
||||
stream: Final = await _ask({"stream": True})
|
||||
assert isinstance(stream, AnthropicSSEStream)
|
||||
|
||||
await _read_through_message_stop(stream)
|
||||
await stream.aclose()
|
||||
|
||||
assert await _settled_rows(stream_recorder) == ("anthropic_messages",)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridged_stream_holds_its_row_for_the_proxy_end_of_stream_guardrails(
|
||||
stream_recorder: _SuccessRowRecorder, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
monkeypatch.setattr(litellm, "_async_success_callback", [stream_recorder])
|
||||
logging_obj: Final = Logging(
|
||||
model="hosted_vllm/served-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
call_type="anthropic_messages",
|
||||
start_time=datetime.datetime.now(datetime.timezone.utc),
|
||||
litellm_call_id="6825beef-0000-4000-8000-000000000007",
|
||||
function_id="1234",
|
||||
)
|
||||
|
||||
async def release(*held: object) -> None:
|
||||
return None
|
||||
|
||||
setattr(logging_obj, "_on_deferred_stream_complete", release) # noqa: B010 # the proxy arms this slot dynamically
|
||||
stream: Final = await LiteLLMMessagesToCompletionTransformationHandler.async_anthropic_messages_handler(
|
||||
max_tokens=10,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
model="hosted_vllm/served-model",
|
||||
stream=True,
|
||||
api_base=API_BASE,
|
||||
api_key="sk-chat",
|
||||
litellm_logging_obj=logging_obj,
|
||||
)
|
||||
assert isinstance(stream, AnthropicSSEStream)
|
||||
frames: Final = [frame async for frame in stream]
|
||||
rows_before_release: Final = await _settled_rows(stream_recorder)
|
||||
held: Final[object] = getattr(logging_obj, "_deferred_stream_complete_args", None)
|
||||
|
||||
assert frames[-1].startswith(b"event: message_stop")
|
||||
assert rows_before_release == ()
|
||||
(parked,) = _PARKED_COMPLETION.validate_python(held)
|
||||
assert asyncio.iscoroutine(parked)
|
||||
await _release(parked)
|
||||
assert stream_recorder.call_types == ("anthropic_messages",)
|
||||
|
||||
|
||||
def test_sync_bridged_stream_logs_one_row(stream_recorder: _SuccessRowRecorder, monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "success_callback", [stream_recorder])
|
||||
logging_obj: Final = Logging(
|
||||
model="hosted_vllm/served-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
call_type="anthropic_messages",
|
||||
start_time=datetime.datetime.now(datetime.timezone.utc),
|
||||
litellm_call_id="6825beef-0000-4000-8000-000000000006",
|
||||
function_id="1234",
|
||||
)
|
||||
|
||||
stream: Final = _sync_bridge(logging_obj, stream=True)
|
||||
assert isinstance(stream, Iterator)
|
||||
frames: Final = list(cast(Iterator[bytes], stream)) # cast-ok: the sync bridge streams SSE frames as bytes
|
||||
|
||||
assert frames[-1].startswith(b"event: message_stop")
|
||||
assert asyncio.run(_settled_rows(stream_recorder)) == ("anthropic_messages",)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridged_empty_stream_still_logs_one_row(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
respx_mock.post(f"{API_BASE}/chat/completions").mock(
|
||||
return_value=httpx.Response(200, content=b"data: [DONE]\n\n", headers={"content-type": "text/event-stream"})
|
||||
)
|
||||
recorder: Final = _SuccessRowRecorder()
|
||||
monkeypatch.setattr(litellm, "callbacks", [recorder])
|
||||
|
||||
# deepinfra sends no stream_options, so the empty stream assembles no response to log
|
||||
stream: Final = await litellm.anthropic.messages.acreate(
|
||||
model="deepinfra/meta-llama/Meta-Llama-3.1-8B-Instruct",
|
||||
api_base=API_BASE,
|
||||
api_key="sk-chat",
|
||||
max_tokens=10,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
)
|
||||
assert isinstance(stream, AnthropicSSEStream)
|
||||
frames: Final = [frame async for frame in stream]
|
||||
|
||||
assert frames[-1].startswith(b"event: message_stop")
|
||||
assert await _settled_rows(recorder) == ("anthropic_messages",)
|
||||
|
|
@ -19,9 +19,9 @@ import json
|
|||
import os
|
||||
import sys
|
||||
import threading
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from collections.abc import AsyncIterator, Callable, Iterator, Mapping, Sequence
|
||||
from datetime import datetime
|
||||
from typing import Final, Iterator, List, Optional
|
||||
from typing import Final, List, Optional
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import httpx
|
||||
|
|
@ -40,11 +40,12 @@ from litellm.llms.anthropic.messages.adapters.streaming_iterator import (
|
|||
AnthropicStreamWrapper,
|
||||
_mid_stream_error_sse_event,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.held_stream_completion import HeldStreamCompletion
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StandardLoggingPayload, StreamingChoices, Usage
|
||||
|
||||
|
||||
def _make_chunk(delta: Delta, finish_reason: Optional[str] = None) -> MagicMock:
|
||||
def _make_chunk(delta: Delta, finish_reason: str | None = None) -> MagicMock:
|
||||
chunk = MagicMock()
|
||||
chunk.choices = [
|
||||
StreamingChoices(finish_reason=finish_reason, index=0, delta=delta, logprobs=None)
|
||||
|
|
@ -78,7 +79,7 @@ def _parse_sse(raw: bytes) -> tuple[str, dict]:
|
|||
return event_line.removeprefix("event: "), json.loads(data_line.removeprefix("data: "))
|
||||
|
||||
|
||||
async def _drain_sse(wrapper: AnthropicStreamWrapper) -> List[bytes]:
|
||||
async def _drain_sse(wrapper: AnthropicStreamWrapper) -> list[bytes]:
|
||||
return [event async for event in wrapper.async_anthropic_sse_wrapper()]
|
||||
|
||||
|
||||
|
|
@ -385,3 +386,51 @@ async def test_mid_stream_error_emits_error_event_without_logging_obj():
|
|||
events = await _drain_sse(_failing_wrapper(None))
|
||||
|
||||
assert _parse_sse(events[-1])[0] == "error"
|
||||
|
||||
|
||||
class _AsyncSuccessRecorder(CustomLogger):
|
||||
calls: tuple[object, ...] = ()
|
||||
|
||||
async def async_log_success_event(
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
self.calls = (*self.calls, response_obj)
|
||||
|
||||
|
||||
class _CompletesThenRaises:
|
||||
def __init__(self, exc: Exception, held: HeldStreamCompletion) -> None:
|
||||
self._exc: Final = exc
|
||||
self._held: Final = held
|
||||
|
||||
def __iter__(self) -> Iterator[ModelResponseStream]:
|
||||
return iter(())
|
||||
|
||||
def __aiter__(self) -> AsyncIterator[ModelResponseStream]:
|
||||
return self._chunks()
|
||||
|
||||
async def _chunks(self) -> AsyncIterator[ModelResponseStream]:
|
||||
yield ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content="partial"))])
|
||||
self._held.take(litellm.ModelResponse())
|
||||
raise self._exc
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_stream_that_fails_after_the_executor_completed_logs_no_success_row():
|
||||
success_recorder: Final = _AsyncSuccessRecorder()
|
||||
async_recorder: Final = _AsyncFailureRecorder()
|
||||
logging_obj: Final = _make_logging_obj("fails-after-complete", async_recorder, _SyncFailureRecorder())
|
||||
logging_obj.dynamic_async_success_callbacks = [success_recorder]
|
||||
held: Final = HeldStreamCompletion(logging_obj)
|
||||
wrapper: Final = AnthropicStreamWrapper(
|
||||
completion_stream=_CompletesThenRaises(_bedrock_drop(), held),
|
||||
model="bedrock-converse-sonnet-4-6",
|
||||
litellm_logging_obj=logging_obj,
|
||||
held_completion=held,
|
||||
)
|
||||
|
||||
events: Final = await _drain_sse(wrapper)
|
||||
await asyncio.sleep(0.2)
|
||||
|
||||
assert _parse_sse(events[-1])[0] == "error"
|
||||
assert [str(exc) for exc in async_recorder.exceptions] == ["ConverseStream ended without messageStop"]
|
||||
assert success_recorder.calls == ()
|
||||
|
|
|
|||
|
|
@ -103,15 +103,18 @@ async def _mock_streaming_chunks():
|
|||
)
|
||||
|
||||
|
||||
_PLANNED_CHAT = "litellm.llms.anthropic.messages.adapters.handler.aexecute_planned_chat"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_polyfill_round_trip_non_streaming():
|
||||
captured = {}
|
||||
|
||||
async def fake_acompletion(**kwargs):
|
||||
captured.update(kwargs)
|
||||
async def fake_chat(plan):
|
||||
captured.update({**plan.kwargs, **plan.named})
|
||||
return _mock_completion_response()
|
||||
|
||||
with patch("litellm.acompletion", side_effect=fake_acompletion):
|
||||
with patch(_PLANNED_CHAT, side_effect=fake_chat):
|
||||
response = await litellm.anthropic.messages.acreate(
|
||||
model=MODEL,
|
||||
messages=_make_history(n_pairs=5),
|
||||
|
|
@ -173,11 +176,11 @@ async def test_polyfill_round_trip_non_streaming():
|
|||
async def test_polyfill_trigger_not_met_passes_through_unchanged():
|
||||
captured = {}
|
||||
|
||||
async def fake_acompletion(**kwargs):
|
||||
captured.update(kwargs)
|
||||
async def fake_chat(plan):
|
||||
captured.update({**plan.kwargs, **plan.named})
|
||||
return _mock_completion_response()
|
||||
|
||||
with patch("litellm.acompletion", side_effect=fake_acompletion):
|
||||
with patch(_PLANNED_CHAT, side_effect=fake_chat):
|
||||
response = await litellm.anthropic.messages.acreate(
|
||||
model=MODEL,
|
||||
messages=_make_history(n_pairs=2),
|
||||
|
|
@ -209,10 +212,10 @@ async def test_polyfill_trigger_not_met_passes_through_unchanged():
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_polyfill_streaming_attaches_to_message_delta():
|
||||
async def fake_acompletion(**kwargs):
|
||||
async def fake_chat(plan):
|
||||
return _mock_streaming_chunks()
|
||||
|
||||
with patch("litellm.acompletion", side_effect=fake_acompletion):
|
||||
with patch(_PLANNED_CHAT, side_effect=fake_chat):
|
||||
response = await litellm.anthropic.messages.acreate(
|
||||
model=MODEL,
|
||||
messages=_make_history(n_pairs=5),
|
||||
|
|
|
|||
|
|
@ -2,20 +2,20 @@ import asyncio
|
|||
import copy
|
||||
import json
|
||||
import os
|
||||
import unittest.mock
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Dict, Final, List
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import responses
|
||||
import respx
|
||||
import pytest
|
||||
import respx
|
||||
from fastapi.testclient import TestClient
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import litellm
|
||||
from litellm.anthropic_interface import messages
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
|
@ -30,7 +30,6 @@ from litellm.types.utils import (
|
|||
StreamingChoices,
|
||||
)
|
||||
from litellm.utils import _invalidate_model_cost_lowercase_map
|
||||
import unittest.mock
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -340,8 +339,10 @@ async def test_bedrock_converse_budget_tokens_preserved():
|
|||
usage={"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
|
||||
)
|
||||
|
||||
with patch("litellm.acompletion", new_callable=AsyncMock) as mock_acompletion:
|
||||
mock_acompletion.return_value = mock_response
|
||||
with patch(
|
||||
"litellm.llms.anthropic.messages.adapters.handler.aexecute_planned_chat", new_callable=AsyncMock
|
||||
) as mock_chat:
|
||||
mock_chat.return_value = mock_response
|
||||
|
||||
try:
|
||||
await messages.acreate(
|
||||
|
|
@ -353,13 +354,9 @@ async def test_bedrock_converse_budget_tokens_preserved():
|
|||
except (ValueError, TypeError, AttributeError):
|
||||
pass # Expected due to response format conversion
|
||||
|
||||
mock_acompletion.assert_called_once()
|
||||
mock_chat.assert_called_once()
|
||||
|
||||
call_kwargs = mock_acompletion.call_args.kwargs
|
||||
print("acompletion call kwargs: ", json.dumps(call_kwargs, indent=4, default=str))
|
||||
|
||||
# Verify thinking parameter is passed through with budget_tokens preserved
|
||||
thinking_param = call_kwargs.get("thinking")
|
||||
thinking_param = mock_chat.call_args.args[0].named.get("thinking")
|
||||
assert thinking_param is not None, "thinking parameter should be passed to acompletion"
|
||||
assert thinking_param.get("type") == "enabled", "thinking.type should be 'enabled'"
|
||||
assert thinking_param.get("budget_tokens") == 1024, (
|
||||
|
|
@ -1241,7 +1238,10 @@ async def test_messages_streaming_reports_provider_local_model(requested_model,
|
|||
async def __anext__(self):
|
||||
raise StopAsyncIteration
|
||||
|
||||
with patch("litellm.acompletion", new=AsyncMock(return_value=_EmptyStream())):
|
||||
with patch(
|
||||
"litellm.llms.anthropic.messages.adapters.handler.aexecute_planned_chat",
|
||||
new=AsyncMock(return_value=_EmptyStream()),
|
||||
):
|
||||
stream = await litellm.anthropic.messages.acreate(
|
||||
max_tokens=100,
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
|
|
@ -2455,3 +2455,81 @@ async def test_anthropic_messages_strict_tools_rejection_is_sent_again_without_s
|
|||
assert first["tools"] == [tool]
|
||||
assert second == {**first, "tools": [{"name": "lookup_record", "input_schema": {"type": "object"}}]}
|
||||
assert _JSON_BODY.validate_python(response)["id"] == "msg_strict_retry"
|
||||
|
||||
|
||||
_REASONING_RESPONSE: Final = {
|
||||
"id": "resp_thinking",
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"status": "completed",
|
||||
"model": "gpt-5.5",
|
||||
"output": [
|
||||
{"type": "reasoning", "id": "rs_1", "summary": [{"type": "summary_text", "text": "worked it out"}]},
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_1",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "text": "the answer", "annotations": []}],
|
||||
},
|
||||
],
|
||||
"parallel_tool_calls": True,
|
||||
"tool_choice": "auto",
|
||||
"tools": [],
|
||||
"usage": {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7},
|
||||
}
|
||||
|
||||
|
||||
class _DeploymentHookCounter(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.calls = 0
|
||||
|
||||
async def async_pre_call_deployment_hook(self, kwargs: Mapping[str, object], call_type: object) -> None:
|
||||
self.calls += 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_forced_chat_sends_openai_thinking_to_responses_through_the_chat_bridge(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
counter: Final = _DeploymentHookCounter()
|
||||
monkeypatch.setattr(litellm, "callbacks", [counter])
|
||||
monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", True)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
responses_route: Final = upstream.post("http://openai.test/v1/responses").mock(
|
||||
return_value=httpx.Response(200, json=_REASONING_RESPONSE)
|
||||
)
|
||||
chat_route: Final = upstream.post("http://openai.test/v1/chat/completions")
|
||||
response: Final = await litellm.anthropic.messages.acreate(
|
||||
model="openai/gpt-5.5",
|
||||
api_base="http://openai.test/v1",
|
||||
api_key="sk-test",
|
||||
max_tokens=1024,
|
||||
messages=[
|
||||
{"role": "user", "content": "weather?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "text", "text": "let me check"},
|
||||
{"type": "tool_use", "id": "toolu_1", "name": "get_w", "input": {"c": "a"}},
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "sunny"}]},
|
||||
],
|
||||
tools=[{"name": "get_w", "description": "d", "input_schema": {"type": "object", "properties": {}}}],
|
||||
thinking={"type": "enabled", "budget_tokens": 1024},
|
||||
)
|
||||
|
||||
assert isinstance(response, dict)
|
||||
assert [block["type"] for block in response["content"]] == ["thinking", "text"]
|
||||
assert response["content"][1]["text"] == "the answer"
|
||||
assert (responses_route.call_count, chat_route.call_count) == (1, 0)
|
||||
assert counter.calls == 1
|
||||
sent_input: Final = json.loads(responses_route.calls[0].request.content)["input"]
|
||||
assert [item["type"] for item in sent_input] == ["message", "message", "function_call", "function_call_output"]
|
||||
assert sent_input[1]["role"] == "assistant"
|
||||
assert sent_input[3]["output"] == [{"type": "input_text", "text": "sunny"}]
|
||||
|
|
|
|||
|
|
@ -32,13 +32,7 @@ def _call_handler_and_capture_optional_params(thinking=None, **extra_kwargs):
|
|||
with patch(
|
||||
"litellm.llms.anthropic.messages.native.handler."
|
||||
"base_llm_http_handler"
|
||||
) as mock_handler, patch(
|
||||
"litellm.llms.anthropic.messages.native.handler."
|
||||
"ProviderConfigManager"
|
||||
) as mock_pcm:
|
||||
# Make get_provider_anthropic_messages_config return a non-None config
|
||||
# so the handler takes the native Anthropic path
|
||||
mock_pcm.get_provider_anthropic_messages_config.return_value = MagicMock()
|
||||
) as mock_handler:
|
||||
mock_handler.anthropic_messages_handler.return_value = MagicMock()
|
||||
|
||||
kwargs = dict(extra_kwargs)
|
||||
|
|
|
|||
|
|
@ -1,30 +1,40 @@
|
|||
import asyncio
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import AsyncIterator, Mapping
|
||||
from collections.abc import AsyncIterator, Callable, Coroutine, Mapping
|
||||
from itertools import chain
|
||||
from typing import Final, cast
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../..")))
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.custom_prompt_management import CustomPromptManagement
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
TOOL_RESULT_IMAGE_BOUNDARY,
|
||||
TOOL_RESULT_IMAGE_PLACEHOLDER,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.responses_adapters.handler import (
|
||||
LiteLLMMessagesToResponsesAPIHandler,
|
||||
_build_responses_kwargs,
|
||||
_build_responses_request,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.responses_adapters.streaming_iterator import AnthropicResponsesSSEByteStream
|
||||
from litellm.router import Router
|
||||
from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS, converted_stream_requested
|
||||
from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.prompts.init_prompts import PromptSpec
|
||||
from litellm.types.utils import StandardCallbackDynamicParams
|
||||
|
||||
MESSAGES = [{"role": "user", "content": "hello"}]
|
||||
MESSAGES: Final[list[dict[str, object]]] = [{"role": "user", "content": "hello"}]
|
||||
CLAUDE_CODE_USER_ID = json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": "session-abc"})
|
||||
|
||||
RESPONSES_SSE_BODY = (
|
||||
|
|
@ -40,20 +50,8 @@ RESPONSES_SSE_BODY = (
|
|||
)
|
||||
|
||||
|
||||
def test_build_responses_kwargs_never_asks_the_nested_call_to_replay_a_stream():
|
||||
responses_kwargs: Final = _build_responses_kwargs(
|
||||
max_tokens=1024,
|
||||
messages=MESSAGES,
|
||||
model="azure/gpt-5.6-luna",
|
||||
extra_kwargs={"custom_llm_provider": "azure", **{key: True for key in CONVERTED_STREAM_KEYS}},
|
||||
)
|
||||
|
||||
assert not converted_stream_requested(responses_kwargs)
|
||||
assert responses_kwargs["custom_llm_provider"] == "azure"
|
||||
|
||||
|
||||
def test_build_responses_kwargs_derives_prompt_cache_key_from_claude_code_session_id():
|
||||
responses_kwargs = _build_responses_kwargs(
|
||||
def test_build_responses_request_derives_prompt_cache_key_from_claude_code_session_id():
|
||||
responses_kwargs = _build_responses_request(
|
||||
max_tokens=1024,
|
||||
messages=MESSAGES,
|
||||
model="openai/gpt-5.6-luna",
|
||||
|
|
@ -64,8 +62,8 @@ def test_build_responses_kwargs_derives_prompt_cache_key_from_claude_code_sessio
|
|||
assert responses_kwargs["prompt_cache_key"] == "session-abc"
|
||||
|
||||
|
||||
def test_build_responses_kwargs_sets_no_prompt_cache_key_for_plain_user_id():
|
||||
responses_kwargs = _build_responses_kwargs(
|
||||
def test_build_responses_request_sets_no_prompt_cache_key_for_plain_user_id():
|
||||
responses_kwargs = _build_responses_request(
|
||||
max_tokens=1024,
|
||||
messages=MESSAGES,
|
||||
model="openai/gpt-5.6-luna",
|
||||
|
|
@ -76,8 +74,8 @@ def test_build_responses_kwargs_sets_no_prompt_cache_key_for_plain_user_id():
|
|||
assert "prompt_cache_key" not in responses_kwargs
|
||||
|
||||
|
||||
def test_build_responses_kwargs_prefers_explicit_prompt_cache_key_over_derived():
|
||||
responses_kwargs = _build_responses_kwargs(
|
||||
def test_build_responses_request_prefers_explicit_prompt_cache_key_over_derived():
|
||||
responses_kwargs = _build_responses_request(
|
||||
max_tokens=1024,
|
||||
messages=MESSAGES,
|
||||
model="openai/gpt-5.6-luna",
|
||||
|
|
@ -88,8 +86,8 @@ def test_build_responses_kwargs_prefers_explicit_prompt_cache_key_over_derived()
|
|||
assert responses_kwargs["prompt_cache_key"] == "explicit-key"
|
||||
|
||||
|
||||
def test_build_responses_kwargs_asks_openai_for_encrypted_reasoning_without_thinking():
|
||||
responses_kwargs = _build_responses_kwargs(
|
||||
def test_build_responses_request_asks_openai_for_encrypted_reasoning_without_thinking():
|
||||
responses_kwargs = _build_responses_request(
|
||||
max_tokens=1024,
|
||||
messages=MESSAGES,
|
||||
model="openai/gpt-5.6-luna",
|
||||
|
|
@ -99,8 +97,8 @@ def test_build_responses_kwargs_asks_openai_for_encrypted_reasoning_without_thin
|
|||
assert "reasoning" not in responses_kwargs
|
||||
|
||||
|
||||
def test_build_responses_kwargs_skips_include_for_a_responses_provider_that_rejects_it():
|
||||
responses_kwargs = _build_responses_kwargs(
|
||||
def test_build_responses_request_skips_include_for_a_responses_provider_that_rejects_it():
|
||||
responses_kwargs = _build_responses_request(
|
||||
max_tokens=1024,
|
||||
messages=MESSAGES,
|
||||
model="perplexity/sonar",
|
||||
|
|
@ -118,11 +116,11 @@ def test_build_responses_kwargs_skips_include_for_a_responses_provider_that_reje
|
|||
("openai/gpt-5.6-luna", "openai", "detailed"),
|
||||
],
|
||||
)
|
||||
def test_build_responses_kwargs_auto_summary_asks_each_target_for_a_level_it_accepts(
|
||||
def test_build_responses_request_auto_summary_asks_each_target_for_a_level_it_accepts(
|
||||
monkeypatch: pytest.MonkeyPatch, model: str, provider: str, summary: str
|
||||
) -> None:
|
||||
monkeypatch.setenv("LITELLM_REASONING_AUTO_SUMMARY", "true")
|
||||
responses_kwargs: Final = _build_responses_kwargs(
|
||||
responses_kwargs: Final = _build_responses_request(
|
||||
max_tokens=1024,
|
||||
messages=MESSAGES,
|
||||
model=model,
|
||||
|
|
@ -132,8 +130,8 @@ def test_build_responses_kwargs_auto_summary_asks_each_target_for_a_level_it_acc
|
|||
assert cast(Mapping[str, object], responses_kwargs["reasoning"])["summary"] == summary
|
||||
|
||||
|
||||
def test_build_responses_kwargs_keeps_the_deployment_include_next_to_encrypted_reasoning():
|
||||
responses_kwargs = _build_responses_kwargs(
|
||||
def test_build_responses_request_keeps_the_deployment_include_next_to_encrypted_reasoning():
|
||||
responses_kwargs = _build_responses_request(
|
||||
max_tokens=1024,
|
||||
messages=MESSAGES,
|
||||
model="openai/gpt-5.6-luna",
|
||||
|
|
@ -142,8 +140,8 @@ def test_build_responses_kwargs_keeps_the_deployment_include_next_to_encrypted_r
|
|||
assert responses_kwargs["include"] == ["reasoning.encrypted_content", "file_search_call.results"]
|
||||
|
||||
|
||||
def test_build_responses_kwargs_without_metadata_sets_no_prompt_cache_key():
|
||||
responses_kwargs = _build_responses_kwargs(
|
||||
def test_build_responses_request_without_metadata_sets_no_prompt_cache_key():
|
||||
responses_kwargs = _build_responses_request(
|
||||
max_tokens=1024,
|
||||
messages=MESSAGES,
|
||||
model="openai/gpt-5.6-luna",
|
||||
|
|
@ -194,8 +192,6 @@ async def test_streaming_hands_the_logging_object_the_message_id_the_caller_is_s
|
|||
The bridge mints the ``msg_`` id itself, and it is the only request id a streaming
|
||||
/v1/messages caller ever sees, so the spend row has to be keyed on that same value.
|
||||
"""
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-lit6825-test")
|
||||
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
|
|
@ -425,7 +421,7 @@ def _request_with_client_cache_marks(cache_control: dict[str, str] | None) -> di
|
|||
def test_client_cache_control_reaches_gpt_5_6_as_a_prompt_cache_breakpoint_on_the_same_block(
|
||||
extra_kwargs: dict[str, object],
|
||||
) -> None:
|
||||
responses_kwargs: Final = _build_responses_kwargs(
|
||||
responses_kwargs: Final = _build_responses_request(
|
||||
model="openai/gpt-5.6",
|
||||
extra_kwargs=extra_kwargs,
|
||||
**_request_with_client_cache_marks({**EPHEMERAL, "ttl": "1h"}),
|
||||
|
|
@ -453,7 +449,7 @@ def test_client_cache_control_reaches_gpt_5_6_as_a_prompt_cache_breakpoint_on_th
|
|||
|
||||
|
||||
def test_client_cache_control_on_a_screenshot_inside_a_tool_result_reaches_gpt_5_6_on_that_screenshot() -> None:
|
||||
responses_kwargs: Final = _build_responses_kwargs(
|
||||
responses_kwargs: Final = _build_responses_request(
|
||||
model="openai/gpt-5.6",
|
||||
extra_kwargs=OPENAI_API,
|
||||
max_tokens=1024,
|
||||
|
|
@ -532,7 +528,7 @@ RAW_QUESTION: Final = {"type": "input_text", "text": "q", "prompt_cache_breakpoi
|
|||
def test_raw_prompt_cache_breakpoints_reach_a_deployment_unless_drop_params_drops_them_for_a_model_that_takes_none(
|
||||
model: str, extra_kwargs: dict[str, object], log_output: object, question: dict[str, object]
|
||||
) -> None:
|
||||
responses_kwargs: Final = _build_responses_kwargs(
|
||||
responses_kwargs: Final = _build_responses_request(
|
||||
model=model,
|
||||
extra_kwargs=extra_kwargs,
|
||||
max_tokens=1024,
|
||||
|
|
@ -558,7 +554,7 @@ def test_raw_prompt_cache_breakpoints_reach_a_deployment_unless_drop_params_drop
|
|||
|
||||
|
||||
def _gpt_5_6_input_for_a_cached_block(drop_params: bool, raw_mark: Mapping[str, object]) -> object:
|
||||
responses_kwargs: Final = _build_responses_kwargs(
|
||||
responses_kwargs: Final = _build_responses_request(
|
||||
model="openai/gpt-5.6",
|
||||
extra_kwargs={**OPENAI_API, "drop_params": drop_params},
|
||||
max_tokens=1024,
|
||||
|
|
@ -589,10 +585,10 @@ def test_a_malformed_raw_marker_beside_cache_control_gives_way_to_it_only_under_
|
|||
def test_client_cache_control_changes_nothing_for_a_deployment_that_does_not_read_prompt_cache_breakpoint(
|
||||
model: str, extra_kwargs: dict[str, object]
|
||||
) -> None:
|
||||
marked: Final = _build_responses_kwargs(
|
||||
marked: Final = _build_responses_request(
|
||||
model=model, extra_kwargs=extra_kwargs, **_request_with_client_cache_marks(EPHEMERAL)
|
||||
)
|
||||
unmarked: Final = _build_responses_kwargs(
|
||||
unmarked: Final = _build_responses_request(
|
||||
model=model, extra_kwargs=extra_kwargs, **_request_with_client_cache_marks(None)
|
||||
)
|
||||
|
||||
|
|
@ -623,7 +619,7 @@ def _tool_result_turn_with(text_block: Mapping[str, object]) -> list[dict[str, o
|
|||
@pytest.mark.parametrize("text_block", MALFORMED_TEXT_BLOCKS)
|
||||
def test_malformed_tool_result_text_answers_400_naming_the_block(text_block: Mapping[str, object]) -> None:
|
||||
with pytest.raises(litellm.BadRequestError) as rejected:
|
||||
_build_responses_kwargs(
|
||||
_build_responses_request(
|
||||
model="openai/gpt-5.6",
|
||||
extra_kwargs=OPENAI_API,
|
||||
max_tokens=1024,
|
||||
|
|
@ -650,7 +646,7 @@ def test_many_malformed_tool_result_texts_answer_one_bounded_error() -> None:
|
|||
]
|
||||
|
||||
with pytest.raises(litellm.BadRequestError) as rejected:
|
||||
_build_responses_kwargs(model="openai/gpt-5.6", extra_kwargs=OPENAI_API, max_tokens=1024, messages=messages)
|
||||
_build_responses_request(model="openai/gpt-5.6", extra_kwargs=OPENAI_API, max_tokens=1024, messages=messages)
|
||||
|
||||
assert rejected.value.status_code == 400
|
||||
assert f"tool_result {tool_use_id} content.0.text: Input should be a valid string" in rejected.value.message
|
||||
|
|
@ -659,7 +655,7 @@ def test_many_malformed_tool_result_texts_answer_one_bounded_error() -> None:
|
|||
|
||||
@pytest.mark.parametrize("text_block", MALFORMED_TEXT_BLOCKS)
|
||||
def test_drop_params_drops_a_malformed_tool_result_text(text_block: Mapping[str, object]) -> None:
|
||||
responses_kwargs: Final = _build_responses_kwargs(
|
||||
responses_kwargs: Final = _build_responses_request(
|
||||
model="openai/gpt-5.6",
|
||||
extra_kwargs={**OPENAI_API, "drop_params": True},
|
||||
max_tokens=1024,
|
||||
|
|
@ -672,7 +668,7 @@ def test_drop_params_drops_a_malformed_tool_result_text(text_block: Mapping[str,
|
|||
|
||||
|
||||
def test_tool_result_text_block_without_text_still_reaches_the_model_as_empty_text() -> None:
|
||||
responses_kwargs: Final = _build_responses_kwargs(
|
||||
responses_kwargs: Final = _build_responses_request(
|
||||
model="openai/gpt-5.6",
|
||||
extra_kwargs=OPENAI_API,
|
||||
max_tokens=1024,
|
||||
|
|
@ -1084,3 +1080,344 @@ async def test_the_sync_path_reports_its_edits_on_the_streamed_message_delta(
|
|||
]
|
||||
message_delta: Final = next(event for event in events if event["type"] == "message_delta")
|
||||
assert message_delta["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2
|
||||
SEARCH_MODEL: Final = "gpt-bridge-search-cost"
|
||||
SEARCH_RATES: Final = {"input_cost_per_token": 2e-06, "output_cost_per_token": 1e-05}
|
||||
SEARCH_COST_PER_QUERY: Final = 0.03
|
||||
SEARCH_RESPONSE: Final = {
|
||||
**REASONING_RESPONSE,
|
||||
"id": "resp_bridge_search",
|
||||
"model": SEARCH_MODEL,
|
||||
"output": [
|
||||
*(
|
||||
{
|
||||
"type": "web_search_call",
|
||||
"id": f"ws_{i}",
|
||||
"status": "completed",
|
||||
"action": {"type": "search", "query": "q"},
|
||||
}
|
||||
for i in range(3)
|
||||
),
|
||||
REASONING_RESPONSE["output"][1],
|
||||
],
|
||||
"usage": {"input_tokens": 100, "output_tokens": 20, "total_tokens": 120},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_served_bridge_response_is_billed_for_each_web_search_call_the_provider_made(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
|
||||
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-bridge-search-test")
|
||||
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
SEARCH_MODEL,
|
||||
{
|
||||
"litellm_provider": "openai",
|
||||
"mode": "responses",
|
||||
"search_context_cost_per_query": {
|
||||
f"search_context_size_{size}": SEARCH_COST_PER_QUERY for size in ("low", "medium", "high")
|
||||
},
|
||||
**SEARCH_RATES,
|
||||
},
|
||||
)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
respx_mock.post("https://api.openai.com/v1/responses").respond(status_code=200, json=SEARCH_RESPONSE)
|
||||
logging_obj: Final = Logging(
|
||||
model=SEARCH_MODEL,
|
||||
messages=MESSAGES,
|
||||
stream=False,
|
||||
call_type="anthropic_messages",
|
||||
start_time=datetime.datetime.now(datetime.timezone.utc),
|
||||
litellm_call_id="bridge-search-0000-4000-8000-000000000001",
|
||||
function_id="1234",
|
||||
)
|
||||
|
||||
served: Final = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
|
||||
max_tokens=412,
|
||||
messages=MESSAGES,
|
||||
model=f"openai/{SEARCH_MODEL}",
|
||||
custom_llm_provider="openai",
|
||||
tools=[{"type": "web_search_20250305", "name": "web_search"}],
|
||||
litellm_logging_obj=logging_obj,
|
||||
)
|
||||
assert isinstance(served, dict)
|
||||
|
||||
cost: Final = ProxyBaseLLMRequestProcessing._response_cost_from_logging_obj(
|
||||
response=dict(served), logging_obj=logging_obj
|
||||
)
|
||||
|
||||
expected_tokens: Final = 100 * SEARCH_RATES["input_cost_per_token"] + 20 * SEARCH_RATES["output_cost_per_token"]
|
||||
assert cost == pytest.approx(expected_tokens + 3 * SEARCH_COST_PER_QUERY)
|
||||
assert logging_obj.cost_breakdown is not None
|
||||
assert logging_obj.cost_breakdown["tool_usage_cost"] == pytest.approx(3 * SEARCH_COST_PER_QUERY)
|
||||
|
||||
|
||||
def _messages_logging_obj(stream: bool) -> Logging:
|
||||
return Logging(
|
||||
model="gpt-5.6-luna",
|
||||
messages=MESSAGES,
|
||||
stream=stream,
|
||||
call_type="anthropic_messages",
|
||||
start_time=datetime.datetime.now(datetime.timezone.utc),
|
||||
litellm_call_id="6825beef-0000-4000-8000-000000000004",
|
||||
function_id="1234",
|
||||
)
|
||||
|
||||
|
||||
def _untyped_bridge() -> Callable[..., Coroutine[object, object, object]]:
|
||||
return LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler
|
||||
|
||||
|
||||
async def _logged_bridge_call(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, upstream_response: httpx.Response, **extra: object
|
||||
) -> tuple[object, respx.Route]:
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-native-test")
|
||||
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=upstream_response)
|
||||
response: Final = await _untyped_bridge()(
|
||||
max_tokens=1024,
|
||||
messages=MESSAGES,
|
||||
model="openai/gpt-5.6-luna",
|
||||
custom_llm_provider="openai",
|
||||
litellm_logging_obj=_messages_logging_obj(stream=False),
|
||||
**extra,
|
||||
)
|
||||
return response, upstream
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridge_answers_a_non_stream_request_with_a_message_even_when_the_caller_asked_for_stream_replay(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
response, upstream = await _logged_bridge_call(
|
||||
respx_mock,
|
||||
monkeypatch,
|
||||
httpx.Response(200, json=TOOL_CALL_RESPONSE),
|
||||
**{key: True for key in CONVERTED_STREAM_KEYS},
|
||||
)
|
||||
|
||||
assert isinstance(response, dict) and response["type"] == "message"
|
||||
assert json.loads(upstream.calls.last.request.content).get("stream") is not True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridge_sends_the_deployment_prompt_cache_options_upstream(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
_, upstream = await _logged_bridge_call(
|
||||
respx_mock,
|
||||
monkeypatch,
|
||||
httpx.Response(200, json=TOOL_CALL_RESPONSE),
|
||||
prompt_cache_options={"mode": "explicit"},
|
||||
)
|
||||
|
||||
assert json.loads(upstream.calls.last.request.content)["prompt_cache_options"] == {"mode": "explicit"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridge_raises_the_litellm_rate_limit_error_for_an_upstream_429(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
with pytest.raises(litellm.RateLimitError):
|
||||
await _logged_bridge_call(
|
||||
respx_mock,
|
||||
monkeypatch,
|
||||
httpx.Response(429, json={"error": {"message": "slow down", "type": "rate_limit_error"}}),
|
||||
)
|
||||
|
||||
|
||||
class _TemplatePromptManager(CustomPromptManagement):
|
||||
async def async_get_chat_completion_prompt(
|
||||
self,
|
||||
model: str,
|
||||
messages: list[AllMessageValues],
|
||||
non_default_params: dict[str, object],
|
||||
prompt_id: str | None,
|
||||
prompt_variables: dict[str, object] | None,
|
||||
dynamic_callback_params: StandardCallbackDynamicParams,
|
||||
litellm_logging_obj: Logging,
|
||||
prompt_spec: PromptSpec | None = None,
|
||||
tools: list[dict[str, object]] | None = None,
|
||||
prompt_label: str | None = None,
|
||||
prompt_version: int | None = None,
|
||||
ignore_prompt_manager_model: bool | None = False,
|
||||
ignore_prompt_manager_optional_params: bool | None = False,
|
||||
) -> tuple[str, list[AllMessageValues], dict[str, object]]:
|
||||
return model, [{"role": "user", "content": "from the template"}], non_default_params
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridge_applies_the_prompt_template_named_by_prompt_id(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
monkeypatch.setattr(litellm, "callbacks", [_TemplatePromptManager()])
|
||||
|
||||
_, upstream = await _logged_bridge_call(
|
||||
respx_mock, monkeypatch, httpx.Response(200, json=TOOL_CALL_RESPONSE), prompt_id="bridge_template"
|
||||
)
|
||||
|
||||
assert "from the template" in upstream.calls.last.request.content.decode()
|
||||
|
||||
|
||||
def _logged_sync_bridge_call(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, upstream_response: httpx.Response
|
||||
) -> tuple[object, respx.Route]:
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-native-test")
|
||||
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=upstream_response)
|
||||
response: Final = LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
|
||||
max_tokens=1024,
|
||||
messages=MESSAGES,
|
||||
model="openai/gpt-5.6-luna",
|
||||
custom_llm_provider="openai",
|
||||
litellm_logging_obj=_messages_logging_obj(stream=False),
|
||||
)
|
||||
return response, upstream
|
||||
|
||||
|
||||
def test_sync_bridge_answers_with_a_message_from_one_upstream_call(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
response, upstream = _logged_sync_bridge_call(respx_mock, monkeypatch, httpx.Response(200, json=TOOL_CALL_RESPONSE))
|
||||
|
||||
assert isinstance(response, dict) and response["type"] == "message"
|
||||
assert [block["id"] for block in response["content"] if block["type"] == "tool_use"] == ["call_curl"]
|
||||
assert upstream.call_count == 1
|
||||
|
||||
|
||||
def test_sync_bridge_raises_the_litellm_rate_limit_error_for_an_upstream_429(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
with pytest.raises(litellm.RateLimitError):
|
||||
_logged_sync_bridge_call(
|
||||
respx_mock,
|
||||
monkeypatch,
|
||||
httpx.Response(429, json={"error": {"message": "slow down", "type": "rate_limit_error"}}),
|
||||
)
|
||||
|
||||
|
||||
class _SuccessRowRecorder(CustomLogger):
|
||||
call_types: tuple[str, ...] = ()
|
||||
|
||||
async def async_log_success_event(
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
self.call_types = (*self.call_types, str(kwargs.get("call_type")))
|
||||
|
||||
|
||||
_PARKED_COMPLETION: Final = TypeAdapter(tuple[object])
|
||||
|
||||
|
||||
async def _release(parked: Coroutine[object, object, object]) -> None:
|
||||
await parked
|
||||
|
||||
|
||||
async def _settled_rows(recorder: _SuccessRowRecorder) -> tuple[str, ...]:
|
||||
for _ in range(200):
|
||||
if recorder.call_types:
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
await asyncio.sleep(0.05)
|
||||
return recorder.call_types
|
||||
|
||||
|
||||
async def _bridged_stream(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, logging_obj: Logging
|
||||
) -> tuple[AnthropicResponsesSSEByteStream, _SuccessRowRecorder]:
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-native-test")
|
||||
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
recorder: Final = _SuccessRowRecorder()
|
||||
monkeypatch.setattr(litellm, "_async_success_callback", [recorder])
|
||||
respx_mock.post("https://api.openai.com/v1/responses").mock(
|
||||
return_value=httpx.Response(200, headers={"Content-Type": "text/event-stream"}, content=RESPONSES_SSE_BODY)
|
||||
)
|
||||
stream: Final = await _untyped_bridge()(
|
||||
max_tokens=1024,
|
||||
messages=MESSAGES,
|
||||
model="openai/gpt-5.6-luna",
|
||||
stream=True,
|
||||
custom_llm_provider="openai",
|
||||
litellm_logging_obj=logging_obj,
|
||||
)
|
||||
assert isinstance(stream, AnthropicResponsesSSEByteStream)
|
||||
return stream, recorder
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridged_stream_opens_with_the_cost_breakdown_headers(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
from litellm.proxy.common_request_processing import cost_headers
|
||||
|
||||
logging_obj: Final = _messages_logging_obj(stream=True)
|
||||
stream, _ = await _bridged_stream(respx_mock, monkeypatch, logging_obj)
|
||||
|
||||
headers: Final = cost_headers(None, logging_obj)
|
||||
await stream.aclose()
|
||||
|
||||
assert headers["x-litellm-response-cost-output"] == "0.0"
|
||||
assert headers["x-litellm-response-cost-discount-amount"] == "0.0"
|
||||
assert headers["x-litellm-response-cost-margin-amount"] == "0.0"
|
||||
|
||||
|
||||
async def _read_through_message_stop(stream: AsyncIterator[bytes]) -> None:
|
||||
async for frame in stream:
|
||||
if frame.startswith(b"event: message_stop"):
|
||||
return
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridged_stream_logs_one_row_when_the_client_stream_ends(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
stream, recorder = await _bridged_stream(respx_mock, monkeypatch, _messages_logging_obj(stream=True))
|
||||
|
||||
await _read_through_message_stop(stream)
|
||||
rows_while_the_client_stream_is_open: Final = await _settled_rows(recorder)
|
||||
remaining: Final[list[bytes]] = [frame async for frame in stream]
|
||||
|
||||
assert rows_while_the_client_stream_is_open == ()
|
||||
assert remaining == []
|
||||
assert await _settled_rows(recorder) == ("anthropic_messages",)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridged_stream_still_logs_when_the_client_leaves_after_message_stop(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
stream, recorder = await _bridged_stream(respx_mock, monkeypatch, _messages_logging_obj(stream=True))
|
||||
|
||||
await _read_through_message_stop(stream)
|
||||
await stream.aclose()
|
||||
|
||||
assert await _settled_rows(recorder) == ("anthropic_messages",)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridged_stream_holds_its_row_for_the_proxy_end_of_stream_guardrails(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
logging_obj: Final = _messages_logging_obj(stream=True)
|
||||
|
||||
async def release(*held: object) -> None:
|
||||
return None
|
||||
|
||||
setattr(logging_obj, "_on_deferred_stream_complete", release) # noqa: B010 # the proxy arms this slot dynamically
|
||||
stream, recorder = await _bridged_stream(respx_mock, monkeypatch, logging_obj)
|
||||
|
||||
frames: Final = [frame async for frame in stream]
|
||||
rows_before_release: Final = await _settled_rows(recorder)
|
||||
held: Final[object] = getattr(logging_obj, "_deferred_stream_complete_args", None)
|
||||
|
||||
assert frames[-1].startswith(b"event: message_stop")
|
||||
assert rows_before_release == ()
|
||||
(parked,) = _PARKED_COMPLETION.validate_python(held)
|
||||
assert asyncio.iscoroutine(parked)
|
||||
await _release(parked)
|
||||
assert recorder.call_types == ("anthropic_messages",)
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import datetime
|
|||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable, Mapping
|
||||
from types import SimpleNamespace
|
||||
|
||||
import anyio
|
||||
|
|
@ -18,10 +18,12 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.
|
|||
|
||||
import litellm
|
||||
from litellm.exceptions import MidStreamFallbackError
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
encrypted_reasoning_signature,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.held_stream_completion import HeldStreamCompletion
|
||||
from litellm.llms.anthropic.messages.native.streaming_iterator import aclose_superseded_attempt
|
||||
from litellm.llms.anthropic.messages.native.utils import INCOMPLETE_STREAM_ERROR_MESSAGE
|
||||
from litellm.llms.anthropic.messages.responses_adapters.streaming_iterator import (
|
||||
|
|
@ -818,3 +820,56 @@ class TestUpstreamFailureReachesTheProxyFailureHook:
|
|||
|
||||
asyncio.run(_disconnect_mid_hook())
|
||||
assert len(finished) == 1
|
||||
|
||||
|
||||
class _AsyncSuccessRecorder(CustomLogger):
|
||||
calls: tuple[object, ...] = ()
|
||||
|
||||
async def async_log_success_event(
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
self.calls = (*self.calls, response_obj)
|
||||
|
||||
|
||||
def test_a_stream_that_fails_after_the_executor_completed_logs_no_success_row():
|
||||
recorder = _AsyncSuccessRecorder()
|
||||
logging_obj = LitellmLogging(
|
||||
model="m",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
call_type="anthropic_messages",
|
||||
start_time=datetime.datetime.now(),
|
||||
litellm_call_id="fails-after-complete",
|
||||
function_id="fails-after-complete",
|
||||
dynamic_async_success_callbacks=[recorder],
|
||||
)
|
||||
held = HeldStreamCompletion(logging_obj)
|
||||
completed = ResponsesAPIResponse(
|
||||
id="resp_1",
|
||||
created_at=1,
|
||||
status="completed",
|
||||
model="m",
|
||||
output=[],
|
||||
parallel_tool_calls=True,
|
||||
tool_choice="auto",
|
||||
tools=[],
|
||||
)
|
||||
|
||||
async def _gen() -> AsyncIterator[dict[str, object]]:
|
||||
yield {"type": "response.created"}
|
||||
held.take(completed)
|
||||
yield {"type": "response.completed", "response": completed}
|
||||
raise ConnectionResetError("Response payload is not completed")
|
||||
|
||||
async def _run() -> list[bytes]:
|
||||
wrapper = AnthropicResponsesStreamWrapper(
|
||||
responses_stream=_gen(), model="m", litellm_logging_obj=logging_obj, held_completion=held
|
||||
)
|
||||
frames: list[bytes] = [frame async for frame in wrapper.async_anthropic_sse_wrapper()]
|
||||
await asyncio.sleep(0.2)
|
||||
return frames
|
||||
|
||||
frames = asyncio.run(_run())
|
||||
|
||||
assert frames[-1].startswith(b"event: error")
|
||||
assert recorder.calls == ()
|
||||
|
|
|
|||
|
|
@ -2411,16 +2411,3 @@ class TestPromptCacheBreakpointToResponses:
|
|||
self.MARKED_SCREENSHOT_PART,
|
||||
]
|
||||
assert log_output == self.LOG_OUTPUT
|
||||
|
||||
def test_prompt_cache_options_forwarded_to_responses_kwargs(self):
|
||||
from litellm.llms.anthropic.messages.responses_adapters.handler import (
|
||||
_build_responses_kwargs,
|
||||
)
|
||||
|
||||
kwargs = _build_responses_kwargs(
|
||||
max_tokens=16,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
model="openai/gpt-5.6",
|
||||
extra_kwargs={"prompt_cache_options": {"mode": "explicit"}},
|
||||
)
|
||||
assert kwargs["prompt_cache_options"] == {"mode": "explicit"}
|
||||
|
|
|
|||
|
|
@ -172,7 +172,7 @@ def test_azure_o_series_routing():
|
|||
assert "stream" not in mock_create.call_args.kwargs["body"]
|
||||
|
||||
|
||||
@patch("litellm.main.azure_o1_chat_completions._get_openai_client")
|
||||
@patch("litellm.native.chat_providers.handlers.azure_o1_chat_completions._get_openai_client")
|
||||
def test_openai_o_series_max_retries_0(mock_get_openai_client):
|
||||
import litellm
|
||||
|
||||
|
|
|
|||
|
|
@ -583,7 +583,9 @@ def test_azure_openai_responses_bridge():
|
|||
|
||||
litellm.turn_on_debug()
|
||||
|
||||
with patch.object(litellm, "responses") as mock_responses:
|
||||
from litellm.responses import bridge_executor
|
||||
|
||||
with patch.object(bridge_executor, "native_responses") as mock_responses:
|
||||
try:
|
||||
response = completion(
|
||||
model="azure/responses/test-azure-computer-use-preview",
|
||||
|
|
@ -597,10 +599,10 @@ def test_azure_openai_responses_bridge():
|
|||
|
||||
mock_responses.assert_called_once()
|
||||
assert (
|
||||
mock_responses.call_args.kwargs["model"]
|
||||
mock_responses.call_args.args[0]["model"]
|
||||
== "azure/test-azure-computer-use-preview"
|
||||
)
|
||||
assert mock_responses.call_args.kwargs["custom_llm_provider"] == "azure"
|
||||
assert mock_responses.call_args.args[0]["custom_llm_provider"] == "azure"
|
||||
|
||||
|
||||
def test_azure_with_content_safety_error():
|
||||
|
|
|
|||
|
|
@ -735,7 +735,7 @@ async def test_ensure_initialize_azure_sdk_client_always_used_azure_text(call_ty
|
|||
# Get appropriate input for this call type
|
||||
input_kwarg = test_inputs.get(call_type.value, {})
|
||||
|
||||
patch_target = "litellm.main.azure_text_completions.initialize_azure_sdk_client"
|
||||
patch_target = "litellm.native.chat_providers.handlers.azure_text_completions.initialize_azure_sdk_client"
|
||||
|
||||
# Mock the initialize_azure_sdk_client function
|
||||
with patch(patch_target) as mock_init_azure:
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ def test_main_azure_ai_claude_completion_passes_timeout_to_azure_anthropic_handl
|
|||
captured.update(kwargs)
|
||||
return ModelResponse()
|
||||
|
||||
with patch("litellm.main.azure_anthropic_chat_completions") as mock_azure_anthropic:
|
||||
with patch("litellm.native.chat_providers.azure.azure_anthropic_chat_completions") as mock_azure_anthropic:
|
||||
mock_azure_anthropic.completion = MagicMock(
|
||||
side_effect=fake_azure_anthropic_completion
|
||||
)
|
||||
|
|
|
|||
|
|
@ -686,7 +686,7 @@ class TestGDCGeminiConfig:
|
|||
class TestCompleteGDC:
|
||||
@patch("litellm.main.base_llm_http_handler.completion")
|
||||
def test_complete_gdc_resolves_key_and_base(self, mock_completion, monkeypatch):
|
||||
from litellm.main import gdc_transformation
|
||||
from litellm.native.chat_providers.handlers import gdc_transformation
|
||||
|
||||
mock_completion.return_value = MagicMock()
|
||||
monkeypatch.setattr(litellm, "gdc_key", "resolved-key", raising=False)
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ def test_gemini_completion_no_api_key():
|
|||
del os.environ[key]
|
||||
|
||||
# Test without mock_response to ensure actual API key validation
|
||||
with pytest.raises(Exception, match='in _complete_vertex_ai_beta') as exc_info:
|
||||
with pytest.raises(Exception, match='in complete_vertex_ai_beta') as exc_info:
|
||||
completion(
|
||||
model="gemini/gemini-1.5-flash",
|
||||
messages=[{"role": "user", "content": "Test message"}],
|
||||
|
|
@ -60,7 +60,7 @@ def test_gemini_completion_no_api_key_with_mock():
|
|||
with patch("litellm.get_secret") as mock_get_secret:
|
||||
mock_get_secret.return_value = None
|
||||
|
||||
with pytest.raises(Exception, match='in _complete_vertex_ai_beta') as exc_info:
|
||||
with pytest.raises(Exception, match='in complete_vertex_ai_beta') as exc_info:
|
||||
completion(
|
||||
model="gemini/gemini-1.5-flash",
|
||||
messages=[{"role": "user", "content": "Test message"}],
|
||||
|
|
|
|||
|
|
@ -668,7 +668,7 @@ def test_e2e_completion_per_user_credential(per_user_credential):
|
|||
|
||||
def test_e2e_completion_per_user_session_token_wins_over_caller_authorization(per_user_credential):
|
||||
"""The session token reaches the wire even when the caller passes their own
|
||||
Authorization in extra_headers (chat dispatches through _complete_custom_openai)."""
|
||||
Authorization in extra_headers (chat dispatches through complete_custom_openai)."""
|
||||
with (
|
||||
github_http(_exchange_responder()),
|
||||
_no_authenticator(),
|
||||
|
|
|
|||
|
|
@ -1934,7 +1934,9 @@ def test_responses_gpt54_with_xhigh_reasoning():
|
|||
Ensure chat->responses bridge sends the correct request payload for
|
||||
openai/responses/gpt-5.4 with reasoning_effort="xhigh".
|
||||
"""
|
||||
with patch("litellm.responses") as mock_responses:
|
||||
from litellm.responses import bridge_executor
|
||||
|
||||
with patch.object(bridge_executor, "native_responses") as mock_responses:
|
||||
mock_responses.side_effect = RuntimeError("stop_after_request_build")
|
||||
|
||||
with pytest.raises(litellm.APIConnectionError):
|
||||
|
|
@ -1946,7 +1948,7 @@ def test_responses_gpt54_with_xhigh_reasoning():
|
|||
)
|
||||
|
||||
mock_responses.assert_called_once()
|
||||
request_body = mock_responses.call_args.kwargs
|
||||
request_body = mock_responses.call_args.args[0]
|
||||
|
||||
assert request_body["model"] == "openai/gpt-5.4"
|
||||
|
||||
|
|
|
|||
|
|
@ -8515,8 +8515,11 @@ async def test_claude_tool_use_with_gemini():
|
|||
return chunk
|
||||
raise StopAsyncIteration
|
||||
|
||||
with patch("litellm.acompletion", new_callable=AsyncMock) as mock_acompletion:
|
||||
mock_acompletion.return_value = MockAsyncStream()
|
||||
with patch(
|
||||
"litellm.llms.anthropic.messages.adapters.handler.aexecute_planned_chat",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_chat:
|
||||
mock_chat.return_value = MockAsyncStream()
|
||||
|
||||
response = await litellm.anthropic.messages.acreate(
|
||||
messages=[
|
||||
|
|
|
|||
|
|
@ -1545,9 +1545,9 @@ def test_system_prompt_only_adds_blank_user_message():
|
|||
|
||||
@pytest.fixture
|
||||
def _cached_vertex_credentials(monkeypatch: pytest.MonkeyPatch) -> str:
|
||||
from litellm.main import (
|
||||
from litellm.main import vertex_embedding
|
||||
from litellm.native.chat_providers.handlers import (
|
||||
vertex_chat_completion,
|
||||
vertex_embedding,
|
||||
vertex_model_garden_chat_completion,
|
||||
vertex_partner_models_chat_completion,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import pytest
|
|||
|
||||
import litellm
|
||||
from litellm._uuid import uuid
|
||||
from litellm.native.chat_providers.handlers import vertex_partner_models_chat_completion
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
anthropic_messages_pt,
|
||||
convert_to_anthropic_tool_result,
|
||||
|
|
@ -406,7 +407,7 @@ async def test_vertex_ai_anthropic_async_completion_inlines_https_images_off_the
|
|||
with (
|
||||
patch.dict(sys.modules, {"vertexai": vertexai}),
|
||||
patch.object( # test-quality-ok: litellm.acompletion has no seam for Vertex token minting
|
||||
litellm.main.vertex_partner_models_chat_completion,
|
||||
vertex_partner_models_chat_completion,
|
||||
"_ensure_access_token",
|
||||
return_value=("token", "test-project"),
|
||||
),
|
||||
|
|
|
|||
|
|
@ -585,7 +585,7 @@ async def test_vertex_ai_anthropic_prompt_caching_does_not_add_beta_header(
|
|||
monkeypatch: pytest.MonkeyPatch,
|
||||
sync_mode: bool,
|
||||
) -> None:
|
||||
from litellm.main import vertex_partner_models_chat_completion
|
||||
from litellm.native.chat_providers.handlers import vertex_partner_models_chat_completion
|
||||
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setitem(
|
||||
|
|
|
|||
|
|
@ -1320,7 +1320,7 @@ def _gemma_cached_access_token():
|
|||
"""Serve a fake token from the handler's credential cache so no auth round-trip runs."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from litellm.main import vertex_gemma_chat_completion
|
||||
from litellm.native.chat_providers.handlers import vertex_gemma_chat_completion
|
||||
|
||||
cache = vertex_gemma_chat_completion._credentials_project_mapping
|
||||
key = (_FAKE_GEMMA_CREDENTIALS, "test")
|
||||
|
|
|
|||
0
tests/unit/native/__init__.py
Normal file
0
tests/unit/native/__init__.py
Normal file
166
tests/unit/native/test_chat_plan.py
Normal file
166
tests/unit/native/test_chat_plan.py
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
import json
|
||||
from datetime import datetime
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm.exceptions import JSONSchemaValidationError
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.native.chat_plan import plan_chat
|
||||
|
||||
API_BASE: Final = "https://bridge-parity.invalid/v1"
|
||||
SCHEMA: Final = {
|
||||
"type": "object",
|
||||
"properties": {"b": {"type": "integer"}},
|
||||
"required": ["b"],
|
||||
"additionalProperties": False,
|
||||
}
|
||||
OFF_SCHEMA_TEXT: Final = '{"b": 2, "extra": true}'
|
||||
_CHAT_REPLY: Final = {
|
||||
"id": "chatcmpl-1",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "m",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": OFF_SCHEMA_TEXT}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
|
||||
}
|
||||
|
||||
|
||||
def _router(litellm_params: dict[str, object], model_info: dict[str, object] | None = None) -> litellm.Router:
|
||||
return litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "m",
|
||||
"litellm_params": {"api_base": API_BASE, "api_key": "k", **litellm_params},
|
||||
**({"model_info": model_info} if model_info is not None else {}),
|
||||
}
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _offline(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("litellm_params", "model_info"),
|
||||
[
|
||||
({"model": "fireworks_ai/accounts/fireworks/models/llama-v3p1-8b-instruct"}, None),
|
||||
({"model": "openai/gpt-4.1", "use_chat_completions_api": True}, None),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_messages_request_bridged_to_chat_validates_its_json_schema(
|
||||
litellm_params: dict[str, object],
|
||||
model_info: dict[str, object] | None,
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(litellm, "enable_json_schema_validation", True)
|
||||
respx_mock.post(f"{API_BASE}/chat/completions").mock(return_value=httpx.Response(200, json=_CHAT_REPLY))
|
||||
|
||||
with pytest.raises(JSONSchemaValidationError):
|
||||
await _router(litellm_params, model_info).aanthropic_messages(
|
||||
model="m",
|
||||
max_tokens=32,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
output_format={"type": "json_schema", "schema": SCHEMA},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_responses_request_bridged_to_chat_validates_its_json_schema(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
monkeypatch.setattr(litellm, "enable_json_schema_validation", True)
|
||||
respx_mock.post(f"{API_BASE}/chat/completions").mock(return_value=httpx.Response(200, json=_CHAT_REPLY))
|
||||
|
||||
with pytest.raises(JSONSchemaValidationError):
|
||||
await _router({"model": "deepseek/deepseek-reasoner"}, {"mode": "responses"}).aresponses(
|
||||
model="m",
|
||||
input="hi",
|
||||
text={"format": {"type": "json_schema", "name": "x", "strict": True, "schema": SCHEMA}},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_provider_failure_on_the_messages_to_chat_bridge_is_mapped(respx_mock: respx.MockRouter):
|
||||
respx_mock.post(f"{API_BASE}/chat/completions").mock(return_value=httpx.Response(200, content=b"not json"))
|
||||
|
||||
with pytest.raises(litellm.exceptions.APIConnectionError):
|
||||
await _router({"model": "groq/llama-3.1-8b-instant"}, {"mode": "responses"}).aanthropic_messages(
|
||||
model="m", max_tokens=32, messages=[{"role": "user", "content": "hi"}]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_responses_request_on_a_chat_api_deployment_stays_on_chat_despite_mode_responses(
|
||||
respx_mock: respx.MockRouter,
|
||||
):
|
||||
chat: Final = respx_mock.post(f"{API_BASE}/chat/completions").mock(
|
||||
return_value=httpx.Response(200, json=_CHAT_REPLY)
|
||||
)
|
||||
|
||||
await _router({"model": "openai/gpt-4.1", "use_chat_completions_api": True}, {"mode": "responses"}).aresponses(
|
||||
model="m", input="hi"
|
||||
)
|
||||
|
||||
assert chat.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_skip_bridge_flag_is_never_sent_to_the_provider(respx_mock: respx.MockRouter):
|
||||
chat: Final = respx_mock.post(f"{API_BASE}/chat/completions").mock(
|
||||
return_value=httpx.Response(200, json=_CHAT_REPLY)
|
||||
)
|
||||
|
||||
await _router({"model": "openai/gpt-4.1"}).acompletion(
|
||||
model="m", messages=[{"role": "user", "content": "hi"}], _skip_responses_api_bridge=True
|
||||
)
|
||||
|
||||
assert "_skip_responses_api_bridge" not in json.loads(chat.calls.last.request.content)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_skip_bridge_flag_keeps_a_mode_responses_deployment_on_chat(respx_mock: respx.MockRouter):
|
||||
chat: Final = respx_mock.post(f"{API_BASE}/chat/completions").mock(
|
||||
return_value=httpx.Response(200, json=_CHAT_REPLY)
|
||||
)
|
||||
|
||||
await _router({"model": "openai/gpt-4.1"}, {"mode": "responses"}).acompletion(
|
||||
model="m", messages=[{"role": "user", "content": "hi"}], _skip_responses_api_bridge=True
|
||||
)
|
||||
|
||||
assert chat.call_count == 1
|
||||
|
||||
|
||||
def _bridged_chat_logging() -> Logging:
|
||||
return Logging(
|
||||
model="gpt-4.1",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=False,
|
||||
call_type="aresponses",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="planned-chat-cache",
|
||||
function_id="planned-chat-cache",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("response_cache", "planned"), [(None, True), (litellm.Cache(type="local"), False)])
|
||||
def test_a_bridged_chat_call_goes_through_acompletion_when_the_response_cache_is_on(
|
||||
response_cache: litellm.Cache | None, planned: bool, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
monkeypatch.setattr(litellm, "cache", response_cache)
|
||||
|
||||
plan: Final = plan_chat(
|
||||
{"model": "openai/gpt-4.1", "messages": [{"role": "user", "content": "hi"}], "api_key": "k"},
|
||||
_bridged_chat_logging(),
|
||||
)
|
||||
|
||||
assert (plan is not None) is planned
|
||||
288
tests/unit/native/test_route_plan.py
Normal file
288
tests/unit/native/test_route_plan.py
Normal file
|
|
@ -0,0 +1,288 @@
|
|||
import asyncio
|
||||
import json
|
||||
from collections.abc import AsyncIterable, Callable, Coroutine, Mapping, Sequence
|
||||
from typing import Final, cast
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.llms.openai_like.messages.transformation import OpenAILikeAnthropicMessagesConfig
|
||||
from litellm.messages.dispatch import anthropic_messages
|
||||
from litellm.native.route_plan import (
|
||||
NativeChat,
|
||||
NativeMessages,
|
||||
NativeResponses,
|
||||
RoutePlan,
|
||||
plan_chat_route,
|
||||
plan_messages_route,
|
||||
plan_responses_route,
|
||||
)
|
||||
|
||||
MESSAGES_PASSTHROUGH: dict[str, object] = {"supported_endpoints": ["/v1/messages"]}
|
||||
|
||||
|
||||
def _plan(
|
||||
model: str,
|
||||
provider: str,
|
||||
*,
|
||||
model_info: object = None,
|
||||
compaction_requested: bool = False,
|
||||
force_chat_completions: bool = False,
|
||||
) -> RoutePlan:
|
||||
return plan_messages_route(
|
||||
requested_model=f"{provider}/{model}",
|
||||
resolved_model=model,
|
||||
custom_llm_provider=provider,
|
||||
model_info=model_info,
|
||||
api_base=None,
|
||||
compaction_requested=compaction_requested,
|
||||
force_chat_completions=force_chat_completions,
|
||||
)
|
||||
|
||||
|
||||
def test_a_provider_that_speaks_messages_gets_its_own_messages_config():
|
||||
plan = _plan("claude-sonnet-4-5", "anthropic")
|
||||
|
||||
assert isinstance(plan, NativeMessages)
|
||||
assert isinstance(plan.config, AnthropicMessagesConfig)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "provider", "kwargs", "expected"),
|
||||
[
|
||||
("gpt-5.5", "openai", {}, NativeResponses()),
|
||||
("gpt-5.5", "openai", {"force_chat_completions": True}, NativeChat()),
|
||||
("gpt-5.5", "openai", {"compaction_requested": True}, NativeChat()),
|
||||
("my-model", "hosted_vllm", {}, NativeChat()),
|
||||
],
|
||||
)
|
||||
def test_messages_without_a_native_config_bridge_to_responses_only_for_openai(
|
||||
model: str, provider: str, kwargs: dict[str, bool], expected: RoutePlan
|
||||
):
|
||||
assert _plan(model, provider, **kwargs) == expected
|
||||
|
||||
|
||||
_MODEL_COST: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
def _isolate_model_cost(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "model_cost", _MODEL_COST.validate_python(litellm.model_cost))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cache_control_ttl", [True, False])
|
||||
def test_a_deployment_that_opts_into_messages_passthrough_skips_translation(cache_control_ttl: bool):
|
||||
plan = _plan("my-model", "hosted_vllm", model_info={**MESSAGES_PASSTHROUGH, "cache_control_ttl": cache_control_ttl})
|
||||
|
||||
assert isinstance(plan, NativeMessages)
|
||||
assert isinstance(plan.config, OpenAILikeAnthropicMessagesConfig)
|
||||
assert plan.config.supports_cache_control_ttl() is cache_control_ttl
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("has_config", "use_chat_completions_api", "expected"),
|
||||
[
|
||||
(True, False, NativeResponses()),
|
||||
(True, True, NativeChat()),
|
||||
(False, False, NativeChat()),
|
||||
(False, True, NativeChat()),
|
||||
],
|
||||
)
|
||||
def test_responses_reach_chat_without_a_provider_config_or_when_asked(
|
||||
has_config: bool, use_chat_completions_api: bool, expected: RoutePlan
|
||||
):
|
||||
config = OpenAIResponsesAPIConfig() if has_config else None
|
||||
|
||||
assert plan_responses_route(config, use_chat_completions_api) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("has_config", "expected"),
|
||||
[
|
||||
(True, NativeResponses()),
|
||||
(False, NativeChat()),
|
||||
],
|
||||
)
|
||||
def test_chat_bridges_to_responses_only_when_the_provider_serves_it(has_config: bool, expected: RoutePlan):
|
||||
assert plan_chat_route(OpenAIResponsesAPIConfig() if has_config else None) == expected
|
||||
|
||||
|
||||
CHAT_BASE: Final = "http://chat-upstream.test/v1"
|
||||
RESPONSES_BASE: Final = "http://responses-upstream.test/v1"
|
||||
CHAT_MODEL: Final = "hosted_vllm/served-model"
|
||||
RESPONSES_ONLY_MODEL: Final = "openai/bridge-once-model"
|
||||
USER_TURN: Final = ({"role": "user", "content": "hi"},)
|
||||
|
||||
_CHAT_REPLY: Final[Mapping[str, object]] = {
|
||||
"id": "chatcmpl-once",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "served-model",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "once"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
|
||||
}
|
||||
_CHAT_CHUNKS: Final[tuple[Mapping[str, object], ...]] = (
|
||||
{
|
||||
"id": "chatcmpl-once",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "served-model",
|
||||
"choices": [{"index": 0, "delta": {"role": "assistant", "content": "once"}, "finish_reason": None}],
|
||||
},
|
||||
{
|
||||
"id": "chatcmpl-once",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "served-model",
|
||||
"choices": [{"index": 0, "delta": dict[str, object](), "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
|
||||
},
|
||||
)
|
||||
_OUTPUT_MESSAGE: Final[Mapping[str, object]] = {
|
||||
"type": "message",
|
||||
"id": "msg_once",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "once", "annotations": list[object]()}],
|
||||
}
|
||||
_RESPONSES_REPLY: Final[Mapping[str, object]] = {
|
||||
"id": "resp_once",
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"model": "bridge-once-model",
|
||||
"status": "completed",
|
||||
"output": [_OUTPUT_MESSAGE],
|
||||
"parallel_tool_calls": True,
|
||||
"tool_choice": "auto",
|
||||
"tools": list[object](),
|
||||
"usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
|
||||
}
|
||||
_RESPONSES_EVENTS: Final[tuple[Mapping[str, object], ...]] = (
|
||||
{"type": "response.created", "sequence_number": 0, "response": {**_RESPONSES_REPLY, "status": "in_progress"}},
|
||||
{"type": "response.output_item.added", "sequence_number": 1, "output_index": 0, "item": _OUTPUT_MESSAGE},
|
||||
{
|
||||
"type": "response.output_text.delta",
|
||||
"sequence_number": 2,
|
||||
"item_id": "msg_once",
|
||||
"output_index": 0,
|
||||
"content_index": 0,
|
||||
"delta": "once",
|
||||
},
|
||||
{"type": "response.output_item.done", "sequence_number": 3, "output_index": 0, "item": _OUTPUT_MESSAGE},
|
||||
{"type": "response.completed", "sequence_number": 4, "response": _RESPONSES_REPLY},
|
||||
)
|
||||
|
||||
|
||||
def _sse(events: Sequence[Mapping[str, object]], *, done: bool) -> bytes:
|
||||
frames: Final = "".join(f"data: {json.dumps(event)}\n\n" for event in events)
|
||||
return (frames + ("data: [DONE]\n\n" if done else "")).encode()
|
||||
|
||||
|
||||
class _CallCounter(CustomLogger):
|
||||
pre_call: int = 0
|
||||
success: int = 0
|
||||
|
||||
async def async_pre_call_deployment_hook(self, kwargs: Mapping[str, object], call_type: object) -> None:
|
||||
self.pre_call += 1
|
||||
|
||||
async def async_log_success_event(
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
self.success += 1
|
||||
|
||||
|
||||
async def _messages(model: str, api_base: str, stream: bool) -> object:
|
||||
return await anthropic_messages(
|
||||
model=model, api_base=api_base, api_key="sk-x", max_tokens=10, messages=list(USER_TURN), stream=stream
|
||||
)
|
||||
|
||||
|
||||
async def _messages_to_chat(stream: bool) -> object:
|
||||
return await _messages(CHAT_MODEL, CHAT_BASE, stream)
|
||||
|
||||
|
||||
async def _messages_to_responses(stream: bool) -> object:
|
||||
return await _messages("openai/gpt-5.6-luna", RESPONSES_BASE, stream)
|
||||
|
||||
|
||||
async def _chat_to_responses(stream: bool) -> object:
|
||||
return await litellm.acompletion(
|
||||
model=RESPONSES_ONLY_MODEL, api_base=RESPONSES_BASE, api_key="sk-x", messages=list(USER_TURN), stream=stream
|
||||
)
|
||||
|
||||
|
||||
async def _responses_to_chat(stream: bool) -> object:
|
||||
return await litellm.aresponses(
|
||||
model="deepseek/served-model", api_base=CHAT_BASE, api_key="sk-x", input="hi", stream=stream
|
||||
)
|
||||
|
||||
|
||||
async def _drained(reply: object) -> None:
|
||||
if isinstance(reply, AsyncIterable):
|
||||
async for _ in cast(AsyncIterable[object], reply): # cast-ok: every bridged stream is drained the same way
|
||||
pass
|
||||
|
||||
|
||||
async def _settled(counter: _CallCounter) -> _CallCounter:
|
||||
for _ in range(200):
|
||||
if counter.success:
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
await asyncio.sleep(0.1)
|
||||
return counter
|
||||
|
||||
|
||||
_BRIDGES: Final = (
|
||||
pytest.param(_messages_to_chat, "chat", id="messages_to_chat"),
|
||||
pytest.param(_messages_to_responses, "responses", id="messages_to_responses"),
|
||||
pytest.param(_chat_to_responses, "responses", id="chat_to_responses"),
|
||||
pytest.param(_responses_to_chat, "chat", id="responses_to_chat"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
@pytest.mark.parametrize(("bridge", "upstream_api"), _BRIDGES)
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_bridged_request_runs_hooks_logs_and_calls_the_provider_once(
|
||||
bridge: Callable[[bool], Coroutine[object, object, object]],
|
||||
upstream_api: str,
|
||||
stream: bool,
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
counter: Final = _CallCounter()
|
||||
_isolate_model_cost(monkeypatch)
|
||||
monkeypatch.setattr(litellm, "callbacks", [counter])
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
litellm.register_model(
|
||||
{RESPONSES_ONLY_MODEL: {"litellm_provider": "openai", "mode": "responses", "max_tokens": 100}}
|
||||
)
|
||||
upstream: Final = (
|
||||
respx_mock.post(f"{CHAT_BASE}/chat/completions").mock(
|
||||
return_value=httpx.Response(
|
||||
200, headers={"content-type": "text/event-stream"}, content=_sse(_CHAT_CHUNKS, done=True)
|
||||
)
|
||||
if stream
|
||||
else httpx.Response(200, json=_CHAT_REPLY)
|
||||
)
|
||||
if upstream_api == "chat"
|
||||
else respx_mock.post(f"{RESPONSES_BASE}/responses").mock(
|
||||
return_value=httpx.Response(
|
||||
200, headers={"content-type": "text/event-stream"}, content=_sse(_RESPONSES_EVENTS, done=False)
|
||||
)
|
||||
if stream
|
||||
else httpx.Response(200, json=_RESPONSES_REPLY)
|
||||
)
|
||||
)
|
||||
|
||||
await _drained(await bridge(stream))
|
||||
settled: Final = await _settled(counter)
|
||||
|
||||
assert upstream.call_count == 1
|
||||
assert (settled.pre_call, settled.success) == (1, 1)
|
||||
|
|
@ -20,7 +20,7 @@ class TestGetFileContentsFromS3:
|
|||
"""Test suite for S3 config loading functionality."""
|
||||
|
||||
@patch("boto3.client")
|
||||
@patch("litellm.main.bedrock_converse_chat_completion")
|
||||
@patch("litellm.native.chat_providers.handlers.bedrock_converse_chat_completion")
|
||||
@patch("yaml.safe_load")
|
||||
def test_get_file_contents_from_s3_no_temp_file_creation(
|
||||
self, mock_yaml_load, mock_bedrock, mock_boto3_client
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import logging
|
|||
from collections.abc import Callable, Coroutine, Mapping
|
||||
from datetime import datetime
|
||||
from threading import Event
|
||||
from functools import partial
|
||||
from typing import Any, Final, cast
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
|
@ -28,7 +29,6 @@ import respx
|
|||
|
||||
|
||||
import litellm
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.litellm_core_utils.llm_cost_calc.finalization import finalize_cost
|
||||
from litellm.types.utils import StandardLoggingPayload
|
||||
|
|
@ -59,6 +59,7 @@ def _attach_mock_success_dispatch(mock_logging_obj, async_success_fn):
|
|||
|
||||
mock_logging_obj.dispatch_success_handlers = dispatch_success_handlers
|
||||
mock_logging_obj.async_success_handler = async_success_fn
|
||||
mock_logging_obj.hold_stream_completion = partial(Logging.hold_stream_completion, mock_logging_obj)
|
||||
|
||||
|
||||
async def _wait_until(condition: Callable[[], bool]) -> None:
|
||||
|
|
@ -593,6 +594,7 @@ class TestDeferredStreamingClosure:
|
|||
response args on logging_obj instead of calling the closure directly."""
|
||||
mock_logging_obj = MagicMock()
|
||||
mock_logging_obj._on_deferred_stream_complete = MagicMock()
|
||||
mock_logging_obj.hold_stream_completion = partial(Logging.hold_stream_completion, mock_logging_obj)
|
||||
|
||||
resp = await litellm.acompletion(
|
||||
model="gpt-3.5-turbo",
|
||||
|
|
@ -1388,6 +1390,7 @@ class TestResponsesIteratorDeferredLogging:
|
|||
|
||||
logging_obj = MagicMock()
|
||||
logging_obj.dispatch_success_handlers = dispatch_success_handlers
|
||||
logging_obj.hold_stream_completion = partial(Logging.hold_stream_completion, logging_obj)
|
||||
return logging_obj, recorded
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import datetime
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from itertools import chain
|
||||
|
|
@ -13,6 +14,7 @@ from pydantic import BaseModel
|
|||
import litellm
|
||||
from litellm import CustomLLM
|
||||
from litellm.constants import REDACTED_BY_LITELLM
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.responses.litellm_completion_transformation.handler import (
|
||||
LiteLLMCompletionTransformationHandler,
|
||||
|
|
@ -92,8 +94,8 @@ async def test_aresponses_forwards_timeout_to_acompletion():
|
|||
and similar providers, with calls falling back to the provider SDK default
|
||||
(~600s for Anthropic).
|
||||
"""
|
||||
with patch("litellm.acompletion", new_callable=AsyncMock) as mock_acompletion:
|
||||
mock_acompletion.return_value = ModelResponse(
|
||||
with patch("litellm.native.chat_plan.aexecute_planned_chat", new_callable=AsyncMock) as mock_chat:
|
||||
mock_chat.return_value = ModelResponse(
|
||||
id="id",
|
||||
created=0,
|
||||
model="anthropic/claude-sonnet-4-5",
|
||||
|
|
@ -108,8 +110,8 @@ async def test_aresponses_forwards_timeout_to_acompletion():
|
|||
api_key="sk-ant-fake",
|
||||
)
|
||||
|
||||
assert mock_acompletion.call_count == 1
|
||||
forwarded_timeout = mock_acompletion.call_args.kwargs.get("timeout")
|
||||
assert mock_chat.call_count == 1
|
||||
forwarded_timeout = mock_chat.call_args.args[0].named.get("timeout")
|
||||
assert forwarded_timeout == 42, (
|
||||
f"timeout was not forwarded to acompletion (got {forwarded_timeout!r}); "
|
||||
"this means Router(timeout=N) silently fails for providers on the "
|
||||
|
|
@ -1569,3 +1571,151 @@ async def test_previous_response_id_keeps_the_tools_an_earlier_codex_tool_search
|
|||
[("tool_result", "toolu_standup"), ("text", None)],
|
||||
]
|
||||
assert prisma_client.db.queries == 1
|
||||
class _DeploymentHookCounter(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.calls = 0
|
||||
|
||||
async def async_pre_call_deployment_hook(self, kwargs: Mapping[str, object], call_type: object) -> None:
|
||||
self.calls += 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridged_responses_call_runs_deployment_hooks_once(monkeypatch: pytest.MonkeyPatch):
|
||||
counter: Final = _DeploymentHookCounter()
|
||||
monkeypatch.setattr(litellm, "callbacks", [counter])
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
messages_route: Final = upstream.post("http://anthropic.test/v1/messages").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "msg_bridge",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-bridge",
|
||||
"content": [{"type": "text", "text": "from upstream"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 10, "output_tokens": 5},
|
||||
},
|
||||
)
|
||||
)
|
||||
response: Final = await litellm.aresponses(
|
||||
model="anthropic/claude-bridge", input="hi", api_base="http://anthropic.test", api_key="sk-test"
|
||||
)
|
||||
|
||||
assert response.output[0].content[0].text == "from upstream"
|
||||
assert messages_route.call_count == 1
|
||||
assert counter.calls == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridged_responses_call_is_billed_like_the_chat_call_for_one_hour_cache_writes_and_web_searches(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
usage: Final = {
|
||||
"input_tokens": 100,
|
||||
"output_tokens": 50,
|
||||
"cache_creation_input_tokens": 2000,
|
||||
"cache_read_input_tokens": 3000,
|
||||
"cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2000},
|
||||
"server_tool_use": {"web_search_requests": 2},
|
||||
}
|
||||
upstream_message: Final = {
|
||||
"id": "msg_bridge_cost",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-4-5",
|
||||
"content": [{"type": "text", "text": "searched"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": usage,
|
||||
}
|
||||
credentials: Final = {"api_base": "http://anthropic.test", "api_key": "sk-test"}
|
||||
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
upstream.post("http://anthropic.test/v1/messages").mock(return_value=httpx.Response(200, json=upstream_message))
|
||||
chat: Final = await litellm.acompletion(
|
||||
model="anthropic/claude-sonnet-4-5", messages=[{"role": "user", "content": "hi"}], **credentials
|
||||
)
|
||||
bridged: Final = await litellm.aresponses(model="anthropic/claude-sonnet-4-5", input="hi", **credentials)
|
||||
|
||||
chat_cost: Final = chat._hidden_params["response_cost"]
|
||||
assert isinstance(chat_cost, float) and chat_cost > 0
|
||||
assert bridged._hidden_params["response_cost"] == pytest.approx(chat_cost)
|
||||
|
||||
|
||||
_ANTHROPIC_MESSAGE: Final = {
|
||||
"id": "msg_bridge_headers",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-4-5",
|
||||
"content": [{"type": "text", "text": "hi"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 10, "output_tokens": 5},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridged_responses_call_names_the_provider_request_id_in_its_headers(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
upstream.post("http://anthropic.test/v1/messages").mock(
|
||||
return_value=httpx.Response(200, json=_ANTHROPIC_MESSAGE, headers={"x-request-id": "req-upstream-1"})
|
||||
)
|
||||
bridged: Final = await litellm.aresponses(
|
||||
model="anthropic/claude-sonnet-4-5", input="hi", api_base="http://anthropic.test", api_key="sk-test"
|
||||
)
|
||||
|
||||
assert bridged._hidden_params["additional_headers"]["llm_provider-request-id"] == "req-upstream-1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridged_responses_stream_opens_with_the_cost_breakdown_headers(monkeypatch: pytest.MonkeyPatch):
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.proxy.common_request_processing import cost_headers
|
||||
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
logging_obj: Final = Logging(
|
||||
model="anthropic/claude-sonnet-4-5",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
call_type="aresponses",
|
||||
start_time=datetime.datetime.now(datetime.timezone.utc),
|
||||
litellm_call_id="bridge-stream-headers-0000-4000-8000-000000000001",
|
||||
function_id="1234",
|
||||
)
|
||||
events: Final = (
|
||||
{"type": "message_start", "message": {**_ANTHROPIC_MESSAGE, "content": [], "stop_reason": None}},
|
||||
{"type": "message_stop"},
|
||||
)
|
||||
sse: Final = "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events)
|
||||
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
upstream.post("http://anthropic.test/v1/messages").mock(
|
||||
return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, text=sse)
|
||||
)
|
||||
stream: Final = await litellm.aresponses(
|
||||
model="anthropic/claude-sonnet-4-5",
|
||||
input="hi",
|
||||
stream=True,
|
||||
api_base="http://anthropic.test",
|
||||
api_key="sk-test",
|
||||
litellm_logging_obj=logging_obj,
|
||||
)
|
||||
headers: Final = cost_headers(None, logging_obj)
|
||||
async for _ in stream:
|
||||
pass
|
||||
|
||||
assert headers["x-litellm-response-cost-output"] == "0.0"
|
||||
assert headers["x-litellm-response-cost-discount-amount"] == "0.0"
|
||||
assert headers["x-litellm-response-cost-margin-amount"] == "0.0"
|
||||
|
|
|
|||
|
|
@ -23,16 +23,14 @@ async def test_mock_basic_google_ai_studio_responses_api_with_tools():
|
|||
choices=[
|
||||
litellm.utils.Choices(
|
||||
index=0,
|
||||
message=litellm.utils.Message(
|
||||
role="assistant", content="Test response"
|
||||
),
|
||||
message=litellm.utils.Message(role="assistant", content="Test response"),
|
||||
finish_reason="stop",
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
with patch("litellm.acompletion", new_callable=AsyncMock) as mock_acompletion:
|
||||
mock_acompletion.return_value = mock_response
|
||||
with patch("litellm.native.chat_plan.aexecute_planned_chat", new_callable=AsyncMock) as mock_chat:
|
||||
mock_chat.return_value = mock_response
|
||||
|
||||
request_model = "gemini/gemini-2.5-flash"
|
||||
await litellm.aresponses(
|
||||
|
|
@ -41,11 +39,10 @@ async def test_mock_basic_google_ai_studio_responses_api_with_tools():
|
|||
tools=[{"type": "web_search_preview", "search_context_size": "low"}],
|
||||
)
|
||||
|
||||
# Verify that acompletion was called
|
||||
assert mock_acompletion.called
|
||||
assert mock_chat.called
|
||||
|
||||
# Get the call arguments
|
||||
call_args, call_kwargs = mock_acompletion.call_args
|
||||
plan = mock_chat.call_args.args[0]
|
||||
call_kwargs = {**plan.kwargs, **plan.named}
|
||||
|
||||
# Verify the expected parameters were passed
|
||||
print(
|
||||
|
|
@ -65,5 +62,5 @@ async def test_mock_basic_google_ai_studio_responses_api_with_tools():
|
|||
call_kwargs["messages"][0]["content"]
|
||||
== "what is the latest version of supabase python package and when was it released?"
|
||||
)
|
||||
assert "tools" not in call_kwargs
|
||||
assert "tool_choice" not in call_kwargs
|
||||
assert call_kwargs.get("tools") is None
|
||||
assert call_kwargs.get("tool_choice") is None
|
||||
|
|
|
|||
|
|
@ -1,31 +1,24 @@
|
|||
"""Regression tests for the responses -> completion fallback bridge guard.
|
||||
|
||||
When the Responses API falls back to chat completions (no native responses
|
||||
config), it must tag the forwarded ``litellm.completion`` / ``litellm.acompletion``
|
||||
call with ``_skip_responses_api_bridge=True`` so ``completion()`` does not bridge
|
||||
the request straight back to the Responses API and mutually recurse forever.
|
||||
|
||||
Both fallback paths are covered: the sync ``response_api_handler`` (``_is_async``
|
||||
False) and the async ``async_response_api_handler`` (``_is_async`` True). The
|
||||
module-level ``litellm.completion`` / ``litellm.acompletion`` are patched to
|
||||
capture the forwarded kwargs; if the flag-setting line is removed the captured
|
||||
kwargs lack the flag and these tests fail.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import math
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import datetime
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
from unittest.mock import patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.responses.litellm_completion_transformation.handler import (
|
||||
LiteLLMCompletionTransformationHandler,
|
||||
)
|
||||
from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD
|
||||
|
|
@ -35,50 +28,6 @@ class _StopForwarding(Exception):
|
|||
"""Raised by the mocked (a)completion once the forwarded kwargs are captured."""
|
||||
|
||||
|
||||
def test_sync_fallback_tags_skip_responses_api_bridge():
|
||||
handler = LiteLLMCompletionTransformationHandler()
|
||||
captured: dict = {}
|
||||
|
||||
def fake_completion(**kwargs):
|
||||
captured.update(kwargs)
|
||||
raise _StopForwarding()
|
||||
|
||||
with patch("litellm.completion", fake_completion):
|
||||
with pytest.raises(_StopForwarding):
|
||||
handler.response_api_handler(
|
||||
model="gpt-4o",
|
||||
input="hello",
|
||||
responses_api_request={},
|
||||
custom_llm_provider="openai",
|
||||
_is_async=False,
|
||||
)
|
||||
|
||||
assert captured.get("_skip_responses_api_bridge") is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_fallback_tags_skip_responses_api_bridge():
|
||||
handler = LiteLLMCompletionTransformationHandler()
|
||||
captured: dict = {}
|
||||
|
||||
async def fake_acompletion(**kwargs):
|
||||
captured.update(kwargs)
|
||||
raise _StopForwarding()
|
||||
|
||||
with patch("litellm.acompletion", fake_acompletion):
|
||||
coro = handler.response_api_handler(
|
||||
model="gpt-4o",
|
||||
input="hello",
|
||||
responses_api_request={},
|
||||
custom_llm_provider="openai",
|
||||
_is_async=True,
|
||||
)
|
||||
with pytest.raises(_StopForwarding):
|
||||
await coro
|
||||
|
||||
assert captured.get("_skip_responses_api_bridge") is True
|
||||
|
||||
|
||||
_CODEX_ADDITIONAL_TOOLS_ITEM = {
|
||||
"type": "additional_tools",
|
||||
"id": "at_codex",
|
||||
|
|
@ -303,10 +252,88 @@ async def test_bridged_next_turn_replays_a_turn_stored_under_the_id_the_client_h
|
|||
)
|
||||
|
||||
assert provider.request_body is not None, "the bridged turn never reached the provider"
|
||||
assert [
|
||||
(message["role"], json.dumps(message["content"])) for message in provider.request_body["messages"]
|
||||
] == [
|
||||
assert [(message["role"], json.dumps(message["content"])) for message in provider.request_body["messages"]] == [
|
||||
("user", json.dumps([{"type": "text", "text": "My favorite number is 7."}])),
|
||||
("assistant", json.dumps([{"type": "text", "text": "Noted."}])),
|
||||
("user", json.dumps([{"type": "text", "text": "Double it"}])),
|
||||
]
|
||||
|
||||
|
||||
_LOGGED_ROW: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
class _SuccessRowRecorder(CustomLogger):
|
||||
rows: tuple[dict[str, object], ...] = ()
|
||||
|
||||
async def async_log_success_event(
|
||||
self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime
|
||||
) -> None:
|
||||
self.rows = (*self.rows, _LOGGED_ROW.validate_python(kwargs["standard_logging_object"]))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridged_responses_stream_logs_one_responses_row_billed_from_provider_usage(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
recorder: Final = _SuccessRowRecorder()
|
||||
monkeypatch.setattr(litellm, "callbacks", [recorder])
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
prices: Final = {"input_cost_per_token": 1e-6, "output_cost_per_token": 4e-6, "cache_read_input_token_cost": 1e-7}
|
||||
litellm.register_model(
|
||||
{"deepseek/bridge-stream-model": {"litellm_provider": "deepseek", "mode": "chat", "max_tokens": 100, **prices}}
|
||||
)
|
||||
chunk: Final[dict[str, object]] = {"id": "chatcmpl-stream", "object": "chat.completion.chunk", "created": 1}
|
||||
chunks: Final[tuple[dict[str, object], ...]] = (
|
||||
{**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "streamed"}}]},
|
||||
{**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]},
|
||||
{
|
||||
**chunk,
|
||||
"choices": [],
|
||||
"usage": {
|
||||
"prompt_tokens": 100,
|
||||
"completion_tokens": 20,
|
||||
"total_tokens": 120,
|
||||
"prompt_tokens_details": {"cached_tokens": 40},
|
||||
},
|
||||
},
|
||||
)
|
||||
sse: Final = ("".join(f"data: {json.dumps(c)}\n\n" for c in chunks) + "data: [DONE]\n\n").encode()
|
||||
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
chat_route: Final = upstream.post("https://example.invalid/v1/chat/completions").mock(
|
||||
return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, content=sse)
|
||||
)
|
||||
stream: Final = await litellm.aresponses(
|
||||
model="deepseek/bridge-stream-model",
|
||||
api_base="https://example.invalid/v1",
|
||||
api_key="x",
|
||||
input="hi",
|
||||
stream=True,
|
||||
)
|
||||
assert isinstance(stream, ResponsesAPIStreamingIterator)
|
||||
async for _ in stream:
|
||||
pass
|
||||
for _ in range(100):
|
||||
if recorder.rows:
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
assert stream.completed_response is not None
|
||||
assert chat_route.call_count == 1
|
||||
assert len(recorder.rows) == 1
|
||||
row: Final = recorder.rows[0]
|
||||
logged: Final = _LOGGED_ROW.validate_python(row["response"])
|
||||
assert logged["object"] == "response"
|
||||
assert logged["status"] == "completed"
|
||||
assert row["prompt_tokens"] == 100
|
||||
assert row["completion_tokens"] == 20
|
||||
cost: Final = row["response_cost"]
|
||||
assert isinstance(cost, float)
|
||||
assert math.isclose(
|
||||
cost,
|
||||
60 * prices["input_cost_per_token"]
|
||||
+ 40 * prices["cache_read_input_token_cost"]
|
||||
+ 20 * prices["output_cost_per_token"],
|
||||
)
|
||||
|
|
|
|||
|
|
@ -23,33 +23,20 @@ class TestExtractBase64FromDataUrl:
|
|||
def test_extracts_base64_from_data_url(self):
|
||||
"""Should extract pure base64 from data URL with prefix"""
|
||||
data_url = "data:image/png;base64,iVBORw0KGgoAAAANS"
|
||||
result = LiteLLMCompletionResponsesConfig._extract_base64_from_data_url(
|
||||
data_url
|
||||
)
|
||||
result = LiteLLMCompletionResponsesConfig._extract_base64_from_data_url(data_url)
|
||||
assert result == "iVBORw0KGgoAAAANS"
|
||||
|
||||
def test_returns_base64_as_is_if_no_prefix(self):
|
||||
"""Should return base64 as-is if no data: prefix"""
|
||||
pure_base64 = "iVBORw0KGgoAAAANS"
|
||||
result = LiteLLMCompletionResponsesConfig._extract_base64_from_data_url(
|
||||
pure_base64
|
||||
)
|
||||
result = LiteLLMCompletionResponsesConfig._extract_base64_from_data_url(pure_base64)
|
||||
assert result == pure_base64
|
||||
|
||||
def test_handles_invalid_inputs(self):
|
||||
"""Should return None for empty/None/malformed inputs"""
|
||||
assert (
|
||||
LiteLLMCompletionResponsesConfig._extract_base64_from_data_url("") is None
|
||||
)
|
||||
assert (
|
||||
LiteLLMCompletionResponsesConfig._extract_base64_from_data_url(None) is None
|
||||
)
|
||||
assert (
|
||||
LiteLLMCompletionResponsesConfig._extract_base64_from_data_url(
|
||||
"data:image/png;base64"
|
||||
)
|
||||
is None
|
||||
)
|
||||
assert LiteLLMCompletionResponsesConfig._extract_base64_from_data_url("") is None
|
||||
assert LiteLLMCompletionResponsesConfig._extract_base64_from_data_url(None) is None
|
||||
assert LiteLLMCompletionResponsesConfig._extract_base64_from_data_url("data:image/png;base64") is None
|
||||
|
||||
|
||||
class TestExtractImageGenerationOutputItems:
|
||||
|
|
@ -75,10 +62,8 @@ class TestExtractImageGenerationOutputItems:
|
|||
mock_choice.message = mock_message
|
||||
mock_choice.finish_reason = "stop"
|
||||
|
||||
result = (
|
||||
LiteLLMCompletionResponsesConfig._extract_image_generation_output_items(
|
||||
choice=mock_choice,
|
||||
)
|
||||
result = LiteLLMCompletionResponsesConfig._extract_image_generation_output_items(
|
||||
choice=mock_choice,
|
||||
)
|
||||
|
||||
assert len(result) == 2
|
||||
|
|
@ -99,10 +84,8 @@ class TestExtractImageGenerationOutputItems:
|
|||
mock_choice.message = mock_message
|
||||
mock_choice.finish_reason = "stop"
|
||||
|
||||
result = (
|
||||
LiteLLMCompletionResponsesConfig._extract_image_generation_output_items(
|
||||
choice=mock_choice,
|
||||
)
|
||||
result = LiteLLMCompletionResponsesConfig._extract_image_generation_output_items(
|
||||
choice=mock_choice,
|
||||
)
|
||||
|
||||
assert result == []
|
||||
|
|
@ -122,10 +105,8 @@ class TestExtractImageGenerationOutputItems:
|
|||
mock_choice.message = mock_message
|
||||
mock_choice.finish_reason = "length"
|
||||
|
||||
result = (
|
||||
LiteLLMCompletionResponsesConfig._extract_image_generation_output_items(
|
||||
choice=mock_choice,
|
||||
)
|
||||
result = LiteLLMCompletionResponsesConfig._extract_image_generation_output_items(
|
||||
choice=mock_choice,
|
||||
)
|
||||
|
||||
assert result[0].status == "incomplete"
|
||||
|
|
@ -200,10 +181,7 @@ class TestImageGenerationOutputItemIds:
|
|||
|
||||
def _choice_with_images(self, count):
|
||||
mock_message = Mock(spec=Message)
|
||||
mock_message.images = [
|
||||
{"image_url": {"url": f"data:image/png;base64,IMG{idx}"}}
|
||||
for idx in range(count)
|
||||
]
|
||||
mock_message.images = [{"image_url": {"url": f"data:image/png;base64,IMG{idx}"}} for idx in range(count)]
|
||||
mock_choice = Mock(spec=Choices)
|
||||
mock_choice.message = mock_message
|
||||
mock_choice.finish_reason = "stop"
|
||||
|
|
|
|||
|
|
@ -178,16 +178,12 @@ class TestEncryptedReasoningRoundTrip:
|
|||
item = {
|
||||
"type": "reasoning",
|
||||
"id": "rs_1",
|
||||
"encrypted_content": json.dumps(
|
||||
[{"type": "thinking", "thinking": "hidden", "signature": "sig-one"}]
|
||||
),
|
||||
"encrypted_content": json.dumps([{"type": "thinking", "thinking": "hidden", "signature": "sig-one"}]),
|
||||
}
|
||||
messages = _transform_item(item)
|
||||
assert len(messages) == 1
|
||||
assert messages[0]["content"] is None
|
||||
assert messages[0]["thinking_blocks"] == [
|
||||
{"type": "thinking", "thinking": "hidden", "signature": "sig-one"}
|
||||
]
|
||||
assert messages[0]["thinking_blocks"] == [{"type": "thinking", "thinking": "hidden", "signature": "sig-one"}]
|
||||
|
||||
def test_unsigned_blocks_dropped(self):
|
||||
"""Blocks without a signature or redacted payload are not replayed."""
|
||||
|
|
@ -229,9 +225,7 @@ class TestEncryptedReasoningRoundTrip:
|
|||
)
|
||||
assert len(messages) == 1
|
||||
assert messages[0]["reasoning_content"] == "look it up"
|
||||
assert messages[0]["thinking_blocks"] == [
|
||||
{"type": "thinking", "thinking": "hidden", "signature": "sig-one"}
|
||||
]
|
||||
assert messages[0]["thinking_blocks"] == [{"type": "thinking", "thinking": "hidden", "signature": "sig-one"}]
|
||||
assert len(messages[0]["tool_calls"]) == 1
|
||||
|
||||
def test_replayed_blocks_precede_existing_blocks(self):
|
||||
|
|
|
|||
|
|
@ -58,5 +58,7 @@ def test_a_reasoning_item_is_litellm_minted_by_its_id_or_by_its_encoded_thinking
|
|||
"encrypted_content": A_PROVIDER_OWNED_ENCRYPTED_BLOB,
|
||||
}
|
||||
)
|
||||
assert not is_litellm_minted_reasoning_item({"type": "message", "id": mint_reasoning_item_id(), "role": "assistant"})
|
||||
assert not is_litellm_minted_reasoning_item(
|
||||
{"type": "message", "id": mint_reasoning_item_id(), "role": "assistant"}
|
||||
)
|
||||
assert not is_litellm_minted_reasoning_item("a bare string input")
|
||||
|
|
|
|||
|
|
@ -1245,3 +1245,134 @@ async def test_session_lookup_follows_the_owner_across_keys_and_keeps_ownerless_
|
|||
|
||||
assert await _history_request_ids(sqlite_db, "alice-new-key") == ["alice-old-key", "alice-new-key"]
|
||||
assert await _history_request_ids(sqlite_db, "ownerless-2") == ["ownerless-2"]
|
||||
|
||||
|
||||
def _text(content: object) -> object:
|
||||
if isinstance(content, list):
|
||||
return "".join(part["text"] for part in content)
|
||||
return content
|
||||
|
||||
|
||||
def _responses_spend_log(request_id: str, session_id: str, prompt: str, output: list[dict[str, object]]) -> dict:
|
||||
return {
|
||||
"request_id": request_id,
|
||||
"call_type": "aresponses",
|
||||
"session_id": session_id,
|
||||
"proxy_server_request": {"input": [{"role": "user", "content": prompt}], "model": "claude-bridge"},
|
||||
"response": {"id": request_id, "object": "response", "status": "completed", "output": output},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_message_history_reads_responses_shaped_rows_next_to_chat_shaped_rows():
|
||||
session_id = "0f6f2d8e-6c8a-4b55-9a39-3d0b1f3c9e11"
|
||||
fake_prisma_client = _FakePrismaClient(
|
||||
results=[
|
||||
[
|
||||
_spend_log("chatcmpl-3333", session_id, "My favorite color is chartreuse.", "Got it."),
|
||||
_responses_spend_log(
|
||||
"resp_4444",
|
||||
session_id,
|
||||
"What is the weather in Lisbon?",
|
||||
[
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_1",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "text": "Let me check.", "annotations": []}],
|
||||
},
|
||||
{
|
||||
"type": "function_call",
|
||||
"id": "fc_1",
|
||||
"call_id": "call_weather",
|
||||
"name": "get_weather",
|
||||
"arguments": '{"city": "Lisbon"}',
|
||||
},
|
||||
],
|
||||
),
|
||||
]
|
||||
]
|
||||
)
|
||||
|
||||
with patch("litellm.proxy.proxy_server.prisma_client", fake_prisma_client):
|
||||
result = await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id("resp_4444")
|
||||
|
||||
messages: Final = result["messages"]
|
||||
assert [(message.get("role"), _text(message.get("content"))) for message in messages[:4]] == [
|
||||
("user", "My favorite color is chartreuse."),
|
||||
("assistant", "Got it."),
|
||||
("user", "What is the weather in Lisbon?"),
|
||||
("assistant", "Let me check."),
|
||||
]
|
||||
tool_calls: Final = [call for message in messages[3:] for call in message.get("tool_calls") or []]
|
||||
assert [(call["id"], call["function"]["name"], call["function"]["arguments"]) for call in tool_calls] == [
|
||||
("call_weather", "get_weather", '{"city": "Lisbon"}')
|
||||
]
|
||||
|
||||
|
||||
_WEB_SEARCH_RESULT: Final = {
|
||||
"type": "web_search_tool_result",
|
||||
"tool_use_id": "srvtoolu_1",
|
||||
"content": [{"type": "web_search_result", "url": "https://x.com", "title": "X", "encrypted_content": "EC"}],
|
||||
}
|
||||
|
||||
|
||||
def _bridged_anthropic_web_search_turn() -> dict[str, object]:
|
||||
"""A Claude turn served through the Responses to chat bridge, stored as its Responses object."""
|
||||
return {
|
||||
"id": "resp_1",
|
||||
"object": "response",
|
||||
"output": [
|
||||
{
|
||||
"id": "ws_srvtoolu_1",
|
||||
"type": "web_search_call",
|
||||
"status": "completed",
|
||||
"action": {
|
||||
"type": "search",
|
||||
"query": "weather sf",
|
||||
"sources": [{"type": "url", "url": "https://x.com"}],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_1",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "text": "It is sunny.", "annotations": []}],
|
||||
},
|
||||
],
|
||||
"provider_specific_fields": {"web_search_results": [_WEB_SEARCH_RESULT]},
|
||||
}
|
||||
|
||||
|
||||
def test_message_history_replays_a_bridged_anthropic_web_search_as_server_tool_blocks():
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
|
||||
|
||||
history: Final = ResponsesSessionHandler.extend_chat_completion_message_with_spend_log_payload(
|
||||
spend_log={"request_id": "resp_1", "response": _bridged_anthropic_web_search_turn()},
|
||||
proxy_server_request_dict={"input": "weather in sf?"},
|
||||
chat_completion_message_history=[],
|
||||
)
|
||||
anthropic_messages: Final = anthropic_messages_pt(
|
||||
messages=history, model="claude-sonnet-4-5", llm_provider="anthropic"
|
||||
)
|
||||
|
||||
assert [message["role"] for message in anthropic_messages] == ["user", "assistant"]
|
||||
assistant_content: Final = anthropic_messages[1]["content"]
|
||||
assert [block["type"] for block in assistant_content] == ["text", "server_tool_use", "web_search_tool_result"]
|
||||
assert assistant_content[1]["id"] == "srvtoolu_1"
|
||||
assert assistant_content[1]["input"] == {"query": "weather sf"}
|
||||
assert assistant_content[2]["content"] == _WEB_SEARCH_RESULT["content"]
|
||||
|
||||
|
||||
def test_message_history_keeps_a_web_search_without_a_stored_result_as_text():
|
||||
turn: Final = {**_bridged_anthropic_web_search_turn(), "provider_specific_fields": None}
|
||||
|
||||
history: Final = ResponsesSessionHandler.extend_chat_completion_message_with_spend_log_payload(
|
||||
spend_log={"request_id": "resp_1", "response": turn},
|
||||
proxy_server_request_dict=None,
|
||||
chat_completion_message_history=[],
|
||||
)
|
||||
|
||||
assert str(_text_of(history[0]["content"])).startswith("Hosted web search: ")
|
||||
|
|
|
|||
|
|
@ -78,9 +78,7 @@ class TestColdStorageObjectKeyIntegration:
|
|||
metadata = {"user_api_key": "test_key", "user_api_key_team_id": "test_team"}
|
||||
|
||||
# Call the function
|
||||
result = _get_spend_logs_metadata(
|
||||
metadata=metadata, cold_storage_object_key="test/path/to/object.json"
|
||||
)
|
||||
result = _get_spend_logs_metadata(metadata=metadata, cold_storage_object_key="test/path/to/object.json")
|
||||
|
||||
# Verify the object key is stored in the result
|
||||
assert result.get("cold_storage_object_key") == "test/path/to/object.json"
|
||||
|
|
@ -104,11 +102,7 @@ class TestColdStorageObjectKeyIntegration:
|
|||
}
|
||||
|
||||
# Test the extraction method
|
||||
object_key = (
|
||||
ResponsesSessionHandler._get_cold_storage_object_key_from_spend_log(
|
||||
spend_log
|
||||
)
|
||||
)
|
||||
object_key = ResponsesSessionHandler._get_cold_storage_object_key_from_spend_log(spend_log)
|
||||
|
||||
assert object_key == "test/path/to/object.json"
|
||||
|
||||
|
|
@ -128,11 +122,7 @@ class TestColdStorageObjectKeyIntegration:
|
|||
}
|
||||
|
||||
# Test the extraction method
|
||||
object_key = (
|
||||
ResponsesSessionHandler._get_cold_storage_object_key_from_spend_log(
|
||||
spend_log
|
||||
)
|
||||
)
|
||||
object_key = ResponsesSessionHandler._get_cold_storage_object_key_from_spend_log(spend_log)
|
||||
|
||||
assert object_key == "test/path/to/object.json"
|
||||
|
||||
|
|
@ -148,20 +138,17 @@ class TestColdStorageObjectKeyIntegration:
|
|||
|
||||
# Mock the custom logger
|
||||
mock_logger = AsyncMock()
|
||||
mock_logger.get_proxy_server_request_from_cold_storage_with_object_key = (
|
||||
AsyncMock(return_value={"test": "data"})
|
||||
mock_logger.get_proxy_server_request_from_cold_storage_with_object_key = AsyncMock(
|
||||
return_value={"test": "data"}
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
handler, "_select_custom_logger_for_cold_storage", return_value="s3_v2"
|
||||
),
|
||||
patch.object(handler, "_select_custom_logger_for_cold_storage", return_value="s3_v2"),
|
||||
patch(
|
||||
"litellm.logging_callback_manager.get_active_custom_logger_for_callback_name",
|
||||
return_value=mock_logger,
|
||||
),
|
||||
):
|
||||
|
||||
result = await handler.get_proxy_server_request_from_cold_storage_with_object_key(
|
||||
object_key="test/path/to/object.json"
|
||||
)
|
||||
|
|
@ -184,9 +171,7 @@ class TestColdStorageObjectKeyIntegration:
|
|||
s3_logger = S3Logger(s3_bucket_name="test-bucket")
|
||||
|
||||
# Mock the _download_object_from_s3 method
|
||||
with patch.object(
|
||||
s3_logger, "_download_object_from_s3", return_value={"test": "data"}
|
||||
) as mock_download:
|
||||
with patch.object(s3_logger, "_download_object_from_s3", return_value={"test": "data"}) as mock_download:
|
||||
result = await s3_logger.get_proxy_server_request_from_cold_storage_with_object_key(
|
||||
object_key="test/path/to/object.json"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -38,9 +38,7 @@ from litellm.types.utils import (
|
|||
)
|
||||
|
||||
CHAT_COMPLETION_ID = "chatcmpl-77d33d09-effa-4cd2-9c0d-c742d4358256"
|
||||
RESPONSE_ID_EVENT_TYPES = frozenset(
|
||||
{"response.created", "response.in_progress", "response.completed"}
|
||||
)
|
||||
RESPONSE_ID_EVENT_TYPES = frozenset({"response.created", "response.in_progress", "response.completed"})
|
||||
|
||||
|
||||
def _chunk(content: str, finish_reason: str | None = None) -> ModelResponseStream:
|
||||
|
|
@ -93,11 +91,7 @@ def _build_iterator(chunks) -> LiteLLMCompletionStreamingIterator:
|
|||
|
||||
|
||||
def _response_ids(events) -> list[str]:
|
||||
return [
|
||||
event.response.id
|
||||
for event in events
|
||||
if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES
|
||||
]
|
||||
return [event.response.id for event in events if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES]
|
||||
|
||||
|
||||
def test_tool_call_delta_is_emitted_as_responses_events():
|
||||
|
|
@ -228,9 +222,7 @@ async def test_web_search_stream_preserves_hosted_and_client_calls(sync_mode, to
|
|||
content=None,
|
||||
provider_specific_fields={
|
||||
"web_search_results": [first_result],
|
||||
"web_search_calls": [
|
||||
build_web_search_call(call_ids[0], {"query": "one"}, first_result)
|
||||
]
|
||||
"web_search_calls": [build_web_search_call(call_ids[0], {"query": "one"}, first_result)]
|
||||
if tool_type != "function" and first_result["type"] == "web_search_tool_result"
|
||||
else [],
|
||||
},
|
||||
|
|
@ -329,9 +321,7 @@ async def test_web_search_stream_preserves_hosted_and_client_calls(sync_mode, to
|
|||
assert {event["output_index"] for event in search_events} == {completed_indexes[item["id"]]}
|
||||
assert search_events[0]["item"]["status"] == "in_progress"
|
||||
assert search_events[-1]["item"] == item
|
||||
assert item["status"] == (
|
||||
"failed" if result_kind == "error" and call_id.endswith("01Search") else "completed"
|
||||
)
|
||||
assert item["status"] == ("failed" if result_kind == "error" and call_id.endswith("01Search") else "completed")
|
||||
assert item["action"]["type"] == "search"
|
||||
assert item["action"]["query"] == ("one" if call_id.endswith("01Search") else "two")
|
||||
assert item["action"]["queries"] == [item["action"]["query"]]
|
||||
|
|
@ -440,9 +430,7 @@ def test_tool_call_arguments_are_chunked_to_match_openai_behavior():
|
|||
)
|
||||
|
||||
# Create a chunk with a large arguments string that should be split
|
||||
large_arguments = (
|
||||
'{"param1": "value1", "param2": "value2", "param3": "value3"}' # 67 chars
|
||||
)
|
||||
large_arguments = '{"param1": "value1", "param2": "value2", "param3": "value3"}' # 67 chars
|
||||
chunk = ModelResponseStream(
|
||||
id="chunk-1",
|
||||
created=123,
|
||||
|
|
@ -497,9 +485,7 @@ def test_tool_call_arguments_are_chunked_to_match_openai_behavior():
|
|||
|
||||
# Keep draining pending events (expected: ceil(67 / 10) = 7 delta events)
|
||||
while iterator._pending_tool_events:
|
||||
evt = iterator._transform_chat_completion_chunk_to_response_api_chunk(
|
||||
empty_chunk
|
||||
)
|
||||
evt = iterator._transform_chat_completion_chunk_to_response_api_chunk(empty_chunk)
|
||||
if evt and evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA:
|
||||
delta_events.append(evt)
|
||||
|
||||
|
|
@ -552,20 +538,12 @@ def test_tool_call_delta_without_id_uses_index_mapping():
|
|||
while iterator._pending_tool_events:
|
||||
all_events.append(iterator._pending_tool_events.pop(0))
|
||||
|
||||
delta_events = [
|
||||
evt
|
||||
for evt in all_events
|
||||
if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA
|
||||
]
|
||||
delta_events = [evt for evt in all_events if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA]
|
||||
streamed_arguments = "".join(evt.delta for evt in delta_events)
|
||||
|
||||
assert streamed_arguments == '{"location": "New York"}'
|
||||
|
||||
output_item_added_events = [
|
||||
evt
|
||||
for evt in all_events
|
||||
if evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED
|
||||
]
|
||||
output_item_added_events = [evt for evt in all_events if evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED]
|
||||
assert len(output_item_added_events) == 1
|
||||
assert output_item_added_events[0].item.id == "fc_call_abc123"
|
||||
assert output_item_added_events[0].item.call_id == "call_abc123"
|
||||
|
|
@ -606,18 +584,10 @@ def test_parallel_tool_calls_without_ids_use_index_mapping():
|
|||
while iterator._pending_tool_events:
|
||||
all_events.append(iterator._pending_tool_events.pop(0))
|
||||
|
||||
output_item_added_events = [
|
||||
evt
|
||||
for evt in all_events
|
||||
if evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED
|
||||
]
|
||||
output_item_added_events = [evt for evt in all_events if evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED]
|
||||
assert len(output_item_added_events) == 2
|
||||
|
||||
delta_events = [
|
||||
evt
|
||||
for evt in all_events
|
||||
if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA
|
||||
]
|
||||
delta_events = [evt for evt in all_events if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA]
|
||||
arguments_by_call_id = {}
|
||||
for evt in delta_events:
|
||||
arguments_by_call_id.setdefault(evt.item_id, "")
|
||||
|
|
@ -670,11 +640,7 @@ def test_reused_index_with_new_call_id_marks_fallback_ambiguous():
|
|||
while iterator._pending_tool_events:
|
||||
all_events.append(iterator._pending_tool_events.pop(0))
|
||||
|
||||
delta_events = [
|
||||
evt
|
||||
for evt in all_events
|
||||
if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA
|
||||
]
|
||||
delta_events = [evt for evt in all_events if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA]
|
||||
arguments_by_call_id = {}
|
||||
for evt in delta_events:
|
||||
arguments_by_call_id.setdefault(evt.item_id, "")
|
||||
|
|
@ -741,22 +707,17 @@ def test_sync_streaming_events_share_the_chat_completion_response_id():
|
|||
assert len(response_ids) == 3
|
||||
assert len(set(response_ids)) == 1
|
||||
assert (
|
||||
ResponsesAPIRequestUtils.decode_responses_api_response_id(response_ids[0])["response_id"]
|
||||
== CHAT_COMPLETION_ID
|
||||
ResponsesAPIRequestUtils.decode_responses_api_response_id(response_ids[0])["response_id"] == CHAT_COMPLETION_ID
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_emits_every_chunk_after_priming_the_response_id():
|
||||
iterator = _build_iterator(
|
||||
[_chunk("Hel"), _chunk("lo"), _chunk("!", finish_reason="stop")]
|
||||
)
|
||||
iterator = _build_iterator([_chunk("Hel"), _chunk("lo"), _chunk("!", finish_reason="stop")])
|
||||
|
||||
events = [event async for event in iterator]
|
||||
|
||||
deltas = "".join(
|
||||
event.delta for event in events if getattr(event, "type", None) == "response.output_text.delta"
|
||||
)
|
||||
deltas = "".join(event.delta for event in events if getattr(event, "type", None) == "response.output_text.delta")
|
||||
assert deltas == "Hello!"
|
||||
|
||||
|
||||
|
|
@ -802,7 +763,9 @@ async def test_leading_empty_choices_chunk_does_not_kill_the_stream():
|
|||
|
||||
event_types = [getattr(event, "type", None) for event in events]
|
||||
assert event_types.count(ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED) == 1
|
||||
assert "".join(event.delta for event in events if event.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA) == "Hello!"
|
||||
assert (
|
||||
"".join(event.delta for event in events if event.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA) == "Hello!"
|
||||
)
|
||||
assert event_types[-1] == ResponsesAPIStreamEvents.RESPONSE_COMPLETED
|
||||
|
||||
|
||||
|
|
@ -986,9 +949,7 @@ def test_streamed_unrecognized_tool_choice_is_echoed_as_auto() -> None:
|
|||
litellm_metadata={},
|
||||
)
|
||||
|
||||
response_events: Final = [
|
||||
event for event in iterator if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES
|
||||
]
|
||||
response_events: Final = [event for event in iterator if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES]
|
||||
|
||||
assert [event.response.tool_choice for event in response_events] == ["auto", "auto", "auto"]
|
||||
|
||||
|
|
@ -1038,9 +999,7 @@ def _signed_thinking_and_text_in_one_chunk() -> tuple[ModelResponseStream, ...]:
|
|||
chunk: Final = _chunk("Hello!", finish_reason="stop")
|
||||
signed_block: Final = {"type": "thinking", "thinking": "", "signature": "sig_only"}
|
||||
delta: Final = Delta(**{**chunk.choices[0].delta.model_dump(), "thinking_blocks": [signed_block]})
|
||||
return (
|
||||
chunk.model_copy(update={"choices": [StreamingChoices(index=0, delta=delta, finish_reason="stop")]}),
|
||||
)
|
||||
return (chunk.model_copy(update={"choices": [StreamingChoices(index=0, delta=delta, finish_reason="stop")]}),)
|
||||
|
||||
|
||||
StreamChunks = Callable[[], tuple[ModelResponseStream, ...]]
|
||||
|
|
@ -1560,9 +1519,7 @@ async def test_anthropic_tool_search_call_streams_as_tool_search_call_item(sync_
|
|||
model="claude-sonnet-5-5",
|
||||
object="chat.completion.chunk",
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
index=0, delta=delta, finish_reason="tool_calls" if index == len(deltas) - 1 else None
|
||||
)
|
||||
StreamingChoices(index=0, delta=delta, finish_reason="tool_calls" if index == len(deltas) - 1 else None)
|
||||
],
|
||||
)
|
||||
for index, delta in enumerate(deltas)
|
||||
|
|
|
|||
|
|
@ -60,11 +60,7 @@ def test_function_call_output_stays_adjacent_to_tool_call():
|
|||
assistant_ok_idx = i
|
||||
elif isinstance(content, list):
|
||||
for block in content:
|
||||
if (
|
||||
isinstance(block, dict)
|
||||
and block.get("type") == "text"
|
||||
and block.get("text") == "Done."
|
||||
):
|
||||
if isinstance(block, dict) and block.get("type") == "text" and block.get("text") == "Done.":
|
||||
assistant_ok_idx = i
|
||||
break
|
||||
|
||||
|
|
|
|||
87
tests/unit/responses/test_bridge_executor.py
Normal file
87
tests/unit/responses/test_bridge_executor.py
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
import json
|
||||
from datetime import datetime
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.responses.bridge_executor import native_logging_obj
|
||||
|
||||
API_BASE: Final = "https://bridge-executor.invalid/v1"
|
||||
_CHAT_REPLY: Final = {
|
||||
"id": "chatcmpl-1",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "m",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _offline(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("surface", "litellm_params", "model_info", "upstream_model"),
|
||||
[
|
||||
("messages", {"model": "openai/gpt-4.1", "use_chat_completions_api": True}, None, "gpt-4.1"),
|
||||
("messages", {"model": "openai/gpt-4.1", "use_chat_completions_api": True}, {"mode": "responses"}, "gpt-4.1"),
|
||||
("messages", {"model": "openai/chat_completions/o3-pro"}, None, "o3-pro"),
|
||||
("responses", {"model": "openai/chat_completions/o3-pro"}, None, "o3-pro"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_deployment_asking_for_chat_completions_is_sent_to_chat_without_the_prefix(
|
||||
surface: str,
|
||||
litellm_params: dict[str, object],
|
||||
model_info: dict[str, object] | None,
|
||||
upstream_model: str,
|
||||
respx_mock: respx.MockRouter,
|
||||
):
|
||||
chat: Final = respx_mock.post(f"{API_BASE}/chat/completions").mock(
|
||||
return_value=httpx.Response(200, json=_CHAT_REPLY)
|
||||
)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "m",
|
||||
"litellm_params": {"api_base": API_BASE, "api_key": "k", **litellm_params},
|
||||
**({"model_info": model_info} if model_info is not None else {}),
|
||||
}
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
|
||||
if surface == "messages":
|
||||
await router.aanthropic_messages(model="m", max_tokens=32, messages=[{"role": "user", "content": "hi"}])
|
||||
else:
|
||||
await router.aresponses(model="m", input="hi")
|
||||
|
||||
assert chat.call_count == 1
|
||||
assert json.loads(chat.calls.last.request.content)["model"] == upstream_model
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("response_cache", "native"), [(None, True), (litellm.Cache(type="local"), False)])
|
||||
def test_a_bridged_responses_call_goes_through_aresponses_when_the_response_cache_is_on(
|
||||
response_cache: litellm.Cache | None, native: bool, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
monkeypatch.setattr(litellm, "cache", response_cache)
|
||||
logging_obj: Final = Logging(
|
||||
model="gpt-4.1",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=False,
|
||||
call_type="anthropic_messages",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="bridge-executor-cache",
|
||||
function_id="bridge-executor-cache",
|
||||
)
|
||||
|
||||
chosen: Final = native_logging_obj(logging_obj, {"api_key": "k"}, {"model": "openai/gpt-4.1", "input": "hi"})
|
||||
|
||||
assert (chosen is logging_obj) is native
|
||||
|
|
@ -15,7 +15,7 @@ import litellm
|
|||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig
|
||||
from litellm.responses.main import _resolve_responses_api_provider_config
|
||||
from litellm.responses.main import resolve_responses_api_provider_config
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
|
|
@ -115,13 +115,13 @@ def _respx_interceptable_httpx_client(monkeypatch):
|
|||
],
|
||||
)
|
||||
def test_resolver_opt_in_gates_openai_like_config(model_info, expected_type):
|
||||
config = _resolve_responses_api_provider_config("my-model", "custom_openai", model_info, None)
|
||||
config = resolve_responses_api_provider_config("my-model", "custom_openai", model_info, None)
|
||||
assert type(config) is expected_type
|
||||
|
||||
|
||||
def test_resolver_keeps_native_provider_config():
|
||||
"""`openai/` already routes /v1/responses natively; the opt-in must not swap its config."""
|
||||
config = _resolve_responses_api_provider_config("gpt-4.1", "openai", OPT_IN, None)
|
||||
config = resolve_responses_api_provider_config("gpt-4.1", "openai", OPT_IN, None)
|
||||
assert type(config) is OpenAIResponsesAPIConfig
|
||||
|
||||
|
||||
|
|
|
|||
68
tests/unit/test_check_native_imports.py
Normal file
68
tests/unit/test_check_native_imports.py
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from scripts.check_native_imports import main, violations
|
||||
|
||||
CHECKER: Final = Path(__file__).resolve().parents[2] / "scripts/check_native_imports.py"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path, source",
|
||||
(
|
||||
("litellm/native/chat.py", "from litellm.main import execute_chat"),
|
||||
("litellm/native/chat.py", "from litellm import main"),
|
||||
("litellm/native/chat.py", "import litellm.main"),
|
||||
("litellm/native/chat.py", "import litellm.main as legacy"),
|
||||
("litellm/native/chat.py", "from ..main import completion"),
|
||||
("litellm/native/chat.py", "from .. import main"),
|
||||
("litellm/native/chat_providers/azure.py", "from ...main import completion"),
|
||||
("litellm/native/chat.py", "import litellm\n\ndef run():\n return litellm.main.completion()"),
|
||||
),
|
||||
)
|
||||
def test_native_module_importing_main_is_rejected(path, source):
|
||||
assert violations(Path(path), source)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path, source",
|
||||
(
|
||||
("litellm/native/chat.py", "from litellm.native.chat_providers.azure import complete_azure"),
|
||||
("litellm/native/chat.py", "import litellm\n\ndef run():\n return litellm.completion"),
|
||||
("litellm/native/chat.py", "from litellm.maintenance import x"),
|
||||
("litellm/native/chat.py", "from .chat_mock import mock_completion"),
|
||||
("litellm/native/chat_providers/azure.py", "from ..chat_mock import mock_completion"),
|
||||
("litellm/native/chat.py", "from litellm import utils"),
|
||||
),
|
||||
)
|
||||
def test_native_module_without_main_dependency_passes(path, source):
|
||||
assert violations(Path(path), source) == ()
|
||||
|
||||
|
||||
def test_command_reports_the_offending_line_and_passes_once_it_is_gone(tmp_path, monkeypatch, capsys):
|
||||
package: Final = tmp_path / "litellm/native/chat_providers"
|
||||
package.mkdir(parents=True)
|
||||
module: Final = package / "azure.py"
|
||||
module.write_text("import os\nfrom litellm.main import completion\n")
|
||||
monkeypatch.chdir(tmp_path)
|
||||
assert main() == 1
|
||||
assert "litellm/native/chat_providers/azure.py:2:" in capsys.readouterr().err
|
||||
rejected: Final = subprocess.run(
|
||||
[sys.executable, "-I", str(CHECKER)], cwd=tmp_path, capture_output=True, text=True, check=False
|
||||
)
|
||||
assert rejected.returncode == 1
|
||||
|
||||
module.write_text("import os\n")
|
||||
assert main() == 0
|
||||
accepted: Final = subprocess.run(
|
||||
[sys.executable, "-I", str(CHECKER)], cwd=tmp_path, capture_output=True, text=True, check=False
|
||||
)
|
||||
assert accepted.returncode == 0
|
||||
|
||||
|
||||
def test_the_native_package_has_no_main_dependency(monkeypatch):
|
||||
monkeypatch.chdir(CHECKER.parents[1])
|
||||
assert main() == 0
|
||||
|
|
@ -7,6 +7,7 @@ import json
|
|||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
from collections.abc import Callable, Mapping
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
|
@ -20,7 +21,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
|||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from openai import APITimeoutError, AsyncOpenAI, OpenAI
|
||||
from openai import AsyncOpenAI, OpenAI
|
||||
from openai._legacy_response import LegacyAPIResponse
|
||||
from openai.types.chat.chat_completion import ChatCompletion
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
|
|
@ -45,10 +46,12 @@ from litellm.litellm_core_utils.get_litellm_params import stored_control_options
|
|||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.native import chat as native_chat
|
||||
from litellm.types.litellm_params import ControlOptions
|
||||
from litellm.types.llms.openai import AllMessageValues, HttpxBinaryResponseContent
|
||||
from litellm.types.prompts.init_prompts import PromptSpec
|
||||
from litellm.types.utils import (
|
||||
CallTypes,
|
||||
Delta,
|
||||
ModelResponseStream,
|
||||
StandardCallbackDynamicParams,
|
||||
|
|
@ -517,7 +520,7 @@ def test_drop_anthropic_only_tool_keys_strips_tool_and_function_levels(key: str)
|
|||
"opaque_tool",
|
||||
]
|
||||
|
||||
cleaned = litellm_main._drop_anthropic_only_tool_keys(tools=tools)
|
||||
cleaned = native_chat._drop_anthropic_only_tool_keys(tools=tools)
|
||||
|
||||
assert cleaned == [
|
||||
{"type": "function", "name": "example_tool", "function": {"name": "example_tool"}},
|
||||
|
|
@ -553,6 +556,98 @@ def test_completion_strips_eager_input_streaming_before_openai(respx_mock: respx
|
|||
assert sent_tool["function"]["name"] == "write_file"
|
||||
|
||||
|
||||
class _SyncSuccessCounter(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.calls: list[str] = [] # mutable-ok: test recorder of success-callback events
|
||||
|
||||
def log_success_event(self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object):
|
||||
self.calls.append(str(kwargs.get("call_type")))
|
||||
|
||||
|
||||
def _settled_success_calls(counter: _SyncSuccessCounter, expected: int) -> int:
|
||||
for _ in range(50):
|
||||
if len(counter.calls) >= expected:
|
||||
break
|
||||
time.sleep(0.02)
|
||||
return len(counter.calls)
|
||||
|
||||
|
||||
class _DeploymentHookCounter(CustomLogger):
|
||||
call_types: tuple[str, ...] = ()
|
||||
|
||||
async def async_pre_call_deployment_hook(self, kwargs: dict[str, object], call_type: CallTypes | None) -> None:
|
||||
self.call_types = (*self.call_types, str(call_type))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_mode_model_without_a_responses_api_stays_on_chat(
|
||||
respx_mock: respx.MockRouter, openai_api_response: dict[str, object], monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
counter: Final = _DeploymentHookCounter()
|
||||
monkeypatch.setattr(litellm, "callbacks", [counter])
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
litellm.register_model(
|
||||
{"groq/responses-only-model": {"litellm_provider": "groq", "mode": "responses", "max_tokens": 100}}
|
||||
)
|
||||
api_base: Final = "https://example.invalid/openai/v1"
|
||||
chat_route: Final = respx_mock.post(f"{api_base}/chat/completions").mock(
|
||||
return_value=httpx.Response(status_code=200, json=openai_api_response)
|
||||
)
|
||||
|
||||
response: Final = await litellm.acompletion(
|
||||
model="groq/responses-only-model",
|
||||
api_base=api_base,
|
||||
api_key="sk-x",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
|
||||
assert isinstance(response, litellm.ModelResponse)
|
||||
assert response.id == openai_api_response["id"]
|
||||
assert chat_route.call_count == 1
|
||||
assert counter.call_types == ("CallTypes.acompletion",)
|
||||
|
||||
|
||||
def test_execute_chat_calls_the_provider_without_logging(
|
||||
respx_mock: respx.MockRouter, openai_api_response, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
counter: Final = _SyncSuccessCounter()
|
||||
monkeypatch.setattr(litellm, "callbacks", [counter])
|
||||
api_base: Final = "http://localhost:12347/v1"
|
||||
mock_route: Final = respx_mock.post(url__regex=rf"{api_base}/chat/completions.*").mock(
|
||||
return_value=httpx.Response(status_code=200, json=openai_api_response)
|
||||
)
|
||||
messages: Final = [{"role": "user", "content": "hi"}]
|
||||
logging_obj: Final = LiteLLMLogging(
|
||||
model="hosted_vllm/served-model",
|
||||
messages=messages,
|
||||
stream=False,
|
||||
call_type="anthropic_messages",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="execute-chat-call",
|
||||
function_id="execute-chat-fn",
|
||||
)
|
||||
|
||||
response: Final = litellm_main.execute_chat(
|
||||
model="hosted_vllm/served-model",
|
||||
messages=list(messages),
|
||||
kwargs={"api_base": api_base, "api_key": "sk-x", "litellm_logging_obj": logging_obj},
|
||||
non_default_params={},
|
||||
control_options=ControlOptions(),
|
||||
prompt_id=None,
|
||||
prompt_variables=None,
|
||||
)
|
||||
|
||||
assert isinstance(response, litellm.ModelResponse)
|
||||
assert response.id == openai_api_response["id"]
|
||||
assert mock_route.call_count == 1
|
||||
assert _settled_success_calls(counter, expected=1) == 0
|
||||
|
||||
litellm.completion(model="hosted_vllm/served-model", messages=messages, api_base=api_base, api_key="sk-x")
|
||||
assert _settled_success_calls(counter, expected=1) == 1
|
||||
|
||||
|
||||
_CLAUDE_TOOL_SEARCH_CALL: Final = {
|
||||
"id": "srvtoolu_01search",
|
||||
"type": "function",
|
||||
|
|
@ -2591,7 +2686,7 @@ def test_completion_optional_params_base_model(
|
|||
|
||||
Regression for #29618: overwriting ``model`` with a friendly ``base_model``
|
||||
label made Bedrock drop ``tools``/``tool_choice`` under ``drop_params``."""
|
||||
with patch("litellm.main.get_optional_params") as mock_get_optional_params:
|
||||
with patch("litellm.native.chat.get_optional_params") as mock_get_optional_params:
|
||||
mock_get_optional_params.return_value = MagicMock()
|
||||
|
||||
import litellm
|
||||
|
|
@ -5683,7 +5778,7 @@ def test_bridged_responses_with_openai_http_handler_keeps_forwarded_headers_out_
|
|||
def test_aiohttp_openai_warns_only_when_http2_enabled(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, http2_on: bool
|
||||
):
|
||||
from litellm.main import base_llm_aiohttp_handler
|
||||
from litellm.native.chat_providers.handlers import base_llm_aiohttp_handler
|
||||
|
||||
monkeypatch.setattr(litellm, "http2", http2_on)
|
||||
monkeypatch.delenv("LITELLM_HTTP2", raising=False)
|
||||
|
|
@ -5984,7 +6079,6 @@ def test_drop_params_false_still_rejects_an_invalid_stream_chunk_size() -> None:
|
|||
def test_acompletion_params():
|
||||
import inspect
|
||||
|
||||
from litellm.types.completion import CompletionRequest
|
||||
|
||||
acompletion_params_odict = inspect.signature(acompletion).parameters
|
||||
completion_params_dict = inspect.signature(completion).parameters
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ import pytest
|
|||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.get_model_cost_map import _expand_model_aliases, adopt_model_cost_map
|
||||
from litellm.main import _build_custom_pricing_entry
|
||||
from litellm.native.request_support import _build_custom_pricing_entry
|
||||
from litellm.utils import _invalidate_model_cost_lowercase_map
|
||||
|
||||
|
||||
|
|
@ -1677,7 +1677,7 @@ def test_register_model_scopes_builtin_match_to_the_given_provider():
|
|||
|
||||
|
||||
def test_per_request_custom_pricing_scopes_a_colliding_deployment_id_to_its_provider():
|
||||
from litellm.main import _register_custom_pricing_for_request
|
||||
from litellm.native.request_support import register_custom_pricing_for_request
|
||||
|
||||
deployment_id: Final = "baseten/zai-org/glm-5.2"
|
||||
builtin_key: Final = "baseten/zai-org/GLM-5.2"
|
||||
|
|
@ -1686,7 +1686,7 @@ def test_per_request_custom_pricing_scopes_a_colliding_deployment_id_to_its_prov
|
|||
builtin_row_before: Final = copy.deepcopy(litellm.model_cost[builtin_key])
|
||||
openai_models_before: Final = frozenset(litellm.open_ai_chat_completion_models)
|
||||
try:
|
||||
_register_custom_pricing_for_request(
|
||||
register_custom_pricing_for_request(
|
||||
model="zai-org/GLM-5.2",
|
||||
custom_llm_provider="openai",
|
||||
kwargs={
|
||||
|
|
|
|||
|
|
@ -1843,7 +1843,9 @@ def test_arouter_responses_api_bridge():
|
|||
|
||||
## CONFIRM BRIDGE IS CALLED
|
||||
# an empty stream: the Router reads the first chunk before returning, and a stream of mocks never produces one
|
||||
with patch.object(litellm, "responses", return_value=iter([])) as mock_responses:
|
||||
from litellm.responses import bridge_executor
|
||||
|
||||
with patch.object(bridge_executor, "native_responses", return_value=iter([])) as mock_responses:
|
||||
result = router.completion(
|
||||
model="[IP-approved] o3-pro",
|
||||
messages=[{"role": "user", "content": "Hello, world!"}],
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue