scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 35

Merge pull request #36543 from BerriAI/litellm_decrease_anys_fable5

Browse source 
chore(typing): clear 1.6k basedpyright Any errors across 56 files
This commit is contained in:
Mateo Wang 2026-08-11 12:41:14 -07:00 • committed by GitHub
commit e37ae033cc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
60 changed files with 1499 additions and 845 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
basedpyright-code-budget.json
View file

@ -1,15 +1,15 @@
{
"reportAny": {
"limit": 26391
"limit": 23919
},
"reportArgumentType": {
"limit": 2614
"limit": 2580
},
"reportAssignmentType": {
"limit": 327
"limit": 323
},
"reportAttributeAccessIssue": {
"limit": 514
"limit": 488
},
"reportCallIssue": {
"limit": 114
@ -18,13 +18,13 @@
"limit": 40
},
"reportDeprecated": {
"limit": 215
"limit": 213
},
"reportDuplicateImport": {
"limit": 19
},
"reportExplicitAny": {
"limit": 8319
"limit": 7573
},
"reportFunctionMemberAccess": {
"limit": 7
@ -54,10 +54,10 @@
"limit": 0
},
"reportMissingParameterType": {
"limit": 5825
"limit": 5719
},
"reportMissingTypeArgument": {
"limit": 15695
"limit": 15657
},
"reportMissingTypeStubs": {
"limit": 40
@ -72,7 +72,7 @@
"limit": 0
},
"reportOptionalMemberAccess": {
"limit": 1077
"limit": 1069
},
"reportOptionalOperand": {
"limit": 0
@ -99,37 +99,37 @@
"limit": 0
},
"reportUnknownArgumentType": {
"limit": 44996
"limit": 44832
},
"reportUnknownLambdaType": {
"limit": 113
},
"reportUnknownMemberType": {
"limit": 39643
"limit": 39269
},
"reportUnknownParameterType": {
"limit": 20132
"limit": 19988
},
"reportUnknownVariableType": {
"limit": 31153
"limit": 30923
},
"reportUnnecessaryCast": {
"limit": 118
},
"reportUnnecessaryComparison": {
"limit": 701
"limit": 699
},
"reportUnnecessaryContains": {
"limit": 5
},
"reportUnnecessaryIsInstance": {
"limit": 857
"limit": 853
},
"reportUntypedBaseClass": {
"limit": 0
},
"reportUntypedFunctionDecorator": {
"limit": 33
"limit": 27
},
"reportUnusedClass": {
"limit": 23
@ -138,7 +138,7 @@
"limit": 139
},
"reportUnusedImport": {
"limit": 555
"limit": 545
},
"reportUnusedVariable": {
"limit": 146

28
litellm/completion_extras/litellm_responses_transformation/handler.py
View file

@ -22,6 +22,7 @@ class ResponsesToCompletionBridgeHandlerInputKwargs(TypedDict):
model_response: "ModelResponse"
logging_obj: "LiteLLMLoggingObj"
custom_llm_provider: str
encoding: object
class ResponsesToCompletionBridgeHandler:
@ -102,35 +103,37 @@ class ResponsesToCompletionBridgeHandler:
from litellm import LiteLLMLoggingObj
from litellm.types.utils import ModelResponse
model: Final = kwargs.get("model")
typed_kwargs: Final[dict[str, object]] = kwargs
model: Final = typed_kwargs.get("model")
if model is None or not isinstance(model, str):
raise ValueError("model is required")
custom_llm_provider: Final = kwargs.get("custom_llm_provider")
custom_llm_provider: Final = typed_kwargs.get("custom_llm_provider")
if custom_llm_provider is None or not isinstance(custom_llm_provider, str):
raise ValueError("custom_llm_provider is required")
messages: Final = kwargs.get("messages")
messages: Final = typed_kwargs.get("messages")
if messages is None or not isinstance(messages, list):
raise ValueError("messages is required")
optional_params: Final = kwargs.get("optional_params")
optional_params: Final = typed_kwargs.get("optional_params")
if optional_params is None or not isinstance(optional_params, dict):
raise ValueError("optional_params is required")
litellm_params: Final = kwargs.get("litellm_params")
litellm_params: Final = typed_kwargs.get("litellm_params")
if litellm_params is None or not isinstance(litellm_params, dict):
raise ValueError("litellm_params is required")
headers: Final = kwargs.get("headers")
headers: Final = typed_kwargs.get("headers")
if headers is None or not isinstance(headers, dict):
raise ValueError("headers is required")
model_response: Final = kwargs.get("model_response")
model_response: Final = typed_kwargs.get("model_response")
if model_response is None or not isinstance(model_response, ModelResponse):
raise ValueError("model_response is required")
logging_obj: Final = kwargs.get("logging_obj")
logging_obj: Final = typed_kwargs.get("logging_obj")
if logging_obj is None or not isinstance(logging_obj, LiteLLMLoggingObj):
raise ValueError("logging_obj is required")
@ -143,6 +146,7 @@ class ResponsesToCompletionBridgeHandler:
model_response=model_response,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
encoding=typed_kwargs.get("encoding"),
)
def completion(
@ -205,7 +209,7 @@ class ResponsesToCompletionBridgeHandler:
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
encoding=kwargs.get("encoding"),
encoding=validated_kwargs["encoding"],
api_key=kwargs.get("api_key"),
json_mode=kwargs.get("json_mode"),
)
@ -230,7 +234,7 @@ class ResponsesToCompletionBridgeHandler:
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
encoding=kwargs.get("encoding"),
encoding=validated_kwargs["encoding"],
api_key=kwargs.get("api_key"),
json_mode=kwargs.get("json_mode"),
)
@ -303,7 +307,7 @@ class ResponsesToCompletionBridgeHandler:
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
encoding=kwargs.get("encoding"),
encoding=validated_kwargs["encoding"],
api_key=kwargs.get("api_key"),
json_mode=kwargs.get("json_mode"),
)
@ -328,7 +332,7 @@ class ResponsesToCompletionBridgeHandler:
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
encoding=kwargs.get("encoding"),
encoding=validated_kwargs["encoding"],
api_key=kwargs.get("api_key"),
json_mode=kwargs.get("json_mode"),
)

101
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -4,8 +4,8 @@ Handler for transforming /chat/completions api requests to litellm.responses req
import json
import os
from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping
from typing import TYPE_CHECKING, Any, Final, Literal, Union, cast
from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, Union, cast
from openai.types.responses.custom_tool_param import CustomToolParam
from openai.types.responses.response_input_param import (
@ -45,6 +45,9 @@ from litellm.types.utils import GenericStreamingChunk, ModelResponseStream
if TYPE_CHECKING:
from openai.types.responses import ResponseInputImageParam
from openai.types.responses.response_text_config_param import (
ResponseTextConfigParam as ResponseText,
)
from pydantic import BaseModel
from litellm import LiteLLMLoggingObj, ModelResponse
@ -57,6 +60,19 @@ if TYPE_CHECKING:
ChatCompletionThinkingBlock,
OpenAIMessageContentListBlock,
)
from litellm.types.utils import Choices
class _ReasoningSummaryText(TypedDict):
type: str
text: str
class _BuiltReasoningItem(TypedDict):
type: Literal["reasoning"]
id: str
encrypted_content: str | None
summary: Sequence[_ReasoningSummaryText]
def _get_reasoning_items(
@ -72,13 +88,13 @@ def _get_reasoning_items(
def _build_reasoning_item(
item_id: str,
encrypted_content: str | None,
summary_raw: Any,
) -> dict[str, Any]:
summary_raw: Iterable[object] | None,
) -> _BuiltReasoningItem:
"""Build a ChatCompletionReasoningItem-shaped dict from raw response data.
Handles both pydantic objects (attribute access) and plain dicts.
"""
summary: Final[list[dict[str, Any]]] = []
summary: Final[list[_ReasoningSummaryText]] = []
for s in summary_raw or []:
if isinstance(s, dict):
summary.append({"type": s.get("type", "summary_text"), "text": s.get("text", "")})
@ -98,7 +114,7 @@ def _build_reasoning_item(
class _ChatToolCallDict(ChatCompletionToolCallChunk, total=False):
provider_specific_fields: Mapping[str, Any]
provider_specific_fields: Mapping[str, object]
def _tool_call_dict_from_output_item(item: Mapping[str, Any], index: int) -> _ChatToolCallDict:
@ -142,10 +158,10 @@ def _flat_responses_tool_choice(choice_type: str, name: str) -> ToolChoiceFuncti
def _reasoning_item_to_response_input(
r_item: ChatCompletionReasoningItem | dict[str, Any],
) -> dict[str, Any]:
r_item: ChatCompletionReasoningItem,
) -> dict[str, object]:
"""Convert a stored ChatCompletionReasoningItem back to a Responses API input item."""
r_input: Final[dict[str, Any]] = {
r_input: Final[dict[str, object]] = {
"type": "reasoning",
"id": r_item.get("id") or f"rs_{id(r_item)}",
# summary is always required by the Responses API, even when empty
@ -181,7 +197,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
return _flat_responses_tool_choice(choice_type, nested_name)
return tool_choice
def _handle_raw_dict_response_item(self, item: dict[str, Any], index: int) -> tuple[Any | None, int]:
def _handle_raw_dict_response_item(self, item: dict[str, Any], index: int) -> tuple["Choices | None", int]:
"""
Handle raw dict response items from Responses API (e.g., GPT-5 Codex format).
@ -228,8 +244,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
def convert_chat_completion_messages_to_responses_api(
self, messages: list["AllMessageValues"]
) -> tuple[list[Any], str | None]:
input_items: Final[list[Any]] = []
) -> tuple[list[object], str | None]:
input_items: Final[list[object]] = []
instructions: str | None = None
custom_tool_call_ids: Final = frozenset(
tool_call["id"]
@ -270,7 +286,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
# Convert tool message to function call output format
# The Responses API expects 'output' to be a list with input_text/input_image types
# Using list format for consistency across text and multimodal content
tool_output: list[dict[str, Any]]
tool_output: list[dict[str, object]]
if content is None:
tool_output = []
elif isinstance(content, str):
@ -308,7 +324,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
function = tool_call.get("function")
custom = tool_call.get("custom")
if function:
input_tool_call: dict[str, Any] = {
input_tool_call: dict[str, object] = {
"type": "function_call",
"call_id": tool_call["id"],
}
@ -376,15 +392,15 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
elif key == "web_search_options":
self._add_web_search_tool(responses_api_request, value)
def _build_sanitized_litellm_params(self, litellm_params: dict) -> dict[str, Any]:
def _build_sanitized_litellm_params(self, litellm_params: dict) -> dict[str, object]:
"""Build sanitized litellm_params with merged metadata."""
responses_optional_param_keys: Final = set(ResponsesAPIOptionalRequestParams.__annotations__.keys())
sanitized: Final[dict[str, Any]] = {
sanitized: Final[dict[str, object]] = {
key: value for key, value in litellm_params.items() if key not in responses_optional_param_keys
}
legacy_metadata: Final = litellm_params.get("metadata")
existing_litellm_metadata: Final = litellm_params.get("litellm_metadata")
merged_litellm_metadata: Final[dict[str, Any]] = {}
merged_litellm_metadata: Final[dict[str, object]] = {}
if isinstance(legacy_metadata, dict):
merged_litellm_metadata.update(legacy_metadata)
if isinstance(existing_litellm_metadata, dict):
@ -424,7 +440,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
litellm_params: dict,
headers: dict,
litellm_logging_obj: "LiteLLMLoggingObj",
client: Any | None = None,
client: object | None = None,
) -> dict:
(
input_items,
@ -498,9 +514,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
@staticmethod
def _convert_response_output_to_choices(
output_items: list[Any],
handle_raw_dict_callback: Callable | None = None,
) -> list[Any]:
output_items: Sequence[object],
handle_raw_dict_callback: Callable[..., tuple["Choices | None", int]] | None = None,
) -> list["Choices"]:
"""
Convert Responses API output items to chat completion choices.
@ -529,11 +545,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
choices: Final[list[Choices]] = []
index = 0
reasoning_content: str | None = None
pending_reasoning_item: dict[str, Any] | None = None
pending_reasoning_item: _BuiltReasoningItem | None = None
# Collect all tool calls to put them in a single choice
# (Chat Completions API expects all tool calls in one message)
accumulated_tool_calls: Final[list[dict[str, Any]]] = []
accumulated_tool_calls: Final[list[Mapping[str, object]]] = []
tool_call_index = 0
for item in output_items:
@ -640,7 +656,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
return choices
@classmethod
def _extract_output_from_completed_event(cls, parsed_chunk: dict[str, Any]) -> list[dict[str, Any]] | None:
def _extract_output_from_completed_event(cls, parsed_chunk: Mapping[str, object]) -> list[dict[str, object]] | None:
response_payload: Final = parsed_chunk.get("response")
if not isinstance(response_payload, dict):
return None
@ -650,12 +666,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
return cast(list[dict[str, Any]], response_output)
@classmethod
def _recover_output_items_from_raw_sse(cls, raw_sse: str | None) -> list[dict[str, Any]]:
def _recover_output_items_from_raw_sse(cls, raw_sse: str | None) -> list[dict[str, object]]:
if not raw_sse or not isinstance(raw_sse, str):
return []
recovered_output_items: Final[dict[int, dict[str, Any]]] = {}
recovered_text_only_items: Final[dict[int, dict[str, Any]]] = {}
recovered_output_items: Final[dict[int, dict[str, object]]] = {}
recovered_text_only_items: Final[dict[int, dict[str, object]]] = {}
for chunk in raw_sse.splitlines():
parsed_chunk = parse_sse_json_chunk(chunk)
@ -690,7 +706,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
# but text-only items at indices without a matching OUTPUT_ITEM_DONE
# must still be preserved (e.g. multi-output responses where some
# indices only emitted OUTPUT_TEXT_DONE).
merged_items: Final[dict[int, dict[str, Any]]] = {**recovered_text_only_items}
merged_items: Final[dict[int, dict[str, object]]] = {**recovered_text_only_items}
merged_items.update(recovered_output_items)
if merged_items:
@ -699,7 +715,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
return []
@classmethod
def _recover_output_items_from_logging(cls, logging_obj: "LiteLLMLoggingObj") -> list[dict[str, Any]]:
def _recover_output_items_from_logging(cls, logging_obj: "LiteLLMLoggingObj") -> list[dict[str, object]]:
model_call_details: Final = getattr(logging_obj, "model_call_details", {}) or {}
original_response: Final = model_call_details.get("original_response")
return cls._recover_output_items_from_raw_sse(original_response)
@ -714,7 +730,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
messages: list["AllMessageValues"],
optional_params: dict,
litellm_params: dict,
encoding: Any,
encoding: object,
api_key: str | None = None,
json_mode: bool | None = None,
) -> "ModelResponse":
@ -788,7 +804,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
) -> BaseModelResponseIterator:
return OpenAiResponsesToChatCompletionStreamIterator(streaming_response, sync_stream, json_mode)
def _convert_content_str_to_input_text(self, content: str, role: str) -> dict[str, Any]:
def _convert_content_str_to_input_text(self, content: str, role: str) -> dict[str, object]:
if role == "user" or role == "system" or role == "tool":
return {"type": "input_text", "text": content}
else:
@ -825,13 +841,13 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
def _convert_content_to_responses_format(
self,
content: str
| list[Any]
| list[object]
| Iterable[
Union["OpenAIMessageContentListBlock", "ChatCompletionThinkingBlock", "ChatCompletionRedactedThinkingBlock"]
]
| None,
role: str,
) -> list[dict[str, Any]]:
) -> list[dict[str, object]]:
"""Convert chat completion content to responses API format"""
from litellm.types.llms.openai import ChatCompletionImageObject
@ -973,7 +989,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
return optional_params
def _map_reasoning_effort(self, reasoning_effort: str | dict[str, Any]) -> Reasoning | None:
def _map_reasoning_effort(self, reasoning_effort: str | Reasoning) -> Reasoning | None:
# If dict is passed, convert it directly to Reasoning object
if isinstance(reasoning_effort, dict):
return Reasoning(**reasoning_effort)
@ -1006,7 +1022,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
def _add_web_search_tool(
self,
responses_api_request: ResponsesAPIOptionalRequestParams,
web_search_options: Any,
web_search_options: object,
) -> None:
"""
Add web search tool to responses API request.
@ -1024,14 +1040,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
tools = []
responses_api_request["tools"] = tools
web_search_tool: Final[dict[str, Any]] = {"type": "web_search"}
web_search_tool: Final[dict[str, object]] = {"type": "web_search"}
if isinstance(web_search_options, dict):
web_search_tool.update(web_search_options)
# Cast to Any to match the expected union type for tools list items
tools.append(cast(Any, web_search_tool))
def _transform_response_format_to_text_format(self, response_format: dict[str, Any] | Any) -> dict[str, Any] | None:
def _transform_response_format_to_text_format(self, response_format: object) -> "ResponseText | None":
"""
Transform Chat Completion response_format parameter to Responses API text.format parameter.
@ -1130,7 +1146,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
def __init__(self, streaming_response, sync_stream: bool, json_mode: bool | None = False):
def __init__(
self,
streaming_response: Union[Iterator[str], AsyncIterator[str], "ModelResponse", "BaseModel"],
sync_stream: bool,
json_mode: bool | None = False,
):
super().__init__(streaming_response, sync_stream, json_mode)
self._chat_completion_id: str | None = None
self._tool_call_index_map: dict[int, int] = {} # mutable-ok: per-stream accumulator state
@ -1387,7 +1408,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
finish_reason: Final = "tool_calls" if has_function_calls else "stop"
# Extract reasoning items with encrypted_content for round-tripping
completed_reasoning_items: list[dict[str, Any]] | None = None
completed_reasoning_items: list[_BuiltReasoningItem] | None = None
for item in output_items:
if not isinstance(item, dict) or item.get("type") != "reasoning":
continue
@ -1439,7 +1460,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
]
)
def chunk_parser(self, chunk: dict) -> "ModelResponseStream":
def chunk_parser(self, chunk: dict[str, object]) -> "ModelResponseStream":
"""
Parse a Responses API streaming chunk and convert to OpenAI format.

7
litellm/images/main.py
View file

@ -315,7 +315,12 @@ def image_generation(
or get_secret_str("AZURE_API_KEY")
)
azure_ad_token: Final = optional_params.pop("azure_ad_token", None) or get_secret_str("AZURE_AD_TOKEN")
azure_ad_token_param: Final = optional_params.pop("azure_ad_token", None)
azure_ad_token: Final = (
azure_ad_token_param
if isinstance(azure_ad_token_param, str) and azure_ad_token_param
else get_secret_str("AZURE_AD_TOKEN")
)
# Create azure_ad_token_provider from tenant_id, client_id, client_secret if not already provided
if azure_ad_token_provider is None:

3
litellm/integrations/braintrust_mock_client.py
View file

@ -10,6 +10,7 @@ Usage:
import os
import time
from collections.abc import AsyncIterable, Iterable
from typing import Final
from urllib.parse import urlparse
@ -84,7 +85,7 @@ def _mock_http_handler_post(
timeout=None,
stream=False,
files=None,
content=None,
content: str | bytes | Iterable[bytes] | AsyncIterable[bytes] | None = None,
logging_obj=None,
):
"""Monkey-patched HTTPHandler.post that intercepts Braintrust calls with endpoint-specific responses."""

3
litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py
View file

@ -9,6 +9,7 @@ Usage:
"""
import asyncio
from collections.abc import AsyncIterable, Iterable
from typing import Final
from litellm._logging import verbose_logger
@ -113,7 +114,7 @@ async def _mock_async_handler_delete(
headers=None,
timeout=None,
stream=False,
content=None,
content: str | bytes | Iterable[bytes] | AsyncIterable[bytes] | None = None,
):
"""Monkey-patched AsyncHTTPHandler.delete that intercepts GCS calls."""
# Only mock GCS API calls

19
litellm/integrations/generic_api/generic_api_callback.py
View file

@ -11,7 +11,7 @@ import json
import os
import re
import traceback
from typing import Any, Final, Literal
from typing import Final, Literal
import httpx
@ -158,7 +158,7 @@ class GenericAPILogger(CustomBatchLogger):
"endpoint not set for GenericAPILogger, GENERIC_LOGGER_ENDPOINT not found in environment variables"
)
self.headers: dict = self._get_headers(headers)
self.headers: dict[str, str] = self._get_headers(headers)
self.endpoint: str = endpoint
self.event_types: list[API_EVENT_TYPES] | None = event_types
self.callback_name: str | None = callback_name
@ -248,18 +248,15 @@ class GenericAPILogger(CustomBatchLogger):
await asyncio.sleep(delay)
async def _post_with_retries(self, data: str) -> httpx.Response:
post_kwargs: Final[dict[str, Any]] = {
"url": self.endpoint,
"headers": self.headers,
"data": data,
}
if self.timeout is not None:
post_kwargs["timeout"] = self.timeout
total_attempts: Final = self.max_retries + 1
for attempt in range(total_attempts):
try:
return await self.async_httpx_client.post(**post_kwargs)
return await self.async_httpx_client.post(
url=self.endpoint,
headers=self.headers,
data=data,
timeout=self.timeout,
)
except Exception as e:
is_last_attempt = attempt == self.max_retries
should_retry = self._should_retry_exception(e)

5
litellm/integrations/mock_client_factory.py
View file

@ -8,6 +8,7 @@ making actual network calls.
import asyncio
import json
from collections.abc import AsyncIterable, Iterable
from dataclasses import dataclass
from datetime import timedelta
from typing import Final, cast
@ -140,7 +141,7 @@ def create_mock_client_factory(config: MockClientConfig):
stream=False,
logging_obj=None,
files=None,
content=None,
content: str | bytes | Iterable[bytes] | AsyncIterable[bytes] | None = None,
):
"""Monkey-patched AsyncHTTPHandler.post that intercepts API calls."""
if isinstance(url, str) and _is_mock_url(url):
@ -193,7 +194,7 @@ def create_mock_client_factory(config: MockClientConfig):
timeout=None,
stream=False,
files=None,
content=None,
content: str | bytes | Iterable[bytes] | AsyncIterable[bytes] | None = None,
logging_obj=None,
):
"""Monkey-patched HTTPHandler.post that intercepts API calls."""

64
litellm/integrations/opentelemetry.py
View file

@ -1,7 +1,8 @@
import os
from collections.abc import Mapping
from dataclasses import dataclass, field
from datetime import datetime
from typing import TYPE_CHECKING, Any, Final, cast
from typing import TYPE_CHECKING, Any, Final, TypedDict, cast
import litellm
from litellm._logging import verbose_logger
@ -37,9 +38,11 @@ from litellm.types.utils import (
# OpenTelemetry imports moved to individual functions to avoid import errors when not installed
if TYPE_CHECKING:
from opentelemetry.sdk.trace import TracerProvider as _SDKTracerProvider
from opentelemetry.sdk.trace.export import SpanExporter as _SpanExporter
from opentelemetry.trace import Context as _Context
from opentelemetry.trace import Span as _Span
from opentelemetry.trace import SpanKind as _SpanKind
from opentelemetry.trace import Tracer as _Tracer
from litellm.proxy._types import (
@ -61,6 +64,25 @@ else:
ManagementEndpointLoggingPayload = Any
Context = Any
class _StartSpanRequiredKwargs(TypedDict):
name: str
start_time: int
context: "Context | None"
class _StartSpanKwargs(_StartSpanRequiredKwargs, total=False):
kind: "_SpanKind"
class _UsageCompletionTokensView(TypedDict, total=False):
completion_tokens: int
class _ResponseWithUsageView(TypedDict, total=False):
usage: "_UsageCompletionTokensView | None"
LITELLM_TRACER_NAME: Final = os.getenv("OTEL_TRACER_NAME", "litellm")
LITELLM_METER_NAME: Final = os.getenv("LITELLM_METER_NAME", "litellm")
LITELLM_LOGGER_NAME: Final = os.getenv("LITELLM_LOGGER_NAME", "litellm")
@ -297,9 +319,9 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
config: OpenTelemetryConfig | None = None,
callback_name: str | None = None,
# injection points for testing
tracer_provider: Any | None = None,
logger_provider: Any | None = None,
meter_provider: Any | None = None,
tracer_provider: object | None = None,
logger_provider: object | None = None,
meter_provider: object | None = None,
**kwargs,
):
team_metadata_keys_override: Final = kwargs.pop("baggage_team_metadata_keys", None)
@ -325,7 +347,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
self.OTEL_EXPORTER = self.config.exporter
self.OTEL_ENDPOINT = self.config.endpoint
self.OTEL_HEADERS = self.config.headers
self._tracer_provider_cache: dict[str, Any] = {}
self._tracer_provider_cache: dict[str, _SDKTracerProvider] = {}
self._init_tracing(tracer_provider)
_debug_otel: Final = str(os.getenv("DEBUG_OTEL", "False")).lower()
@ -870,7 +892,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
def _emit_guardrail_spans_from_request_data(
self,
request_data: dict,
parent_span: Any | None,
parent_span: "Span | None",
) -> None:
"""Emit ``guardrail`` spans from the request's proxy-internal metadata bucket
(``standard_logging_guardrail_information``).
@ -896,7 +918,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
# kwargs["litellm_params"]["metadata"]["_otel_internal"]. Pass the
# SAME metadata dict the proxy populated so _handle_failure and
# this hook see the same dedupe markers.
kwargs: Final[dict[str, Any]] = {
kwargs: Final[dict[str, object]] = {
"litellm_params": {"metadata": metadata},
"standard_logging_object": {
"guardrail_information": guardrail_information,
@ -1257,13 +1279,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
response_obj,
start_time,
end_time,
context,
context: "Context | None",
):
from opentelemetry.trace import Status, StatusCode
otel_tracer: Final[Tracer] = self.get_tracer_to_use_for_request(kwargs)
span_kwargs: Final[dict[str, Any]] = {
span_kwargs: Final[_StartSpanKwargs] = {
"name": self._get_span_name(kwargs),
"start_time": self._to_ns(start_time),
"context": context,
@ -1454,7 +1476,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
) = _resolve_metric_attribute_filter(attributes)
self._metric_attr_filter_resolved = True
def _filter_metric_attributes(self, attrs: dict[str, Any]) -> dict[str, Any]:
def _filter_metric_attributes(self, attrs: dict[str, str]) -> dict[str, str]:
if not self._metric_attr_filter_resolved:
self._ensure_metric_attribute_filter()
if self._metric_attr_include is not None:
@ -1559,7 +1581,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
def _record_time_per_output_token_metric(
self,
kwargs: dict,
response_obj: Any | None,
response_obj: "_ResponseWithUsageView | None",
end_time: datetime,
duration_s: float,
common_attrs: dict,
@ -1775,10 +1797,10 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
@staticmethod
def _resolve_guardrail_context(
span: Any | None,
parent_span: Any | None,
fallback_ctx: Any | None,
) -> Any | None:
span: "Span | None",
parent_span: "Span | None",
fallback_ctx: "Context | None",
) -> "Context | None":
"""
Return a valid OTEL context for guardrail child spans so they are
never orphaned (Issue #5). Priority:
@ -1945,7 +1967,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
if should_create_primary_span:
# Span 1: Request sent to litellm SDK
otel_tracer: Final[Tracer] = self.get_tracer_to_use_for_request(kwargs)
span_kwargs: Final[dict[str, Any]] = {
span_kwargs: Final[_StartSpanKwargs] = {
"name": self._get_span_name(kwargs),
"start_time": self._to_ns(start_time),
"context": _parent_context,
@ -2131,10 +2153,10 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
@staticmethod
def _tool_calls_kv_pair(
tool_calls: list[ChatCompletionMessageToolCall],
) -> dict[str, Any]:
) -> dict[str, object]:
from litellm.proxy._types import SpanAttributes
kv_pairs: Final[dict[str, Any]] = {}
kv_pairs: Final[dict[str, object]] = {}
for idx, tool_call in enumerate(tool_calls):
_function = tool_call.get("function")
if not _function:
@ -2691,8 +2713,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
import json
try:
_raw_response = json.loads(_raw_response)
for param, val in _raw_response.items():
_parsed: Final[Mapping[str, object]] = json.loads(_raw_response)
for param, val in _parsed.items():
self.safe_set_attribute(
span=span,
key=f"llm.{custom_llm_provider}.{param}",
@ -2722,7 +2744,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
return int(dt * 1e9)
return int(dt.timestamp() * 1e9)
def _get_span_name(self, kwargs):
def _get_span_name(self, kwargs) -> str:
litellm_params: Final = kwargs.get("litellm_params", {})
metadata: Final = litellm_params.get("metadata") or {}
generation_name: Final = metadata.get("generation_name")

13
litellm/integrations/websearch_interception/handler.py
View file

@ -85,6 +85,11 @@ class _WebSearchSettingsView(TypedDict):
websearch_interception_params: WebSearchInterceptionConfig
class _SearchToolConfig(TypedDict, total=False):
search_tool_name: str
litellm_params: Mapping[str, object] | None
class WebSearchInterceptionLogger(CustomLogger):
"""
CustomLogger that intercepts WebSearch tool calls for models that don't
@ -1487,7 +1492,7 @@ class WebSearchInterceptionLogger(CustomLogger):
return None
def _select_search_tool_from_router(self, llm_router: object) -> dict[str, Any] | None:
def _select_search_tool_from_router(self, llm_router: object) -> "_SearchToolConfig | None":
if llm_router is None or not hasattr(llm_router, "search_tools"):
return None
search_tools: Final = list(getattr(llm_router, "search_tools") or [])
@ -1495,9 +1500,9 @@ class WebSearchInterceptionLogger(CustomLogger):
def _select_search_tool_from_list(
self,
search_tools: list[dict[str, Any]],
search_tools: list[_SearchToolConfig],
source: str,
) -> dict[str, Any] | None:
) -> "_SearchToolConfig | None":
if self.search_tool_name:
matching_tools = [tool for tool in search_tools if tool.get("search_tool_name") == self.search_tool_name]
if matching_tools:
@ -1692,7 +1697,7 @@ class WebSearchInterceptionLogger(CustomLogger):
@staticmethod
def initialize_from_proxy_config(
litellm_settings: dict[str, Any],
litellm_settings: Mapping[str, WebSearchInterceptionConfig],
callback_specific_params: Mapping[str, object],
) -> "WebSearchInterceptionLogger":
"""

57
litellm/litellm_core_utils/litellm_logging.py
View file

@ -10,7 +10,7 @@ import subprocess
import sys
import time
import traceback
from collections.abc import Callable, Mapping
from collections.abc import Callable, Mapping, Sequence
from datetime import datetime as dt_object
from functools import lru_cache
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Union, cast
@ -176,6 +176,9 @@ from .initialize_dynamic_callback_params import (
from .specialty_caches.dynamic_logging_cache import DynamicLoggingCache
if TYPE_CHECKING:
from mcp.types import EmbeddedResource, ImageContent, TextContent
from litellm.integrations.otel.logger import OpenTelemetryV2
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
try:
from litellm_enterprise.enterprise_callbacks.callback_controls import (
@ -211,14 +214,30 @@ except Exception as e:
PagerDutyAlerting = CustomLogger
EnterpriseCallbackControls = None
EnterpriseStandardLoggingPayloadSetupVAR = None
_in_memory_loggers: Final[list[Any]] = []
if TYPE_CHECKING:
from litellm.integrations.generic_api.generic_api_callback import (
GenericAPILogger as _GenericAPILoggerCls,
)
_STANDARD_LOGGING_METADATA_KEYS: Final[frozenset] = frozenset(StandardLoggingMetadata.__annotations__.keys())
_GENERIC_API_LOGGER_CLS: Final = _GenericAPILoggerCls
_RESEND_EMAIL_LOGGER_FACTORY: Final = CustomLogger
_SENDGRID_EMAIL_LOGGER_FACTORY: Final = CustomLogger
_SMTP_EMAIL_LOGGER_FACTORY: Final = CustomLogger
_PAGERDUTY_ALERTING_FACTORY: Final = CustomLogger
else:
_GENERIC_API_LOGGER_CLS: Final = GenericAPILogger
_RESEND_EMAIL_LOGGER_FACTORY: Final = ResendEmailLogger
_SENDGRID_EMAIL_LOGGER_FACTORY: Final = SendGridEmailLogger
_SMTP_EMAIL_LOGGER_FACTORY: Final = SMTPEmailLogger
_PAGERDUTY_ALERTING_FACTORY: Final = PagerDutyAlerting
_in_memory_loggers: Final[list[CustomLogger]] = []
_STANDARD_LOGGING_METADATA_KEYS: Final[frozenset[str]] = frozenset(StandardLoggingMetadata.__annotations__.keys())
### GLOBAL VARIABLES ###
# Cache custom pricing keys as frozenset for O(1) lookups instead of looping through 49 keys
_CUSTOM_PRICING_KEYS: Final[frozenset] = frozenset(CustomPricingLiteLLMParams.model_fields.keys())
_CUSTOM_PRICING_KEYS: Final[frozenset[str]] = frozenset(CustomPricingLiteLLMParams.model_fields.keys())
sentry_sdk_instance = None
capture_exception = None
@ -1285,7 +1304,9 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.exception("LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e)
return response_obj
def _parse_post_mcp_call_hook_response(self, response: MCPPostCallResponseObject | None) -> Any:
def _parse_post_mcp_call_hook_response(
self, response: MCPPostCallResponseObject | None
) -> "Sequence[TextContent | ImageContent | EmbeddedResource] | None":
"""
Parse the response from the post_mcp_tool_call_hook
@ -1729,7 +1750,7 @@ class Logging(LiteLLMLoggingBaseClass):
self.completion_start_time = completion_start_time
self.model_call_details["completion_start_time"] = self.completion_start_time
def normalize_logging_result(self, result: Any) -> Any:
def normalize_logging_result(self, result: Any) -> object:
"""
Some endpoints return a different type of result than what is expected by the logging system.
This function is used to normalize the result to the expected type.
@ -1765,7 +1786,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
return logging_result
def _merge_hidden_params_from_response_into_metadata(self, logging_result: Any) -> None:
def _merge_hidden_params_from_response_into_metadata(self, logging_result: object) -> None:
"""
Copy response._hidden_params into litellm_params.metadata['hidden_params'].
@ -1826,7 +1847,9 @@ class Logging(LiteLLMLoggingBaseClass):
if (standard_logging_payload := self.model_call_details.get("standard_logging_object")) is not None:
emit_standard_logging_payload(standard_logging_payload)
def _build_standard_logging_payload(self, init_response_obj: Any, start_time: Any, end_time: Any) -> Any:
def _build_standard_logging_payload(
self, init_response_obj: object, start_time: Any, end_time: Any
) -> StandardLoggingPayload | None:
"""Build StandardLoggingPayload and accumulate its construction time."""
_start: Final = time.time()
payload: Final = get_standard_logging_object_payload(
@ -1947,7 +1970,7 @@ class Logging(LiteLLMLoggingBaseClass):
def _is_recognized_call_type_for_logging(
self,
logging_result: Any,
logging_result: object,
):
"""
Returns True if the call type is recognized for logging (eg. ModelResponse, ModelResponseStream, etc.)
@ -4216,7 +4239,7 @@ def _init_custom_logger_compatible_class(
for callback in _in_memory_loggers:
if isinstance(callback, PagerDutyAlerting):
return callback
pagerduty_logger: Final = PagerDutyAlerting(**custom_logger_init_args)
pagerduty_logger: Final = _PAGERDUTY_ALERTING_FACTORY(**custom_logger_init_args)
_in_memory_loggers.append(pagerduty_logger)
return pagerduty_logger
elif logging_integration == "anthropic_cache_control_hook":
@ -4246,7 +4269,7 @@ def _init_custom_logger_compatible_class(
return _gcs_pubsub_logger
elif logging_integration == "generic_api":
for callback in _in_memory_loggers:
if isinstance(callback, GenericAPILogger):
if isinstance(callback, _GENERIC_API_LOGGER_CLS):
return callback
generic_api_logger: Final = GenericAPILogger()
_in_memory_loggers.append(generic_api_logger)
@ -4255,21 +4278,21 @@ def _init_custom_logger_compatible_class(
for callback in _in_memory_loggers:
if isinstance(callback, ResendEmailLogger):
return callback
resend_email_logger: Final = ResendEmailLogger()
resend_email_logger: Final = _RESEND_EMAIL_LOGGER_FACTORY()
_in_memory_loggers.append(resend_email_logger)
return resend_email_logger
elif logging_integration == "sendgrid_email":
for callback in _in_memory_loggers:
if isinstance(callback, SendGridEmailLogger):
return callback
sendgrid_email_logger: Final = SendGridEmailLogger()
sendgrid_email_logger: Final = _SENDGRID_EMAIL_LOGGER_FACTORY()
_in_memory_loggers.append(sendgrid_email_logger)
return sendgrid_email_logger
elif logging_integration == "smtp_email":
for callback in _in_memory_loggers:
if isinstance(callback, SMTPEmailLogger):
return callback
smtp_email_logger: Final = SMTPEmailLogger()
smtp_email_logger: Final = _SMTP_EMAIL_LOGGER_FACTORY()
_in_memory_loggers.append(smtp_email_logger)
return smtp_email_logger
elif logging_integration == "humanloop":
@ -4336,7 +4359,7 @@ def _init_custom_logger_compatible_class(
return None
def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list) -> Any | None:
def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[CustomLogger]) -> "OpenTelemetryV2 | None":
"""If ``LITELLM_OTEL_V2`` is on, build (or reuse) a single ``OpenTelemetryV2``
instance configured via the preset for ``callback_name``.
@ -4367,7 +4390,7 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list) -> An
return v2_logger
def _maybe_auto_initialize_arize_phoenix(_in_memory_loggers: list) -> None:
def _maybe_auto_initialize_arize_phoenix(_in_memory_loggers: list[CustomLogger]) -> None:
"""
Auto-initialize ArizePhoenixLogger when Phoenix env vars are detected.
@ -4594,7 +4617,7 @@ def get_custom_logger_compatible_class(
return callback
elif logging_integration == "generic_api":
for callback in _in_memory_loggers:
if isinstance(callback, GenericAPILogger):
if isinstance(callback, _GENERIC_API_LOGGER_CLS):
return callback
elif logging_integration == "resend_email":
for callback in _in_memory_loggers:

50
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -549,7 +549,7 @@ def _fetch_and_extract_template(
return chat_template, bos_token, eos_token
async def ahf_chat_template(model: str, messages: list, chat_template: Any | None = None):
async def ahf_chat_template(model: str, messages: list, chat_template: str | None = None):
"""HuggingFace chat template (async version)"""
from litellm.litellm_core_utils.prompt_templates.huggingface_template_handler import (
_aget_chat_template_file,
@ -576,7 +576,7 @@ async def ahf_chat_template(model: str, messages: list, chat_template: Any | Non
)
def hf_chat_template(model: str, messages: list, chat_template: Any | None = None):
def hf_chat_template(model: str, messages: list, chat_template: str | None = None):
"""HuggingFace chat template (sync version)"""
from litellm.litellm_core_utils.prompt_templates.huggingface_template_handler import (
_get_chat_template_file,
@ -1130,7 +1130,7 @@ def convert_to_azure_openai_messages(
def infer_protocol_value(
value: Any,
value: object,
) -> Literal[
"string_value",
"number_value",
@ -1702,7 +1702,9 @@ def convert_function_to_anthropic_tool_invoke(
_name: Final = get_attribute_or_key(function_call, "name") or ""
_arguments: Final = get_attribute_or_key(function_call, "arguments")
tool_input = parse_tool_call_arguments(_arguments, tool_name=_name, context="Anthropic function to tool invoke")
tool_input: Final = parse_tool_call_arguments(
_arguments, tool_name=_name, context="Anthropic function to tool invoke"
)
anthropic_tool_invoke: Final = [
AnthropicMessagesToolUseParam(
@ -1764,7 +1766,7 @@ def convert_to_anthropic_tool_invoke(
Fixes: https://github.com/BerriAI/litellm/issues/17737
"""
anthropic_tool_invoke: Final[list[AnthropicMessagesToolUseParam | dict[str, Any]]] = []
anthropic_tool_invoke: Final[list[AnthropicMessagesToolUseParam | dict[str, object]]] = []
for tool in tool_calls:
if not get_attribute_or_key(tool, "type") == "function":
@ -1785,7 +1787,7 @@ def convert_to_anthropic_tool_invoke(
# Server tool IDs start with "srvtoolu_"
if tool_id.startswith("srvtoolu_"):
# Create server_tool_use block instead of tool_use
_anthropic_server_tool_use: dict[str, Any] = {
_anthropic_server_tool_use: dict[str, object] = {
"type": "server_tool_use",
"id": tool_id,
"name": tool_name,
@ -2177,7 +2179,7 @@ def _is_orphaned_tool_result(
return False
def _declared_tool_call_ids(message: Mapping[str, Any]) -> frozenset[str]:
def _declared_tool_call_ids(message: Mapping[str, object]) -> frozenset[str]:
tool_calls: Final = message.get("tool_calls")
if not isinstance(tool_calls, list):
return frozenset()
@ -2186,7 +2188,7 @@ def _declared_tool_call_ids(message: Mapping[str, Any]) -> frozenset[str]:
)
def group_tool_exchanges(messages: Sequence[Mapping[str, Any]]) -> tuple[tuple[int, ...], ...]:
def group_tool_exchanges(messages: Sequence[Mapping[str, object]]) -> tuple[tuple[int, ...], ...]:
"""Group message indices into tool exchanges: an assistant row that made
tool calls, together with the tool rows answering the ids it declared.
@ -2204,7 +2206,7 @@ def group_tool_exchanges(messages: Sequence[Mapping[str, Any]]) -> tuple[tuple[i
return tuple(_iter_tool_exchange_groups(messages))
def _iter_tool_exchange_groups(messages: Sequence[Mapping[str, Any]]) -> Iterator[tuple[int, ...]]:
def _iter_tool_exchange_groups(messages: Sequence[Mapping[str, object]]) -> Iterator[tuple[int, ...]]:
index = 0
while index < len(messages):
declared = _declared_tool_call_ids(messages[index])
@ -2409,7 +2411,7 @@ def anthropic_messages_pt(
# Convert ChatCompletionImageUrlObject to dict if needed
image_url_value = m["image_url"]
if isinstance(image_url_value, str):
image_url_input: str | dict[str, Any] = image_url_value
image_url_input: str | dict[str, object] = image_url_value
else:
# ChatCompletionImageUrlObject or dict case - convert to dict
image_url_input = {
@ -3179,7 +3181,7 @@ def _load_image_from_url(image_url):
try:
# Send a GET request to the image URL
client: Final = HTTPHandler(concurrent_limit=1)
response: Final = safe_get(client, image_url)
response: Final[httpx.Response] = safe_get(client, image_url)
response.raise_for_status() # Raise an exception for HTTP errors
# Check the response's content type to ensure it is an image
@ -3382,7 +3384,7 @@ class BedrockImageProcessor:
@staticmethod
def _post_call_image_processing(response: httpx.Response, image_url: str = "") -> tuple[str, str]:
# Check the response's content type to ensure it is an image
content_type = response.headers.get("content-type")
content_type: str | None = response.headers.get("content-type")
# Use helper function to infer content type with fallback logic
content_type = infer_content_type_from_url_and_content(
@ -3406,7 +3408,7 @@ class BedrockImageProcessor:
params={"concurrent_limit": 1},
)
# Send a GET request to the image URL
response: Final = await async_safe_get(client, image_url)
response: Final[httpx.Response] = await async_safe_get(client, image_url)
response.raise_for_status() # Raise an exception for HTTP errors
return BedrockImageProcessor._post_call_image_processing(response, image_url)
@ -3419,7 +3421,7 @@ class BedrockImageProcessor:
try:
client: Final = HTTPHandler(concurrent_limit=1)
# Send a GET request to the image URL
response: Final = safe_get(client, image_url)
response: Final[httpx.Response] = safe_get(client, image_url)
response.raise_for_status() # Raise an exception for HTTP errors
return BedrockImageProcessor._post_call_image_processing(response, image_url)
@ -5328,10 +5330,10 @@ def get_attribute_or_key(tool_or_function, attribute, default=None):
class NormalizedToolCall(TypedDict):
id: str | None
name: str | None
arguments: dict[str, Any]
arguments: dict[str, object]
def _parse_tool_call_arguments(raw: Any, tool_name: str | None, context: str) -> dict[str, Any]:
def _parse_tool_call_arguments(raw: Any, tool_name: str | None, context: str) -> dict[str, object]:
# Anthropic's tool_use blocks already carry a parsed dict in "input";
# chat completions and the Responses API carry a JSON string that may be
# truncated by the model, so route those through the repair-aware parser.
@ -5352,12 +5354,12 @@ def _parse_tool_call_arguments(raw: Any, tool_name: str | None, context: str) ->
def _tool_calls_from_chat_completion_response(
response: Any, include_all_choices: bool = False
response: object, include_all_choices: bool = False
) -> list[NormalizedToolCall]:
choices: Final = get_attribute_or_key(response, "choices", None)
if not (isinstance(choices, list) and choices):
return []
tool_calls: Final[list[Any]] = []
tool_calls: Final[list[object]] = []
for choice in choices if include_all_choices else choices[:1]:
message = get_attribute_or_key(choice, "message", None)
choice_tool_calls = get_attribute_or_key(message, "tool_calls", None) if message else None
@ -5383,7 +5385,7 @@ def _tool_calls_from_chat_completion_response(
return result
def _tool_calls_from_responses_api_response(response: Any) -> list[NormalizedToolCall]:
def _tool_calls_from_responses_api_response(response: object) -> list[NormalizedToolCall]:
output: Final = get_attribute_or_key(response, "output", None)
if not isinstance(output, list):
return []
@ -5406,7 +5408,7 @@ def _tool_calls_from_responses_api_response(response: Any) -> list[NormalizedToo
return result
def _tool_calls_from_anthropic_messages_response(response: Any) -> list[NormalizedToolCall]:
def _tool_calls_from_anthropic_messages_response(response: object) -> list[NormalizedToolCall]:
content: Final = get_attribute_or_key(response, "content", None)
if not isinstance(content, list):
return []
@ -5425,7 +5427,7 @@ def _tool_calls_from_anthropic_messages_response(response: Any) -> list[Normaliz
return result
def get_tool_calls_from_response(response: Any, include_all_choices: bool = False) -> list[NormalizedToolCall]:
def get_tool_calls_from_response(response: object, include_all_choices: bool = False) -> list[NormalizedToolCall]:
"""
Extract tool/function calls from a response object into a normalized
``{"id", "name", "arguments"}`` shape, regardless of which API surface
@ -5456,7 +5458,7 @@ def get_tool_calls_from_response(response: Any, include_all_choices: bool = Fals
return []
def has_tool_with_name(tools: Any, tool_name: str) -> bool:
def has_tool_with_name(tools: object, tool_name: str) -> bool:
"""
Check whether a tools list (as sent to an LLM) includes a tool with the
given name, regardless of shape: OpenAI-style function tools
@ -5482,9 +5484,9 @@ def has_tool_with_name(tools: Any, tool_name: str) -> bool:
def resolve_structured_messages(
messages: list[dict[str, Any]] | None,
messages: list[dict[str, object]] | None,
request_kwargs: dict[str, Any],
) -> list[dict[str, Any]] | None:
) -> list[dict[str, object]] | None:
"""
Normalize a request's messages to OpenAI-spec chat-completions shape,
regardless of which API surface produced them (chat completions,

10
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -145,7 +145,7 @@ class ChunkProcessor:
if first_hidden_params.get("created_at"):
def _created_at(chunk: Any) -> int | float:
def _created_at(chunk: object) -> int | float:
if isinstance(chunk, dict):
params = chunk.get("_hidden_params", {})
else:
@ -158,7 +158,7 @@ class ChunkProcessor:
return chunks
def update_model_response_with_hidden_params(
self, model_response: ModelResponse, chunk: dict[str, Any] | None = None
self, model_response: ModelResponse, chunk: Mapping[str, dict[str, object]] | None = None
) -> ModelResponse:
if chunk is None:
return model_response
@ -176,7 +176,7 @@ class ChunkProcessor:
if not chunks:
return
model: Final = getattr(response, "model", None)
model: Final[str | None] = getattr(response, "model", None)
if not model:
return
@ -214,7 +214,7 @@ class ChunkProcessor:
)
@staticmethod
def _get_chunk_id(chunks: list[dict[str, Any]]) -> str:
def _get_chunk_id(chunks: Sequence[Mapping[str, str]]) -> str:
"""
Chunks:
[{"id": ""}, {"id": "1"}, {"id": "1"}]
@ -225,7 +225,7 @@ class ChunkProcessor:
return ""
@staticmethod
def _get_model_from_chunks(chunks: list[dict[str, Any]], first_chunk_model: str) -> str:
def _get_model_from_chunks(chunks: Sequence[Mapping[str, str]], first_chunk_model: str) -> str:
"""
Get the actual model from chunks, preferring a model that differs from the first chunk.

100
litellm/litellm_core_utils/streaming_handler.py
View file

@ -6,13 +6,14 @@ import logging
import threading
import time
import traceback
from collections.abc import AsyncIterator, Callable, Iterator
from collections.abc import AsyncIterator, Callable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from typing import Any, Final, NoReturn, TypeVar, cast
from typing import Any, Final, NoReturn, Protocol, TypeVar, cast
import anyio
import httpx
from pydantic import BaseModel
from typing_extensions import NotRequired, TypedDict
import litellm
from litellm import verbose_logger
@ -54,7 +55,7 @@ _SYNC_ITER_EXHAUSTED: Final = object()
_GCHUNK_FIELDS: Final[frozenset] = frozenset(GChunk.__annotations__)
def _next_sync_or_exhausted(it: Any) -> Any:
def _next_sync_or_exhausted(it: Any) -> object:
"""
Call next(it) from a thread and return _SYNC_ITER_EXHAUSTED on StopIteration.
@ -68,7 +69,7 @@ def _next_sync_or_exhausted(it: Any) -> Any:
return _SYNC_ITER_EXHAUSTED
def is_async_iterable(obj: Any) -> bool:
def is_async_iterable(obj: object) -> bool:
"""
Check if an object is an async iterable (can be used with 'async for').
@ -81,7 +82,7 @@ def is_async_iterable(obj: Any) -> bool:
return isinstance(obj, collections.abc.AsyncIterable)
def print_verbose(print_statement):
def print_verbose(print_statement: object):
try:
if litellm.set_verbose:
print(print_statement) # noqa: T201
@ -96,18 +97,70 @@ class _ProviderChunkParsed:
@dataclass(frozen=True, slots=True)
class _ProviderChunkEarlyReturn:
value: Any
value: "ModelResponseStream | None"
_ProviderChunkResult = _ProviderChunkParsed | _ProviderChunkEarlyReturn
class _PredibaseStreamData(TypedDict):
token: NotRequired[Mapping[str, str]]
details: Mapping[str, str]
generated_text: str | None
error: str | None
class _Ai21StreamData(TypedDict):
completions: Sequence[Mapping[str, Mapping[str, str]]]
class _MaritalkStreamData(TypedDict):
answer: str
class _NlpCloudStreamData(TypedDict):
generated_text: str
class _AlephAlphaStreamData(TypedDict):
completions: Sequence[Mapping[str, str]]
class _AzureStreamChoice(TypedDict):
delta: Mapping[str, str] | None
finish_reason: str | None
class _AzureStreamData(TypedDict):
choices: Sequence[_AzureStreamChoice]
class _BasetenModelOutput(TypedDict):
data: NotRequired[Sequence[str]]
class _BasetenStreamData(TypedDict):
token: NotRequired[Mapping[str, str]]
model_output: NotRequired["_BasetenModelOutput | str"]
completion: NotRequired[object]
class _DeltaDumpDict(TypedDict):
role: NotRequired[str | None]
tool_calls: NotRequired[Sequence[Mapping[str, object]]]
class _TextCompletionChoiceLike(Protocol):
text: str
finish_reason: str | None
class CustomStreamWrapper:
def __init__(
self,
completion_stream,
model,
logging_obj: Any,
logging_obj: LiteLLMLoggingObject,
custom_llm_provider: str | None = None,
stream_options=None,
make_call: Callable | None = None,
@ -186,7 +239,7 @@ class CustomStreamWrapper:
# Snapshot assumes self._hidden_params is populated from litellm_params
# at init and never mutated during the stream. If that ever changes,
# this cache must be removed.
self._base_hidden_params: dict[str, Any] = {
self._base_hidden_params: dict[str, object] = {
**self._hidden_params,
"response_cost": None,
}
@ -416,7 +469,7 @@ class CustomStreamWrapper:
finish_reason = ""
print_verbose(f"chunk: {chunk}")
if chunk.startswith("data:"):
data_json: Final = json.loads(chunk[5:])
data_json: Final[_PredibaseStreamData] = json.loads(chunk[5:])
print_verbose(f"data json: {data_json}")
if "token" in data_json and "text" in data_json["token"]:
text = data_json["token"]["text"]
@ -446,7 +499,7 @@ class CustomStreamWrapper:
def handle_ai21_chunk(self, chunk): # fake streaming
chunk = chunk.decode("utf-8")
data_json: Final = json.loads(chunk)
data_json: Final[_Ai21StreamData] = json.loads(chunk)
try:
text: Final = data_json["completions"][0]["data"]["text"]
is_finished: Final = True
@ -461,7 +514,7 @@ class CustomStreamWrapper:
def handle_maritalk_chunk(self, chunk): # fake streaming
chunk = chunk.decode("utf-8")
data_json: Final = json.loads(chunk)
data_json: Final[_MaritalkStreamData] = json.loads(chunk)
try:
text: Final = data_json["answer"]
is_finished: Final = True
@ -482,7 +535,7 @@ class CustomStreamWrapper:
if self.model and "dolphin" in self.model:
chunk = self.process_chunk(chunk=chunk)
else:
data_json: Final = json.loads(chunk)
data_json: Final[_NlpCloudStreamData] = json.loads(chunk)
chunk = data_json["generated_text"]
text = chunk
if "[DONE]" in text:
@ -499,7 +552,7 @@ class CustomStreamWrapper:
def handle_aleph_alpha_chunk(self, chunk):
chunk = chunk.decode("utf-8")
data_json: Final = json.loads(chunk)
data_json: Final[_AlephAlphaStreamData] = json.loads(chunk)
try:
text: Final = data_json["completions"][0]["completion"]
is_finished: Final = True
@ -527,7 +580,7 @@ class CustomStreamWrapper:
"finish_reason": finish_reason,
}
elif chunk.startswith("data:"):
data_json: Final = json.loads(chunk[5:]) # chunk.startswith("data:"):
data_json: Final[_AzureStreamData] = json.loads(chunk[5:]) # chunk.startswith("data:"):
try:
if len(data_json["choices"]) > 0:
delta: Final = data_json["choices"][0]["delta"]
@ -616,7 +669,7 @@ class CustomStreamWrapper:
text = ""
is_finished = False
finish_reason = None
choices: Final = getattr(chunk, "choices", [])
choices: Final[Sequence[_TextCompletionChoiceLike]] = getattr(chunk, "choices", [])
if len(choices) > 0:
text = choices[0].text
if choices[0].finish_reason is not None:
@ -637,7 +690,7 @@ class CustomStreamWrapper:
is_finished = False
finish_reason = None
usage = None
choices: Final = getattr(chunk, "choices", [])
choices: Final[Sequence[_TextCompletionChoiceLike]] = getattr(chunk, "choices", [])
if len(choices) > 0:
text = choices[0].text
if choices[0].finish_reason is not None:
@ -654,12 +707,12 @@ class CustomStreamWrapper:
except Exception as e:
raise e
def handle_baseten_chunk(self, chunk):
def handle_baseten_chunk(self, chunk) -> str:
try:
chunk = chunk.decode("utf-8")
if len(chunk) > 0:
if chunk.startswith("data:"):
data_json = json.loads(chunk[5:])
data_json: _BasetenStreamData = json.loads(chunk[5:])
if "token" in data_json and "text" in data_json["token"]:
return data_json["token"]["text"]
else:
@ -1325,13 +1378,14 @@ class CustomStreamWrapper:
if response_obj["is_finished"]:
self.received_finish_reason = response_obj["finish_reason"]
if "usage" in response_obj is not None:
_codestral_usage: Final[Usage] = response_obj["usage"]
setattr(
model_response,
"usage",
litellm.Usage(
prompt_tokens=response_obj["usage"].prompt_tokens,
completion_tokens=response_obj["usage"].completion_tokens,
total_tokens=response_obj["usage"].total_tokens,
prompt_tokens=_codestral_usage.prompt_tokens,
completion_tokens=_codestral_usage.completion_tokens,
total_tokens=_codestral_usage.total_tokens,
),
)
elif self.custom_llm_provider == "azure_text":
@ -1474,7 +1528,7 @@ class CustomStreamWrapper:
is None
):
t.function.arguments = ""
_json_delta: Final = delta.model_dump()
_json_delta: Final[_DeltaDumpDict] = delta.model_dump()
if "role" not in _json_delta or _json_delta["role"] is None:
_json_delta["role"] = "assistant" # mistral's api returns role as None
if "tool_calls" in _json_delta and isinstance(_json_delta["tool_calls"], list):
@ -1744,7 +1798,7 @@ class CustomStreamWrapper:
usage.cost, copy it into _hidden_params so litellm's cost
calculator uses it instead of a token-based estimate.
"""
_usage: Final = getattr(response, "usage", None)
_usage: Final[Usage | None] = getattr(response, "usage", None)
if _usage is not None and hasattr(_usage, "cost") and _usage.cost is not None:
if "additional_headers" not in response._hidden_params:
response._hidden_params["additional_headers"] = {}

5
litellm/llms/anthropic/chat/handler.py
View file

@ -58,6 +58,7 @@ from ..common_utils import AnthropicError, process_anthropic_headers
from .transformation import ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY, AnthropicConfig
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.llms.base_llm.chat.transformation import BaseConfig
@ -206,7 +207,7 @@ class AnthropicChatCompletion(BaseLLM):
client: AsyncHTTPHandler | None,
encoding,
api_key,
logging_obj,
logging_obj: "LiteLLMLoggingObj",
stream,
_is_function_call,
data: dict,
@ -324,7 +325,7 @@ class AnthropicChatCompletion(BaseLLM):
print_verbose: Callable,
encoding,
api_key,
logging_obj,
logging_obj: "LiteLLMLoggingObj",
optional_params: dict,
timeout: float | httpx.Timeout,
litellm_params: dict,

6
litellm/llms/azure/azure.py
View file

@ -228,7 +228,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
litellm_params=litellm_params,
)
data = {"model": None, "messages": messages, **optional_params}
data: dict[str, object] = {"model": None, "messages": messages, **optional_params}
elif litellm.AzureOpenAIGPT5Config.is_model_gpt_5_model(model=litellm_params.get("base_model") or model):
data = litellm.AzureOpenAIGPT5Config().transform_request(
model=model,
@ -482,12 +482,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
def streaming(
self,
logging_obj,
logging_obj: LiteLLMLoggingObj,
api_base: str,
api_key: str | None,
api_version: str,
dynamic_params: bool,
data: dict,
data: dict[str, object],
model: str,
timeout: Any,
max_retries: int,

5
litellm/llms/azure/chat/o_series_handler.py
View file

@ -5,10 +5,11 @@ Written separately to handle faking streaming for o1 and o3 models.
"""
from collections.abc import Callable
from typing import TYPE_CHECKING, Any, Optional
from typing import TYPE_CHECKING, Optional
import httpx
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.types.utils import ModelResponse
from ...openai.openai import OpenAIChatCompletion
@ -25,7 +26,7 @@ class AzureOpenAIO1ChatCompletion(BaseAzureLLM, OpenAIChatCompletion):
timeout: float | httpx.Timeout,
optional_params: dict,
litellm_params: dict,
logging_obj: Any,
logging_obj: LiteLLMLoggingObj,
model: str | None = None,
messages: list | None = None,
print_verbose: Callable | None = None,

9
litellm/llms/azure/completion/handler.py
View file

@ -3,6 +3,7 @@ from typing import Any, Final
from openai import AsyncAzureOpenAI, AzureOpenAI
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.prompt_templates.factory import prompt_factory
from litellm.utils import CustomStreamWrapper, ModelResponse, TextCompletionResponse
@ -39,9 +40,9 @@ class AzureTextCompletion(BaseAzureLLM):
azure_ad_token_provider: Callable | None,
print_verbose: Callable,
timeout,
logging_obj,
logging_obj: LiteLLMLoggingObj,
optional_params,
litellm_params,
litellm_params: dict[str, object],
logger_fn,
acompletion: bool = False,
headers: dict | None = None,
@ -246,7 +247,7 @@ class AzureTextCompletion(BaseAzureLLM):
def streaming(
self,
logging_obj,
logging_obj: LiteLLMLoggingObj,
api_base: str,
api_key: str | None,
api_version: str,
@ -299,7 +300,7 @@ class AzureTextCompletion(BaseAzureLLM):
async def async_streaming(
self,
logging_obj,
logging_obj: LiteLLMLoggingObj,
api_base: str,
api_key: str | None,
api_version: str,

3
litellm/llms/azure_ai/anthropic/handler.py
View file

@ -9,6 +9,7 @@ from typing import Final
import httpx
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.chat.handler import AnthropicChatCompletion
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
@ -40,7 +41,7 @@ class AzureAnthropicChatCompletion(AnthropicChatCompletion):
print_verbose: Callable,
encoding,
api_key,
logging_obj,
logging_obj: LiteLLMLoggingObj,
optional_params: dict,
timeout: float | httpx.Timeout,
litellm_params: dict,

2
litellm/llms/azure_ai/chat/transformation.py
View file

@ -248,7 +248,7 @@ class AzureAIStudioConfig(OpenAIConfig):
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
encoding=encoding,
encoding=encoding if encoding is not None else None,
api_key=api_key,
json_mode=json_mode,
)

2
litellm/llms/bedrock/chat/converse_handler.py
View file

@ -89,7 +89,7 @@ class BedrockConverseLLM(BaseAWSLLM):
model_response: ModelResponse,
timeout: float | httpx.Timeout | None,
encoding,
logging_obj,
logging_obj: LiteLLMLoggingObject,
stream,
optional_params: dict,
litellm_params: dict,

4
litellm/llms/codestral/completion/handler.py
View file

@ -195,7 +195,7 @@ class CodestralTextCompletion:
print_verbose: Callable,
encoding,
api_key: str,
logging_obj,
logging_obj: LiteLLMLogging,
optional_params: dict,
timeout: float | httpx.Timeout,
acompletion=None,
@ -383,7 +383,7 @@ class CodestralTextCompletion:
print_verbose: Callable,
encoding,
api_key,
logging_obj,
logging_obj: LiteLLMLogging,
data: dict,
timeout: float | httpx.Timeout,
optional_params=None,

2
litellm/llms/custom_httpx/aiohttp_handler.py
View file

@ -221,7 +221,7 @@ class BaseLLMAIOHTTPHandler:
timeout=timeout,
stream=stream,
files=files,
content=content,
content=content if content is not None else None,
params=params,
)
except httpx.HTTPStatusError as e:

61
litellm/llms/custom_httpx/http_handler.py
View file

@ -7,9 +7,9 @@ import ssl
import sys
import threading
import time
from collections.abc import Callable, Mapping
from collections.abc import AsyncIterable, Callable, Iterable, Mapping
from http.cookiejar import CookieJar, DefaultCookiePolicy
from typing import TYPE_CHECKING, Any, Final, Optional
from typing import TYPE_CHECKING, Any, Final, Optional, TypeAlias, TypedDict
import certifi
import httpx
@ -62,8 +62,23 @@ except Exception:
# https://docs.aiohttp.org/en/stable/client_reference.html#aiohttp.TCPConnector
_AIOHTTP_SUPPORTS_SOCKET_FACTORY: Final = "socket_factory" in inspect.signature(TCPConnector.__init__).parameters
_AddrInfo: TypeAlias = tuple[int | socket.AddressFamily, int | socket.SocketKind, int, str, tuple[object, ...]]
def _build_aiohttp_keepalive_socket_factory() -> Callable[[tuple[Any, ...]], socket.socket] | None:
_RequestContent: TypeAlias = str | bytes | Iterable[bytes] | AsyncIterable[bytes]
class _TCPConnectorKwargs(TypedDict, total=False):
local_addr: tuple[str, int] | None
ssl: "ssl.SSLContext | bool"
keepalive_timeout: float
ttl_dns_cache: int
enable_cleanup_closed: bool
limit: int
limit_per_host: int
socket_factory: Callable[[_AddrInfo], socket.socket]
def _build_aiohttp_keepalive_socket_factory() -> Callable[[_AddrInfo], socket.socket] | None:
"""
Build a socket_factory that enables SO_KEEPALIVE on aiohttp TCP sockets.
@ -78,7 +93,7 @@ def _build_aiohttp_keepalive_socket_factory() -> Callable[[tuple[Any, ...]], soc
if not AIOHTTP_SO_KEEPALIVE or not _AIOHTTP_SUPPORTS_SOCKET_FACTORY:
return None
def factory(addr_info: tuple[Any, ...]) -> socket.socket:
def factory(addr_info: _AddrInfo) -> socket.socket:
family, type_, proto = addr_info[0], addr_info[1], addr_info[2]
sock: Final = socket.socket(family=family, type=type_, proto=proto)
sock.setblocking(False)
@ -163,8 +178,8 @@ _STREAMING_ERROR_BODY_READ_EXECUTOR: Final = concurrent.futures.ThreadPoolExecut
def _prepare_request_data_and_content(
data: dict | str | bytes | None = None,
content: Any = None,
) -> tuple[dict | Mapping | None, Any]:
content: _RequestContent | None = None,
) -> tuple[dict | Mapping | None, _RequestContent | None]:
"""
Helper function to route data/content parameters correctly for httpx requests
@ -528,7 +543,7 @@ class AsyncHTTPHandler:
def __init__(
self,
timeout: float | httpx.Timeout | None = None,
event_hooks: Mapping[str, list[Callable[..., Any]]] | None = None,
event_hooks: Mapping[str, list[Callable[..., object]]] | None = None,
concurrent_limit=None, # Kept for backward compatibility, but ignored (no limits)
client_alias: str | None = None, # name for client in logs
ssl_verify: VerifyTypes | None = None,
@ -566,7 +581,7 @@ class AsyncHTTPHandler:
def create_client(
self,
timeout: float | httpx.Timeout | None,
event_hooks: Mapping[str, list[Callable[..., Any]]] | None,
event_hooks: Mapping[str, list[Callable[..., object]]] | None,
ssl_verify: VerifyTypes | None = None,
shared_session: Optional["ClientSession"] = None,
) -> httpx.AsyncClient:
@ -648,7 +663,7 @@ class AsyncHTTPHandler:
stream: bool = False,
logging_obj: LiteLLMLoggingObject | None = None,
files: RequestFiles | None = None,
content: Any = None,
content: _RequestContent | None = None,
):
start_time: Final = time.time()
try:
@ -691,7 +706,7 @@ class AsyncHTTPHandler:
end_time: Final = time.time()
time_delta: Final = round(end_time - start_time, 3)
headers = {}
error_response: Final = getattr(e, "response", None)
error_response: Final[httpx.Response | None] = getattr(e, "response", None)
if error_response is not None:
for key, value in error_response.headers.items():
headers[f"response_headers-{key}"] = value
@ -716,7 +731,7 @@ class AsyncHTTPHandler:
headers: dict | None = None,
timeout: float | httpx.Timeout | None = None,
stream: bool = False,
content: Any = None,
content: _RequestContent | None = None,
):
try:
if timeout is None:
@ -755,7 +770,7 @@ class AsyncHTTPHandler:
await new_client.aclose()
except httpx.TimeoutException as e:
headers = {}
error_response: Final = getattr(e, "response", None)
error_response: Final[httpx.Response | None] = getattr(e, "response", None)
if error_response is not None:
for key, value in error_response.headers.items():
headers[f"response_headers-{key}"] = value
@ -780,7 +795,7 @@ class AsyncHTTPHandler:
headers: dict | None = None,
timeout: float | httpx.Timeout | None = None,
stream: bool = False,
content: Any = None,
content: _RequestContent | None = None,
):
try:
if timeout is None:
@ -819,7 +834,7 @@ class AsyncHTTPHandler:
await new_client.aclose()
except httpx.TimeoutException as e:
headers = {}
error_response: Final = getattr(e, "response", None)
error_response: Final[httpx.Response | None] = getattr(e, "response", None)
if error_response is not None:
for key, value in error_response.headers.items():
headers[f"response_headers-{key}"] = value
@ -844,7 +859,7 @@ class AsyncHTTPHandler:
headers: dict | None = None,
timeout: float | httpx.Timeout | None = None,
stream: bool = False,
content: Any = None,
content: _RequestContent | None = None,
):
try:
if timeout is None:
@ -895,7 +910,7 @@ class AsyncHTTPHandler:
params: dict | None = None,
headers: dict | None = None,
stream: bool = False,
content: Any = None,
content: _RequestContent | None = None,
):
"""
Making POST request for a single connection client.
@ -993,7 +1008,7 @@ class AsyncHTTPHandler:
def _get_ssl_connector_kwargs(
ssl_verify: bool | None = None,
ssl_context: ssl.SSLContext | None = None,
) -> dict[str, Any]:
) -> _TCPConnectorKwargs:
"""
Helper method to get SSL connector initialization arguments for aiohttp TCPConnector.
@ -1004,7 +1019,7 @@ class AsyncHTTPHandler:
Returns:
Dict with appropriate SSL configuration for TCPConnector
"""
connector_kwargs: Final[dict[str, Any]] = {
connector_kwargs: Final[_TCPConnectorKwargs] = {
"local_addr": ("0.0.0.0", 0) if litellm.force_ipv4 else None,
}
@ -1054,7 +1069,7 @@ class AsyncHTTPHandler:
verbose_logger.debug("Creating AiohttpTransport...")
transport_connector_kwargs: Final = {
transport_connector_kwargs: Final[_TCPConnectorKwargs] = {
"keepalive_timeout": AIOHTTP_KEEPALIVE_TIMEOUT,
"ttl_dns_cache": AIOHTTP_TTL_DNS_CACHE,
**connector_kwargs,
@ -1212,7 +1227,7 @@ class HTTPHandler:
stream: bool = False,
timeout: float | httpx.Timeout | None = None,
files: dict | RequestFiles | None = None,
content: Any = None,
content: _RequestContent | None = None,
logging_obj: LiteLLMLoggingObject | None = None,
):
try:
@ -1265,7 +1280,7 @@ class HTTPHandler:
headers: dict | None = None,
stream: bool = False,
timeout: float | httpx.Timeout | None = None,
content: Any = None,
content: _RequestContent | None = None,
):
try:
# Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix)
@ -1315,7 +1330,7 @@ class HTTPHandler:
headers: dict | None = None,
stream: bool = False,
timeout: float | httpx.Timeout | None = None,
content: Any = None,
content: _RequestContent | None = None,
):
try:
# Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix)
@ -1364,7 +1379,7 @@ class HTTPHandler:
headers: dict | None = None,
timeout: float | httpx.Timeout | None = None,
stream: bool = False,
content: Any = None,
content: _RequestContent | None = None,
):
try:
# Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix)

64
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -5,7 +5,8 @@ import ssl
from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping
from contextlib import asynccontextmanager
from functools import lru_cache
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypeVar, Union, cast, get_type_hints
from types import ModuleType
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypedDict, TypeVar, Union, cast, get_type_hints
from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
import httpx
@ -148,6 +149,7 @@ from .http_handler import get_shared_realtime_ssl_context
if TYPE_CHECKING:
from aiohttp import ClientSession
from websockets.asyncio.client import ClientConnection
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -176,6 +178,19 @@ else:
_ResponseT = TypeVar("_ResponseT")
class _DeleteRequestKwargs(TypedDict, total=False):
url: str
headers: dict[str, str]
timeout: float | httpx.Timeout | None
json: dict[str, object]
class _MediaUploadKwargs(TypedDict, total=False):
headers: dict[str, str]
content: Iterator[bytes] | AsyncIterator[bytes]
timeout: float | httpx.Timeout
def _google_genai_streaming_hidden_params(
*,
api_base: str,
@ -1413,7 +1428,7 @@ class BaseLLMHTTPHandler:
headers: dict[str, object] | None,
provider_config: BaseOCRConfig,
litellm_params: dict,
) -> tuple[dict[str, Any], str, dict[str, Any], None]:
) -> tuple[dict[str, object], str, dict[str, object], None]:
"""
Shared logic for preparing OCR requests.
Returns: (headers, complete_url, data, files)
@ -1479,7 +1494,7 @@ class BaseLLMHTTPHandler:
headers: dict[str, object] | None,
provider_config: BaseOCRConfig,
litellm_params: dict,
) -> tuple[dict[str, Any], str, dict[str, Any], None]:
) -> tuple[dict[str, object], str, dict[str, object], None]:
"""
Async version of _prepare_ocr_request for providers that need async transforms.
Returns: (headers, complete_url, data, files)
@ -2361,14 +2376,14 @@ class BaseLLMHTTPHandler:
model: str,
input: str | ResponseInputParam,
custom_llm_provider: str,
response_api_optional_request_params: dict[str, Any],
response_api_optional_request_params: dict[str, object],
litellm_params: GenericLiteLLMParams,
logging_obj: LiteLLMLoggingObj,
) -> tuple[
str,
str | ResponseInputParam,
str,
dict[str, Any],
dict[str, object],
GenericLiteLLMParams,
]:
if not _has_pre_call_deployment_hook(logging_obj):
@ -2894,7 +2909,7 @@ class BaseLLMHTTPHandler:
},
)
delete_kwargs: Final[dict[str, Any]] = {
delete_kwargs: Final[_DeleteRequestKwargs] = {
"url": url,
"headers": headers,
"timeout": timeout,
@ -2984,7 +2999,7 @@ class BaseLLMHTTPHandler:
},
)
delete_kwargs: Final[dict[str, Any]] = {
delete_kwargs: Final[_DeleteRequestKwargs] = {
"url": url,
"headers": headers,
"timeout": timeout,
@ -3725,7 +3740,7 @@ class BaseLLMHTTPHandler:
timeout: float | httpx.Timeout | None,
) -> httpx.Response:
headers: Final = {**base_headers, "Content-Type": content_type}
kwargs: Final[dict[str, Any]] = {
kwargs: Final[_MediaUploadKwargs] = {
"headers": headers,
"content": self._iter_in_blocks(body_stream.iter_bytes(), self._MEDIA_UPLOAD_BLOCK_SIZE),
}
@ -3762,7 +3777,7 @@ class BaseLLMHTTPHandler:
break
yield cast(bytes, block)
kwargs: Final[dict[str, Any]] = {"headers": headers, "content": _abody()}
kwargs: Final[_MediaUploadKwargs] = {"headers": headers, "content": _abody()}
if timeout is not None:
kwargs["timeout"] = timeout
resp: Final = await client.client.post(url, **kwargs)
@ -5242,7 +5257,7 @@ class BaseLLMHTTPHandler:
def _wrap_responses_response_as_fake_stream(
self,
result: Any,
result: ResponsesAPIResponse,
model: str,
responses_api_provider_config: BaseResponsesAPIConfig,
logging_obj: "LiteLLMLoggingObj",
@ -5365,7 +5380,7 @@ class BaseLLMHTTPHandler:
async def _call_agentic_completion_hooks(
self,
response: Any,
response: object,
model: str,
messages: list[dict],
anthropic_messages_provider_config: "BaseAnthropicMessagesConfig",
@ -5536,7 +5551,7 @@ class BaseLLMHTTPHandler:
async def _call_agentic_chat_completion_hooks(
self,
response: Any,
response: ModelResponse,
model: str,
messages: list[dict],
optional_params: dict,
@ -5760,14 +5775,14 @@ class BaseLLMHTTPHandler:
@staticmethod
async def _open_realtime_backend_ws(
websockets_module: Any,
websockets_module: ModuleType,
url: str,
headers: dict,
ssl_context: Any,
ssl_context: bool | str | ssl.SSLContext,
*,
open_timeout: float = 8.0,
max_attempts: int = 3,
) -> Any:
) -> "ClientConnection":
"""Open the backend realtime websocket, retrying a hung open handshake.
The upstream Live handshake (e.g. Gemini Live) intermittently hangs on
@ -5826,7 +5841,6 @@ class BaseLLMHTTPHandler:
query_params: RealtimeQueryParams | None = None,
):
import websockets
from websockets.asyncio.client import ClientConnection
url: Final = provider_config.get_complete_url(api_base, model, api_key)
headers = provider_config.validate_environment(
@ -5844,12 +5858,12 @@ class BaseLLMHTTPHandler:
ssl_context.verify_mode = ssl.CERT_NONE
backend_ws: Final = await self._open_realtime_backend_ws(websockets, url, headers, ssl_context)
async with backend_ws:
_request_data: Final[dict[str, Any]] = {}
_request_data: Final[dict[str, object]] = {}
if litellm_metadata:
_request_data["litellm_metadata"] = litellm_metadata
realtime_streaming: Final = RealTimeStreaming(
websocket,
cast(ClientConnection, backend_ws),
backend_ws,
logging_obj,
provider_config,
model,
@ -6008,7 +6022,7 @@ class BaseLLMHTTPHandler:
)
else:
url = provider_config.get_complete_url(api_base=api_base, model=model or "", api_version=api_version)
headers: dict[str, Any] = provider_config.validate_environment(
headers: dict[str, object] = provider_config.validate_environment(
headers={}, model=model or "", api_key=api_key
)
else:
@ -6079,7 +6093,7 @@ class BaseLLMHTTPHandler:
if provider_config is not None:
url = provider_config.get_realtime_calls_url(api_base=api_base, model=model or "", api_version=api_version)
headers: dict[str, Any] = provider_config.get_realtime_calls_headers(ephemeral_key=openai_ephemeral_key)
headers: dict[str, object] = provider_config.get_realtime_calls_headers(ephemeral_key=openai_ephemeral_key)
else:
url = f"{api_base.rstrip('/')}/v1/realtime/calls"
headers = {
@ -6247,7 +6261,7 @@ class BaseLLMHTTPHandler:
yield backend
async with _backend_connection() as backend_ws:
_request_data: Final[dict[str, Any]] = {}
_request_data: Final[dict[str, object]] = {}
if litellm_metadata:
_request_data["litellm_metadata"] = litellm_metadata
@ -9444,7 +9458,7 @@ class BaseLLMHTTPHandler:
litellm_params=dict(litellm_params),
extra_body=extra_body,
)
all_optional_params: Final[dict[str, Any]] = dict(litellm_params)
all_optional_params: Final[dict[str, object]] = dict(litellm_params)
all_optional_params.update(vector_store_search_optional_params or {})
headers, signed_json_body = vector_store_provider_config.sign_request(
headers=headers,
@ -9540,7 +9554,7 @@ class BaseLLMHTTPHandler:
extra_body=extra_body,
)
all_optional_params: Final[dict[str, Any]] = dict(litellm_params)
all_optional_params: Final[dict[str, object]] = dict(litellm_params)
all_optional_params.update(vector_store_search_optional_params or {})
headers, signed_json_body = vector_store_provider_config.sign_request(
@ -9860,7 +9874,7 @@ class BaseLLMHTTPHandler:
url: Final = api_base
params: Final[dict[str, Any]] = {}
params: Final[dict[str, object]] = {}
if after is not None:
params["after"] = after
if before is not None:
@ -9938,7 +9952,7 @@ class BaseLLMHTTPHandler:
url: Final = api_base
params: Final[dict[str, Any]] = {}
params: Final[dict[str, object]] = {}
if after is not None:
params["after"] = after
if before is not None:

2
litellm/llms/github_copilot/chat/transformation.py
View file

@ -277,7 +277,7 @@ class GithubCopilotConfig(OpenAIConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: Any,
encoding: object,
api_key: str | None = None,
json_mode: bool | None = None,
) -> "ModelResponse":

23
litellm/llms/oci/chat/transformation.py
View file

@ -10,7 +10,7 @@ implement the LiteLLM BaseConfig interface. Heavy-lifting lives in:
"""
import json
from collections.abc import AsyncIterator, Iterator
from collections.abc import AsyncIterator, Callable, Iterator
from typing import TYPE_CHECKING, Any, Final
import httpx
@ -713,8 +713,25 @@ class OCIChatConfig(BaseConfig):
class OCIStreamWrapper(CustomStreamWrapper):
"""Custom stream wrapper that dispatches OCI SSE chunks to the correct handler."""
def __init__(self, **kwargs: Any):
super().__init__(**kwargs)
def __init__(
self,
completion_stream: object,
model: str,
logging_obj: LiteLLMLoggingObj,
custom_llm_provider: str | None = None,
stream_options: object = None,
make_call: Callable[..., object] | None = None,
_response_headers: dict[str, object] | None = None,
) -> None:
super().__init__(
completion_stream=completion_stream,
model=model,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
stream_options=stream_options,
make_call=make_call,
_response_headers=_response_headers,
)
# Tracks whether any prior Cohere chunk in this stream has emitted
# tool calls. The Cohere handler uses this to decide whether the
# terminal consolidation chunk's tool calls are duplicates (suppress)

4
litellm/llms/openai/completion/handler.py
View file

@ -217,7 +217,7 @@ class OpenAITextCompletion(BaseLLM):
def streaming(
self,
logging_obj,
logging_obj: LiteLLMLoggingObj,
api_key: str,
data: dict,
headers: dict,
@ -274,7 +274,7 @@ class OpenAITextCompletion(BaseLLM):
async def async_streaming(
self,
logging_obj,
logging_obj: LiteLLMLoggingObj,
api_key: str,
data: dict,
headers: dict,

72
litellm/llms/openai/openai.py
View file

@ -1,6 +1,6 @@
import time
import types
from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Iterator
from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Iterator, Mapping
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast
from urllib.parse import urlparse
@ -61,16 +61,17 @@ class MistralEmbeddingConfig:
def __init__(
self,
) -> None:
locals_: Final = locals().copy()
locals_: Final[Mapping[str, object]] = locals().copy()
for key, value in locals_.items():
if key != "self" and value is not None:
setattr(self.__class__, key, value)
@classmethod
def get_config(cls):
config_attrs: Final[Mapping[str, object]] = cls.__dict__
return {
k: v
for k, v in cls.__dict__.items()
for k, v in config_attrs.items()
if not k.startswith("__")
and not isinstance(
v,
@ -153,7 +154,7 @@ class OpenAIConfig(BaseConfig):
top_p: int | None = None,
response_format: dict | None = None,
) -> None:
locals_: Final = locals().copy()
locals_: Final[Mapping[str, object]] = locals().copy()
for key, value in locals_.items():
if key != "self" and value is not None:
setattr(self.__class__, key, value)
@ -261,7 +262,7 @@ class OpenAIConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: Any,
encoding: object,
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:
@ -299,7 +300,7 @@ class OpenAIConfig(BaseConfig):
streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse,
sync_stream: bool,
json_mode: bool | None = False,
) -> Any:
) -> "OpenAIChatCompletionResponseIterator":
return OpenAIChatCompletionResponseIterator(
streaming_response=streaming_response,
sync_stream=sync_stream,
@ -478,14 +479,14 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
async def _call_agentic_completion_hooks_openai(
self,
response: Any,
response: object,
model: str,
messages: list[dict],
optional_params: dict,
logging_obj: LiteLLMLoggingObj,
stream: bool,
litellm_params: dict,
) -> Any | None:
) -> object | None:
"""
Call agentic completion hooks for all custom loggers (OpenAI Chat Completions API).
@ -536,7 +537,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
kwargs_with_provider["custom_llm_provider"] = custom_llm_provider
# For OpenAI Chat Completions, use the chat completion agentic loop method
agentic_response = await callback.async_run_chat_completion_agentic_loop(
agentic_response: object = await callback.async_run_chat_completion_agentic_loop(
tools=tool_calls,
model=model,
messages=messages,
@ -580,7 +581,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
timeout: float | httpx.Timeout,
optional_params: dict,
litellm_params: dict,
logging_obj: Any,
logging_obj: LiteLLMLoggingObj,
model: str | None = None,
messages: list | None = None,
print_verbose: Callable | None = None,
@ -1590,7 +1591,7 @@ class OpenAIFilesAPI(BaseLLM):
client: OpenAI | AsyncOpenAI | None = None,
_is_async: bool = False,
) -> OpenAI | AsyncOpenAI | None:
received_args: Final = locals()
received_args: Final[Mapping[str, object]] = locals()
openai_client: OpenAI | AsyncOpenAI | None = None
if client is None:
data: Final = {}
@ -1628,7 +1629,7 @@ class OpenAIFilesAPI(BaseLLM):
max_retries: int | None,
organization: str | None,
client: OpenAI | AsyncOpenAI | None = None,
) -> OpenAIFileObject | Coroutine[Any, Any, OpenAIFileObject]:
) -> OpenAIFileObject | Coroutine[None, None, OpenAIFileObject]:
openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
api_key=api_key,
api_base=api_base,
@ -1670,7 +1671,7 @@ class OpenAIFilesAPI(BaseLLM):
max_retries: int | None,
organization: str | None,
client: OpenAI | AsyncOpenAI | None = None,
) -> HttpxBinaryResponseContent | Coroutine[Any, Any, HttpxBinaryResponseContent]:
) -> HttpxBinaryResponseContent | Coroutine[None, None, HttpxBinaryResponseContent]:
openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
api_key=api_key,
api_base=api_base,
@ -1948,7 +1949,7 @@ class OpenAIBatchesAPI(BaseLLM):
client: OpenAI | AsyncOpenAI | None = None,
_is_async: bool = False,
) -> OpenAI | AsyncOpenAI | None:
received_args: Final = locals()
received_args: Final[Mapping[str, object]] = locals()
openai_client: OpenAI | AsyncOpenAI | None = None
if client is None:
data: Final = {}
@ -1986,7 +1987,7 @@ class OpenAIBatchesAPI(BaseLLM):
max_retries: int | None,
organization: str | None,
client: OpenAI | AsyncOpenAI | None = None,
) -> LiteLLMBatch | Coroutine[Any, Any, LiteLLMBatch]:
) -> LiteLLMBatch | Coroutine[None, None, LiteLLMBatch]:
openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
api_key=api_key,
api_base=api_base,
@ -2160,7 +2161,7 @@ class OpenAIAssistantsAPI(BaseLLM):
organization: str | None,
client: OpenAI | None = None,
) -> OpenAI:
received_args: Final = locals()
received_args: Final[Mapping[str, object]] = locals()
if client is None:
data: Final = {}
for k, v in received_args.items():
@ -2185,7 +2186,7 @@ class OpenAIAssistantsAPI(BaseLLM):
organization: str | None,
client: AsyncOpenAI | None = None,
) -> AsyncOpenAI:
received_args: Final = locals()
received_args: Final[Mapping[str, object]] = locals()
if client is None:
data: Final = {}
for k, v in received_args.items():
@ -2848,7 +2849,7 @@ class OpenAIAssistantsAPI(BaseLLM):
assistant_id: str,
additional_instructions: str | None,
instructions: str | None,
metadata: dict | None,
metadata: dict[str, str] | None,
model: str | None,
stream: bool | None,
tools: Iterable[AssistantToolParam] | None,
@ -2912,23 +2913,32 @@ class OpenAIAssistantsAPI(BaseLLM):
assistant_id: str,
additional_instructions: str | None,
instructions: str | None,
metadata: dict | None,
metadata: dict[str, str] | None,
model: str | None,
tools: Iterable[AssistantToolParam] | None,
event_handler: AssistantEventHandler | None,
) -> AssistantStreamManager[AssistantEventHandler]:
data: Final[dict[str, Any]] = {
"thread_id": thread_id,
"assistant_id": assistant_id,
"additional_instructions": additional_instructions,
"instructions": instructions,
"metadata": metadata,
"model": model,
"tools": tools,
}
runs_stream: Final = client.beta.threads.runs.stream
if event_handler is not None:
data["event_handler"] = event_handler
return client.beta.threads.runs.stream(**data)
return runs_stream(
thread_id=thread_id,
assistant_id=assistant_id,
additional_instructions=additional_instructions,
instructions=instructions,
metadata=metadata,
model=model,
tools=tools,
event_handler=event_handler,
)
return runs_stream(
thread_id=thread_id,
assistant_id=assistant_id,
additional_instructions=additional_instructions,
instructions=instructions,
metadata=metadata,
model=model,
tools=tools,
)
# fmt: off
@ -2984,7 +2994,7 @@ class OpenAIAssistantsAPI(BaseLLM):
assistant_id: str,
additional_instructions: str | None,
instructions: str | None,
metadata: dict | None,
metadata: dict[str, str] | None,
model: str | None,
stream: bool | None,
tools: Iterable[AssistantToolParam] | None,

5
litellm/llms/openai_like/chat/handler.py
View file

@ -12,6 +12,7 @@ import httpx
import litellm
from litellm import LlmProviders
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.bedrock.chat.invoke_handler import MockResponseIterator
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.databricks.streaming_utils import ModelResponseIterator
@ -112,7 +113,7 @@ class OpenAILikeChatHandler(OpenAILikeBase):
print_verbose: Callable,
encoding,
api_key,
logging_obj,
logging_obj: LiteLLMLoggingObj,
stream,
data: dict,
optional_params=None,
@ -214,7 +215,7 @@ class OpenAILikeChatHandler(OpenAILikeBase):
print_verbose: Callable,
encoding,
api_key: str | None,
logging_obj,
logging_obj: LiteLLMLoggingObj,
optional_params: dict,
acompletion=None,
litellm_params: dict = {},

5
litellm/llms/predibase/chat/handler.py
View file

@ -9,6 +9,7 @@ from typing import Final
import httpx
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
get_async_httpx_client,
@ -59,7 +60,7 @@ class PredibaseChatCompletion:
print_verbose: Callable,
encoding,
api_key: str,
logging_obj,
logging_obj: LiteLLMLoggingObj,
optional_params: dict,
litellm_params: dict,
tenant_id: str,
@ -250,7 +251,7 @@ class PredibaseChatCompletion:
print_verbose: Callable,
encoding,
api_key,
logging_obj,
logging_obj: LiteLLMLoggingObj,
data: dict,
timeout: float | httpx.Timeout,
optional_params=None,

5
litellm/llms/replicate/chat/handler.py
View file

@ -6,6 +6,7 @@ from typing import Final
import litellm
from litellm.constants import REPLICATE_POLLING_DELAY_SECONDS
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
@ -128,7 +129,7 @@ def completion(
print_verbose: Callable,
optional_params: dict,
litellm_params: dict,
logging_obj,
logging_obj: LiteLLMLoggingObj,
api_key,
encoding,
custom_prompt_dict={},
@ -246,7 +247,7 @@ async def async_completion(
input_data,
api_key,
api_base,
logging_obj,
logging_obj: LiteLLMLoggingObj,
print_verbose,
headers: dict,
) -> ModelResponse | CustomStreamWrapper:

8
litellm/llms/sagemaker/completion/handler.py
View file

@ -8,6 +8,7 @@ import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.custom_httpx.http_handler import (
_get_httpx_client,
@ -138,7 +139,7 @@ class SagemakerLLM(BaseAWSLLM):
model_response: ModelResponse,
print_verbose: Callable,
encoding,
logging_obj,
logging_obj: LiteLLMLoggingObj,
optional_params: dict,
litellm_params: dict,
timeout: float | httpx.Timeout | None = None,
@ -431,17 +432,18 @@ class SagemakerLLM(BaseAWSLLM):
if not prepared_request.body:
raise ValueError("Prepared request body is empty")
stream_logging_obj: Final[LiteLLMLoggingObj] = logging_obj
completion_stream: Final = await self.make_async_call(
api_base=prepared_request.url,
headers=prepared_request.headers,
data=cast(str, prepared_request.body),
logging_obj=logging_obj,
logging_obj=stream_logging_obj,
)
streaming_response: Final = CustomStreamWrapper(
completion_stream=completion_stream,
model=model,
custom_llm_provider="sagemaker",
logging_obj=logging_obj,
logging_obj=stream_logging_obj,
)
# LOGGING

41
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -3,7 +3,7 @@
## Initial implementation - covers gemini + image gen calls
import json
import time
from collections.abc import Callable, Mapping
from collections.abc import Callable, Mapping, Sequence
from copy import deepcopy
from functools import partial
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Union, cast
@ -208,7 +208,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
presence_penalty: float | None = None,
seed: int | None = None,
) -> None:
locals_: Final = locals().copy()
locals_: Final[Mapping[str, object]] = locals().copy()
for key, value in locals_.items():
if key != "self" and value is not None:
setattr(self.__class__, key, value)
@ -1427,7 +1427,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
@staticmethod
def _extract_server_side_tool_invocations(
parts: list[HttpxPartType],
) -> list[dict[str, Any]] | None:
) -> list[dict[str, object]] | None:
"""Extract server-side tool invocations (toolCall/toolResponse) from parts.
These are returned by Gemini when context circulation is enabled
@ -1438,15 +1438,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
Returns:
List of server-side invocation dicts if any found, None otherwise.
"""
invocations: Final[list[dict[str, Any]]] = []
invocations: Final[list[dict[str, object]]] = []
# Index toolCalls by id so we can pair them with responses
tool_calls_by_id: Final[dict[str, dict[str, Any]]] = {}
tool_responses_by_id: Final[dict[str, dict[str, Any]]] = {}
tool_calls_by_id: Final[dict[str, dict[str, object]]] = {}
tool_responses_by_id: Final[dict[str, dict[str, object]]] = {}
for part in parts:
if "toolCall" in part:
tc = part["toolCall"]
entry: dict[str, Any] = {
entry: dict[str, object] = {
"tool_type": tc.get("toolType"),
"id": tc.get("id"),
"args": tc.get("args"),
@ -1753,7 +1753,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
response_tokens_details: CompletionTokensDetailsWrapper | None = None
usage_metadata: Final = completion_response["usageMetadata"]
def _get_token_count(detail: Mapping[str, Any]) -> int:
def _get_token_count(detail: Mapping[str, object]) -> int:
raw_token_count: Final = detail.get("tokenCount", detail.get("token_count", 0))
return raw_token_count if isinstance(raw_token_count, int) else 0
@ -2068,7 +2068,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
)
@staticmethod
def _get_stream_chunk_attr(chunk: Any, field_name: str) -> Any:
def _get_stream_chunk_attr(chunk: object, field_name: str) -> object:
if isinstance(chunk, dict):
value = chunk.get(field_name)
if value is not None:
@ -2110,10 +2110,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
def apply_assembled_streaming_response_metadata(
self,
response: ModelResponse,
chunks: list[Any],
chunks: list[object],
) -> None:
for field_name in VERTEX_AI_PROVIDER_METADATA_FIELDS:
merged: list[Any] = []
merged: list[object] = []
for chunk in chunks:
value = VertexGeminiConfig._get_stream_chunk_attr(chunk, field_name)
if not value:
@ -2214,8 +2214,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
functions: ChatCompletionToolCallFunctionChunk | None = None
thinking_blocks: list[ChatCompletionThinkingBlock] | None = None
reasoning_content: str | None = None
thought_signatures: Any | None = None
server_side_tool_invocations: list[dict[str, Any]] | None = None
thought_signatures: Sequence[str] | None = None
server_side_tool_invocations: list[dict[str, object]] | None = None
for idx, candidate in enumerate(_candidates):
if "content" not in candidate:
@ -2370,7 +2370,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: Any,
encoding: object,
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:
@ -2486,7 +2486,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
## ADD SERVICE TIER ##
if getattr(raw_response, "headers", None):
if service_tier := raw_response.headers.get("x-gemini-service-tier"):
service_tier: Final[str | None] = raw_response.headers.get("x-gemini-service-tier")
if service_tier:
if service_tier.lower() == "standard":
setattr(model_response, "service_tier", "default")
else:
@ -2660,7 +2661,7 @@ class VertexLLM(VertexBase):
print_verbose: Callable,
data: dict,
timeout: float | httpx.Timeout | None,
encoding,
encoding: object,
logging_obj,
stream,
optional_params: dict,
@ -2756,7 +2757,7 @@ class VertexLLM(VertexBase):
"vertex_ai", "vertex_ai_beta", "gemini"
], # if it's vertex_ai or gemini (google ai studio)
timeout: float | httpx.Timeout | None,
encoding,
encoding: object,
logging_obj,
stream,
optional_params: dict,
@ -2873,7 +2874,7 @@ class VertexLLM(VertexBase):
custom_llm_provider: Literal[
"vertex_ai", "vertex_ai_beta", "gemini"
], # if it's vertex_ai or gemini (google ai studio)
encoding,
encoding: object,
logging_obj,
optional_params: dict,
acompletion: bool,
@ -3122,7 +3123,7 @@ class ModelResponseIterator:
def _apply_stream_candidates(
self,
_candidates: list[Candidates],
model_response: Any,
model_response: "ModelResponseStream",
) -> tuple[list[dict], list[dict], list[dict], list[dict]]:
(
grounding_metadata,
@ -3200,7 +3201,7 @@ class ModelResponseIterator:
def _apply_stream_usage_metadata(
self,
processed_chunk: Any,
processed_chunk: GenerateContentResponseBody,
model_response: Any,
grounding_metadata: list[dict],
) -> Usage | None:

2
litellm/llms/vertex_ai/vertex_ai_non_gemini.py
View file

@ -28,7 +28,7 @@ class TextStreamer:
Fake streaming iterator for Vertex AI Model Garden calls
"""
def __init__(self, text):
def __init__(self, text: str):
self.text = text.split() # let's assume words as a streaming unit
self.index = 0

271
litellm/main.py
View file

@ -19,12 +19,12 @@ import random
import sys
import time
import traceback
from collections.abc import AsyncIterator, Coroutine, Iterable, Mapping
from collections.abc import AsyncIterator, Coroutine, Iterable, Mapping, Sequence
from concurrent import futures
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
from copy import deepcopy
from functools import partial
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Union, cast, get_args
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args
from litellm._logging import _redact_string
from litellm._uuid import uuid
@ -597,7 +597,7 @@ async def acompletion(
_, custom_llm_provider, _, _ = get_llm_provider(
model=model,
custom_llm_provider=custom_llm_provider,
api_base=completion_kwargs.get("base_url", None),
api_base=base_url,
)
fallbacks = fallbacks or litellm.model_fallbacks
@ -634,10 +634,10 @@ async def acompletion(
init_response: Final = await loop.run_in_executor(None, func_with_context)
if isinstance(init_response, dict) or isinstance(init_response, ModelResponse): ## CACHING SCENARIO
if isinstance(init_response, dict):
response = ModelResponse(**init_response)
response = _model_response_from_cached_dict(init_response)
response = init_response
elif asyncio.iscoroutine(init_response):
response = await init_response
response = await _resolve_dispatched_chat_response(init_response)
else:
response = init_response
@ -699,6 +699,20 @@ async def acompletion(
)
async def _resolve_dispatched_chat_response(
pending: Coroutine[object, object, ModelResponse | CustomStreamWrapper],
) -> ModelResponse | CustomStreamWrapper:
return await pending
def _model_response_from_cached_dict(cached_response_dict: Mapping[str, object]) -> ModelResponse:
return ModelResponse(**cached_response_dict)
def _transcription_response_from_cached_dict(cached_response_dict: Mapping[str, object]) -> TranscriptionResponse:
return TranscriptionResponse(**cached_response_dict)
async def _async_streaming(response, model, custom_llm_provider, args):
try:
print_verbose(f"received response in _async_streaming: {response}")
@ -984,12 +998,12 @@ def responses_api_bridge_check(
model: str,
custom_llm_provider: str,
web_search_options: OpenAIWebSearchOptions | None = None,
tools: list[Any] | None = None,
reasoning_effort: Any | None = None,
reasoning_summary: Any | None = None,
tools: Sequence[Mapping[str, object]] | None = None,
reasoning_effort: str | Mapping[str, object] | None = None,
reasoning_summary: object | None = None,
api_base: str | None = None,
) -> tuple[dict, str]:
model_info: dict[str, Any] = {}
model_info: dict[str, object] = {}
# Global flag: route ALL OpenAI chat completions through Responses API.
# Returns early with minimal model_info; callers only inspect the "mode" key.
@ -1111,6 +1125,22 @@ def _drop_input_examples_from_tools(
return cleaned_tools
class _ProxyAuthHeadersProvider(Protocol):
def get_auth_headers(self) -> Mapping[str, str]: ...
def _proxy_auth_headers(proxy_auth: _ProxyAuthHeadersProvider) -> Mapping[str, str]:
return proxy_auth.get_auth_headers()
def _provider_config_items(config: Mapping[str, object]) -> Iterable[tuple[str, object]]:
return config.items()
def _locals_snapshot(values: Mapping[str, object]) -> Mapping[str, object]:
return values
def _build_custom_pricing_entry(
custom_llm_provider: str,
kwargs: dict,
@ -1186,13 +1216,31 @@ def _register_custom_pricing_for_request(
)
def _dispatch_metadata(ctx: _CompletionDispatchContext) -> Mapping[str, object] | None:
return ctx.metadata
def _dispatch_client_http(ctx: _CompletionDispatchContext) -> HTTPHandler | AsyncHTTPHandler | None:
return ctx.client
def _dispatch_client_azure(
ctx: _CompletionDispatchContext,
) -> openai.AzureOpenAI | openai.AsyncAzureOpenAI | HTTPHandler | AsyncHTTPHandler | None:
return ctx.client
def _dispatch_client_openai(ctx: _CompletionDispatchContext) -> openai.OpenAI | openai.AsyncOpenAI | None:
return ctx.client
def _complete_azure(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
_azure_detection_model: Final = ctx._azure_detection_model
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
api_version = ctx.api_version
client: Final = ctx.client
client: Final = _dispatch_client_azure(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
extra_headers: Final = ctx.extra_headers
headers = ctx.headers
@ -1233,7 +1281,8 @@ def _complete_azure(ctx: _CompletionDispatchContext) -> _CompletionDispatchResul
"AZURE_AD_TOKEN"
)
azure_ad_token_provider: Final = litellm_params.get("azure_ad_token_provider", None)
azure_ad_token_provider_value: Final = litellm_params.get("azure_ad_token_provider", None)
azure_ad_token_provider: Final = azure_ad_token_provider_value if callable(azure_ad_token_provider_value) else None
headers = headers or litellm.headers
@ -1245,7 +1294,7 @@ def _complete_azure(ctx: _CompletionDispatchContext) -> _CompletionDispatchResul
if litellm.AzureOpenAIO1Config().is_o_series_model(model=_azure_detection_model):
## LOAD CONFIG - if set
config = litellm.AzureOpenAIO1Config.get_config()
for k, v in config.items():
for k, v in _provider_config_items(config):
if (
k not in optional_params
): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in
@ -1274,7 +1323,7 @@ def _complete_azure(ctx: _CompletionDispatchContext) -> _CompletionDispatchResul
else:
## LOAD CONFIG - if set
config = litellm.AzureOpenAIConfig.get_config()
for k, v in config.items():
for k, v in _provider_config_items(config):
if (
k not in optional_params
): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in
@ -1324,7 +1373,7 @@ def _complete_azure_text(ctx: _CompletionDispatchContext) -> _CompletionDispatch
api_base = ctx.api_base
api_key = ctx.api_key
api_version = ctx.api_version
client: Final = ctx.client
client: Final = _dispatch_client_azure(ctx)
extra_headers: Final = ctx.extra_headers
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -1359,7 +1408,8 @@ def _complete_azure_text(ctx: _CompletionDispatchContext) -> _CompletionDispatch
"AZURE_AD_TOKEN"
)
azure_ad_token_provider: Final = litellm_params.get("azure_ad_token_provider", None)
azure_ad_token_provider_value: Final = litellm_params.get("azure_ad_token_provider", None)
azure_ad_token_provider: Final = azure_ad_token_provider_value if callable(azure_ad_token_provider_value) else None
headers = headers or litellm.headers
@ -1368,7 +1418,7 @@ def _complete_azure_text(ctx: _CompletionDispatchContext) -> _CompletionDispatch
## LOAD CONFIG - if set
config: Final = litellm.AzureOpenAIConfig.get_config()
for k, v in config.items():
for k, v in _provider_config_items(config):
if (
k not in optional_params
): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in
@ -1416,7 +1466,7 @@ def _complete_deepseek(ctx: _CompletionDispatchContext) -> _CompletionDispatchRe
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -1467,7 +1517,7 @@ def _complete_azure_ai(ctx: _CompletionDispatchContext) -> _CompletionDispatchRe
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
extra_headers: Final = ctx.extra_headers
headers = ctx.headers
@ -1623,7 +1673,7 @@ def _complete_text_completion_openai(
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_openai(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -1655,7 +1705,7 @@ def _complete_text_completion_openai(
## LOAD CONFIG - if set
config: Final = litellm.OpenAITextCompletionConfig.get_config()
for k, v in config.items():
for k, v in _provider_config_items(config):
if (
k not in optional_params
): # completion(top_k=3) > openai_text_config(top_k=3) <- allows for dynamic variables to be passed in
@ -1705,7 +1755,7 @@ def _complete_fireworks_ai(
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -1756,7 +1806,7 @@ def _complete_heroku(ctx: _CompletionDispatchContext) -> _CompletionDispatchResu
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -1806,7 +1856,7 @@ def _complete_ragflow(ctx: _CompletionDispatchContext) -> _CompletionDispatchRes
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -1856,7 +1906,7 @@ def _complete_xai(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -1907,7 +1957,7 @@ def _complete_groq(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -1939,7 +1989,7 @@ def _complete_groq(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult
## LOAD CONFIG - if set
config: Final = litellm.GroqChatConfig.get_config()
for k, v in config.items():
for k, v in _provider_config_items(config):
if (
k not in optional_params
): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in
@ -1971,7 +2021,7 @@ def _complete_bedrock_mantle(
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -1988,7 +2038,7 @@ def _complete_bedrock_mantle(
api_key = api_key or litellm.api_key or get_secret("BEDROCK_MANTLE_API_KEY")
headers = headers or litellm.headers
config: Final = litellm.BedrockMantleChatConfig.get_config()
for k, v in config.items():
for k, v in _provider_config_items(config):
if k not in optional_params:
optional_params[k] = v
return base_llm_http_handler.completion(
@ -2015,7 +2065,7 @@ def _complete_a2a(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -2078,7 +2128,7 @@ def _complete_gigachat(ctx: _CompletionDispatchContext) -> _CompletionDispatchRe
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -2140,7 +2190,7 @@ def _complete_sap(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -2156,7 +2206,7 @@ def _complete_sap(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
headers = headers or litellm.headers
## LOAD CONFIG - if set
config: Final = litellm.GenAIHubOrchestrationConfig.get_config()
for k, v in config.items():
for k, v in _provider_config_items(config):
if (
k not in optional_params
): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in
@ -2188,7 +2238,7 @@ def _complete_aiohttp_openai(
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
extra_headers: Final = ctx.extra_headers
headers = ctx.headers
@ -2243,7 +2293,7 @@ def _complete_cometapi(ctx: _CompletionDispatchContext) -> _CompletionDispatchRe
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -2292,7 +2342,7 @@ def _complete_minimax(ctx: _CompletionDispatchContext) -> _CompletionDispatchRes
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -2338,7 +2388,7 @@ def _complete_hosted_vllm(ctx: _CompletionDispatchContext) -> _CompletionDispatc
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key: Final = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -2384,7 +2434,7 @@ def _complete_custom_openai(
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
custom_prompt_dict: Final = ctx.custom_prompt_dict
extra_headers = ctx.extra_headers
@ -2393,7 +2443,7 @@ def _complete_custom_openai(
logger_fn: Final = ctx.logger_fn
logging: Final = ctx.logging
messages: Final = ctx.messages
metadata: Final = ctx.metadata
metadata: Final = _dispatch_metadata(ctx)
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
@ -2446,7 +2496,7 @@ def _complete_custom_openai(
## LOAD CONFIG - if set
config: Final = litellm.OpenAIConfig.get_config()
for k, v in config.items():
for k, v in _provider_config_items(config):
if (
k not in optional_params
): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in
@ -2523,7 +2573,7 @@ def _complete_mistral(ctx: _CompletionDispatchContext) -> _CompletionDispatchRes
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -2674,7 +2724,7 @@ def _complete_anthropic(ctx: _CompletionDispatchContext) -> _CompletionDispatchR
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
custom_prompt_dict = ctx.custom_prompt_dict
headers: Final = ctx.headers
@ -2973,7 +3023,7 @@ def _complete_huggingface(ctx: _CompletionDispatchContext) -> _CompletionDispatc
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -3016,7 +3066,7 @@ def _complete_oci(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -3051,7 +3101,7 @@ def _complete_compactifai(ctx: _CompletionDispatchContext) -> _CompletionDispatc
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -3127,7 +3177,7 @@ def _complete_databricks(ctx: _CompletionDispatchContext) -> _CompletionDispatch
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
@ -3199,7 +3249,7 @@ def _complete_datarobot(ctx: _CompletionDispatchContext) -> _CompletionDispatchR
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -3236,7 +3286,7 @@ def _complete_openrouter(ctx: _CompletionDispatchContext) -> _CompletionDispatch
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
@ -3274,7 +3324,7 @@ def _complete_openrouter(ctx: _CompletionDispatchContext) -> _CompletionDispatch
## Load Config
config: Final = litellm.OpenrouterConfig.get_config()
for k, v in config.items():
for k, v in _provider_config_items(config):
if k == "extra_body":
# we use openai 'extra_body' to pass openrouter specific params - transforms, route, models
if "extra_body" in optional_params:
@ -3315,7 +3365,7 @@ def _complete_vercel_ai_gateway(
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
@ -3352,7 +3402,7 @@ def _complete_vercel_ai_gateway(
## Load Config
config: Final = litellm.VercelAIGatewayConfig.get_config()
for k, v in config.items():
for k, v in _provider_config_items(config):
if k == "extra_body":
# we use openai 'extra_body' to pass vercel specific params - providerOptions
if "extra_body" in optional_params:
@ -3393,7 +3443,7 @@ def _complete_vertex_ai_beta(
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key: Final = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -3458,7 +3508,7 @@ def _complete_vertex_ai_beta(
def _complete_vertex_ai(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
custom_prompt_dict: Final = ctx.custom_prompt_dict
headers: Final = ctx.headers
@ -3755,7 +3805,7 @@ def _complete_text_completion_inception(
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_openai(ctx)
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logger_fn: Final = ctx.logger_fn
@ -3819,7 +3869,7 @@ def _complete_sagemaker_chat(
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -3882,7 +3932,7 @@ def _complete_bedrock(ctx: _CompletionDispatchContext) -> _CompletionDispatchRes
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_prompt_dict = ctx.custom_prompt_dict
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -4006,7 +4056,7 @@ def _complete_watsonx(ctx: _CompletionDispatchContext) -> _CompletionDispatchRes
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_prompt_dict: Final = ctx.custom_prompt_dict
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -4045,7 +4095,7 @@ def _complete_watsonx_text(
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
@ -4157,7 +4207,7 @@ def _complete_ollama(ctx: _CompletionDispatchContext) -> _CompletionDispatchResu
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key: Final = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
@ -4197,7 +4247,7 @@ def _complete_ollama_chat(ctx: _CompletionDispatchContext) -> _CompletionDispatc
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
logging: Final = ctx.logging
@ -4312,7 +4362,7 @@ def _complete_cloudflare(ctx: _CompletionDispatchContext) -> _CompletionDispatch
def _complete_petals(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
api_base = ctx.api_base
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
litellm_params: Final = ctx.litellm_params
logger_fn: Final = ctx.logger_fn
logging: Final = ctx.logging
@ -4354,7 +4404,7 @@ def _complete_snowflake(ctx: _CompletionDispatchContext) -> _CompletionDispatchR
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key: Final = ctx.api_key
client = ctx.client
client = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -4442,7 +4492,7 @@ def _complete_gdc(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -4481,7 +4531,7 @@ def _complete_bytez(ctx: _CompletionDispatchContext) -> _CompletionDispatchResul
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -4521,7 +4571,7 @@ def _complete_lemonade(ctx: _CompletionDispatchContext) -> _CompletionDispatchRe
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
api_key = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -4561,7 +4611,7 @@ def _complete_ovhcloud(ctx: _CompletionDispatchContext) -> _CompletionDispatchRe
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers: Final = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -4604,6 +4654,10 @@ def _complete_ovhcloud(ctx: _CompletionDispatchContext) -> _CompletionDispatchRe
return response
def _custom_api_first_output(resp: httpx.Response | None) -> str:
return resp.json()["data"][0]["output"][0]
def _complete_custom(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
api_base: Final = ctx.api_base
headers: Final = ctx.headers
@ -4652,7 +4706,6 @@ def _complete_custom(ctx: _CompletionDispatchContext) -> _CompletionDispatchResu
**kwargs.get("extra_body", {}),
},
)
response_json: Final = resp.json()
"""
assume all responses from custom api_bases of this format:
{
@ -4666,7 +4719,7 @@ def _complete_custom(ctx: _CompletionDispatchContext) -> _CompletionDispatchResu
]
}
"""
string_response: Final = response_json["data"][0]["output"][0]
string_response: Final = _custom_api_first_output(resp)
## RESPONSE OBJECT
model_response.choices[0].message.content = string_response
model_response.created = int(time.time())
@ -4741,7 +4794,7 @@ def _complete_langgraph(ctx: _CompletionDispatchContext) -> _CompletionDispatchR
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -4790,7 +4843,7 @@ def _complete_langflow(ctx: _CompletionDispatchContext) -> _CompletionDispatchRe
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key
client: Final = ctx.client
client: Final = _dispatch_client_http(ctx)
custom_llm_provider: Final = ctx.custom_llm_provider
headers = ctx.headers
litellm_params: Final = ctx.litellm_params
@ -4948,7 +5001,7 @@ def completion(
thinking = validate_and_fix_thinking_param(thinking=thinking)
######### unpacking kwargs #####################
args: Final = locals()
args: Final = _locals_snapshot(locals())
# Set by the responses->completion fallback so completion() does not bridge
# back to the Responses API: that round-trip mutually recurses forever for a
@ -5039,7 +5092,7 @@ def completion(
# Inject proxy auth headers if configured
if litellm.proxy_auth is not None:
try:
proxy_headers: Final = litellm.proxy_auth.get_auth_headers()
proxy_headers: Final = _proxy_auth_headers(litellm.proxy_auth)
headers.update(proxy_headers)
except Exception as e:
verbose_logger.warning("Failed to get proxy auth headers: %s", e)
@ -5092,7 +5145,7 @@ def completion(
)
######## end of unpacking kwargs ###########
non_default_params: Final = get_non_default_completion_params(kwargs=kwargs)
litellm_params = {} # used to prevent unbound var errors
litellm_params: dict[str, object] = {} # used to prevent unbound var errors
## PROMPT MANAGEMENT HOOKS ##
from litellm.integrations.anthropic_cache_control_hook import (
@ -5915,7 +5968,7 @@ def embedding(
*,
aembedding: Literal[True],
**kwargs,
) -> Coroutine[Any, Any, EmbeddingResponse]:
) -> Coroutine[object, object, EmbeddingResponse]:
...
@ -5966,7 +6019,7 @@ def embedding(
litellm_call_id=None,
logger_fn=None,
**kwargs,
) -> EmbeddingResponse | Coroutine[Any, Any, EmbeddingResponse]:
) -> EmbeddingResponse | Coroutine[object, object, EmbeddingResponse]:
"""
Embedding function that calls an API to generate embeddings for the given input.
@ -6009,7 +6062,7 @@ def embedding(
# Inject proxy auth headers if configured
if litellm.proxy_auth is not None:
try:
proxy_headers: Final = litellm.proxy_auth.get_auth_headers()
proxy_headers: Final = _proxy_auth_headers(litellm.proxy_auth)
headers.update(proxy_headers)
except Exception as e:
verbose_logger.warning("Failed to get proxy auth headers: %s", e)
@ -6086,7 +6139,7 @@ def embedding(
if mock_response is not None:
return mock_embedding(model=model, mock_response=mock_response)
try:
response: EmbeddingResponse | Coroutine[Any, Any, EmbeddingResponse] | None = None
response: EmbeddingResponse | Coroutine[object, object, EmbeddingResponse] | None = None
if azure is True or custom_llm_provider == "azure":
# azure configs
@ -6389,7 +6442,7 @@ def embedding(
response = huggingface_embed.embedding(
model=model,
input=input,
encoding=_get_encoding(),
encoding=sys.modules[__name__].encoding,
api_key=api_key,
api_base=api_base,
logging_obj=logging,
@ -6992,6 +7045,20 @@ def embedding(
###### Text Completion ################
async def _resolve_dispatched_text_completion_response(
pending: Coroutine[
object,
object,
TextCompletionResponse | ModelResponse | CustomStreamWrapper | TextCompletionStreamWrapper,
],
) -> TextCompletionResponse | ModelResponse | CustomStreamWrapper | TextCompletionStreamWrapper:
return await pending
async def _resolve_pending_chat_response(pending: Coroutine[object, object, ModelResponse]) -> ModelResponse:
return await pending
@client
async def atext_completion(*args, **kwargs) -> TextCompletionResponse | TextCompletionStreamWrapper:
"""
@ -7017,7 +7084,7 @@ async def atext_completion(*args, **kwargs) -> TextCompletionResponse | TextComp
else:
response = init_response
elif asyncio.iscoroutine(init_response):
response = await init_response
response = await _resolve_dispatched_text_completion_response(init_response)
else:
response = init_response
@ -7042,7 +7109,7 @@ async def atext_completion(*args, **kwargs) -> TextCompletionResponse | TextComp
if isinstance(response, TextCompletionResponse):
return response
elif asyncio.iscoroutine(response):
response = await response
response = await _resolve_pending_chat_response(response)
text_completion_response = TextCompletionResponse()
text_completion_response = litellm.utils.LiteLLMResponseObjectHandler.convert_chat_to_text_completion(
@ -7332,11 +7399,11 @@ async def aadapter_completion(*, adapter_id: str, **kwargs) -> BaseModel | Adapt
async def aadapter_generate_content(
**kwargs,
) -> dict[str, Any] | AsyncIterator[bytes]:
) -> dict[str, object] | AsyncIterator[bytes]:
from litellm.google_genai.adapters.handler import GenerateContentToCompletionHandler
coro: Final = cast(
Coroutine[Any, Any, dict[str, Any] | AsyncIterator[bytes]],
Coroutine[object, object, dict[str, object] | AsyncIterator[bytes]],
GenerateContentToCompletionHandler.generate_content_handler(**kwargs, _is_async=True),
)
return await coro
@ -7488,7 +7555,7 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse:
# Await normally
init_response: Final = await loop.run_in_executor(None, func_with_context)
if isinstance(init_response, dict):
response = TranscriptionResponse(**init_response)
response = _transcription_response_from_cached_dict(init_response)
elif isinstance(init_response, TranscriptionResponse): ## CACHING SCENARIO
response = init_response
elif asyncio.iscoroutine(init_response):
@ -7543,7 +7610,7 @@ def transcription(
max_retries: int | None = None,
custom_llm_provider=None,
**kwargs,
) -> TranscriptionResponse | Coroutine[Any, Any, TranscriptionResponse]:
) -> TranscriptionResponse | Coroutine[object, object, TranscriptionResponse]:
"""
Calls openai + azure whisper endpoints.
@ -7610,7 +7677,7 @@ def transcription(
custom_llm_provider=custom_llm_provider,
)
response: TranscriptionResponse | Coroutine[Any, Any, TranscriptionResponse] | None = None
response: TranscriptionResponse | Coroutine[object, object, TranscriptionResponse] | None = None
provider_config: Final = ProviderConfigManager.get_provider_audio_transcription_config(
model=model,
@ -7844,7 +7911,7 @@ def speech(
custom_llm_provider: str | None = None,
aspeech: bool | None = None,
**kwargs,
) -> HttpxBinaryResponseContent | Coroutine[Any, Any, HttpxBinaryResponseContent]:
) -> HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent]:
user: Final = kwargs.get("user", None)
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
proxy_server_request: Final = kwargs.get("proxy_server_request", None)
@ -7903,7 +7970,7 @@ def speech(
},
custom_llm_provider=custom_llm_provider,
)
response: HttpxBinaryResponseContent | Coroutine[Any, Any, HttpxBinaryResponseContent] | None = None
response: HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent] | None = None
if custom_llm_provider == "openai" or custom_llm_provider in litellm.openai_compatible_providers:
if voice is None or not (isinstance(voice, str)):
raise litellm.BadRequestError(
@ -8665,7 +8732,7 @@ def stream_chunk_builder(
]
if len(provider_specific_chunks) > 0:
combined_provider_fields: Final[dict[str, Any]] = {}
combined_provider_fields: Final[dict[str, object]] = {}
for chunk in provider_specific_chunks:
fields = chunk["choices"][0]["delta"]["provider_specific_fields"]
if isinstance(fields, dict):
@ -8730,7 +8797,7 @@ def stream_chunk_builder(
async def acount_tokens(
model: str,
messages: list[dict[str, Any]] | None = None,
messages: list[dict[str, object]] | None = None,
tools: list[dict[str, Any]] | None = None,
system: str | None = None,
api_key: str | None = None,
@ -8776,7 +8843,7 @@ async def acount_tokens(
api_base = dynamic_api_base
# Build deployment dict for the token counter
deployment: Final[dict[str, Any]] = {
deployment: Final[dict[str, object]] = {
"litellm_params": {
"model": model,
"api_key": api_key,
@ -8827,29 +8894,37 @@ async def acount_tokens(
# Cache for encoding to avoid repeated __getattr__ calls
_encoding_cache: Any | None = None
_encoding_cache: tiktoken.Encoding | None = None
def _get_encoding():
def _load_module_encoding() -> tiktoken.Encoding:
import sys
return sys.modules[__name__].encoding
def _get_encoding() -> tiktoken.Encoding:
"""Get encoding, loading it lazily if needed."""
global _encoding_cache
if _encoding_cache is None:
import sys
# Access via module to trigger __getattr__ if not cached
_encoding_cache = sys.modules[__name__].encoding
_encoding_cache = _load_module_encoding()
return _encoding_cache
def __getattr__(name: str) -> Any:
def _load_default_encoding() -> tiktoken.Encoding:
from litellm._lazy_imports import _get_default_encoding
return _get_default_encoding()
def __getattr__(name: str) -> tiktoken.Encoding:
"""Lazy import handler for main module"""
if name == "encoding":
# Use _get_default_encoding which properly sets TIKTOKEN_CACHE_DIR
# before loading tiktoken, ensuring the local cache is used
# instead of downloading from the internet
from litellm._lazy_imports import _get_default_encoding
_encoding: Final = _get_default_encoding()
_encoding: Final = _load_default_encoding()
# Cache it in the module's __dict__ for subsequent accesses
import sys

3
litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py
View file

@ -145,9 +145,8 @@ class MCPOAuth2TokenCache(InMemoryCache):
server.server_id,
)
post_kwargs: Final = {"data": data, **({"headers": token_request.headers} if token_request.headers else {})}
try:
response: Final = await client.post(server.token_url, **post_kwargs)
response: Final = await client.post(server.token_url, data=data, headers=token_request.headers or None)
response.raise_for_status()
except httpx.HTTPStatusError as exc:
raise ValueError(

234
litellm/proxy/auth/auth_checks.py
View file

@ -13,8 +13,8 @@ import asyncio
import math
import re
import time
from collections.abc import Sequence
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast
from collections.abc import Iterator, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, cast
from fastapi import HTTPException, Request, status
from pydantic import BaseModel
@ -87,6 +87,7 @@ from litellm.proxy.utils import PrismaClient, ProxyLogging, log_db_metrics
from litellm.repositories.budget_repository import BudgetRepository
from litellm.repositories.object_permission_repository import ObjectPermissionRepository
from litellm.repositories.organization_repository import OrganizationRepository
from litellm.repositories.prisma_protocols import RowT_co
from litellm.repositories.project_repository import ProjectRepository
from litellm.repositories.table_repositories import (
AccessGroupRepository,
@ -110,11 +111,144 @@ from .auth_utils import get_model_from_request, get_request_route_template
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
Span = _Span | Any
Span = _Span
else:
Span = Any
class _PrismaDictableRow(Protocol):
def dict(self) -> Mapping[str, object]: ...
class _PrismaJWTKeyMappingRow(Protocol):
token: str
class _PrismaModelDumpRow(Protocol):
def model_dump(self) -> Mapping[str, object]: ...
class _PrismaTeamRow(Protocol):
def dict(self) -> Mapping[str, object]: ...
def model_dump(self) -> Mapping[str, object]: ...
class _PrismaVectorStoreRow(Protocol):
def dict(self) -> Mapping[str, object]: ...
def model_dump(self) -> Mapping[str, object]: ...
def __iter__(self) -> Iterator[tuple[str, object]]: ...
class _PrismaUserRow(Protocol):
user_id: str
organization_memberships: Sequence[LiteLLM_OrganizationMembershipTable | None] | None
def __iter__(self) -> Iterator[tuple[str, object]]: ...
class _PrismaAuthTable(Protocol[RowT_co]):
async def find_unique(
self, *, where: Mapping[str, object], include: Mapping[str, object] | None = None
) -> RowT_co | None: ...
async def find_first(
self, *, where: Mapping[str, object], include: Mapping[str, object] | None = None
) -> RowT_co | None: ...
async def find_many(
self,
*,
where: Mapping[str, object],
include: Mapping[str, object] | None = None,
take: int | None = None,
) -> Sequence[RowT_co]: ...
async def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> RowT_co | None: ...
async def create(self, *, data: Mapping[str, object], include: Mapping[str, object] | None = None) -> RowT_co: ...
class _PrismaTableHolder(Protocol[RowT_co]):
@property
def table(self) -> _PrismaAuthTable[RowT_co]: ...
def _dictable_table(repo: _PrismaTableHolder[_PrismaDictableRow]) -> _PrismaAuthTable[_PrismaDictableRow]:
return repo.table
def _jwt_key_mapping_table(
repo: _PrismaTableHolder[_PrismaJWTKeyMappingRow],
) -> _PrismaAuthTable[_PrismaJWTKeyMappingRow]:
return repo.table
def _model_dump_table(repo: _PrismaTableHolder[_PrismaModelDumpRow]) -> _PrismaAuthTable[_PrismaModelDumpRow]:
return repo.table
def _team_table(repo: _PrismaTableHolder[_PrismaTeamRow]) -> _PrismaAuthTable[_PrismaTeamRow]:
return repo.table
def _vector_store_table(repo: _PrismaTableHolder[_PrismaVectorStoreRow]) -> _PrismaAuthTable[_PrismaVectorStoreRow]:
return repo.table
def _user_table(repo: _PrismaTableHolder[_PrismaUserRow]) -> _PrismaAuthTable[_PrismaUserRow]:
return repo.table
def _object_permission_table(
repo: _PrismaTableHolder[LiteLLM_ObjectPermissionTable],
) -> _PrismaAuthTable[LiteLLM_ObjectPermissionTable]:
return repo.table
class _PrismaTagRow(Protocol):
tag_name: str
def dict(self) -> Mapping[str, object]: ...
def _tag_table(repo: _PrismaTableHolder[_PrismaTagRow]) -> _PrismaAuthTable[_PrismaTagRow]:
return repo.table
class _RawCacheRead(Protocol):
async def async_get_cache(self, *, key: str) -> object: ...
def _raw_cache(cache: _RawCacheRead) -> _RawCacheRead:
return cache
class _BudgetCacheRead(Protocol):
async def async_get_cache(self, *, key: str) -> "LiteLLM_BudgetTable | Mapping[str, object] | None": ...
def _budget_cache(cache: _BudgetCacheRead) -> _BudgetCacheRead:
return cache
def _typed_request_body(request_body: dict) -> Mapping[str, object]:
return request_body
class _JsonLoadsObj(Protocol):
def __call__(self, data: str) -> object: ...
def _typed_json_loads(fn: _JsonLoadsObj) -> _JsonLoadsObj:
return fn
_safe_json_loads_obj: Final = _typed_json_loads(safe_json_loads)
last_db_access_time: Final = LimitedSizeOrderedDict(max_size=100)
db_cache_expiry: Final = DEFAULT_IN_MEMORY_TTL # refresh every 5s
@ -384,7 +518,7 @@ _GUARDRAIL_MODIFICATION_KEYS: Final[tuple] = (
)
def _guardrail_modification_check(request_body: dict, team_object: LiteLLM_TeamTable | None) -> None:
def _guardrail_modification_check(request_body: Mapping[str, object], team_object: LiteLLM_TeamTable | None) -> None:
"""
Reject user-supplied metadata flags that would modify guardrail behavior
unless the team has explicit permission. Checked keys include the plural
@ -399,7 +533,7 @@ def _guardrail_modification_check(request_body: dict, team_object: LiteLLM_TeamT
"""
from litellm.proxy.guardrails.guardrail_helpers import can_modify_guardrails
def _coerce_to_dict(container: Any) -> dict | None:
def _coerce_to_dict(container: object) -> dict | None:
"""Accept dict or JSON-string (from multipart/form-data or extra_body).
Without this, an attacker can smuggle guardrail keys past the check by
@ -411,11 +545,11 @@ def _guardrail_modification_check(request_body: dict, team_object: LiteLLM_TeamT
if isinstance(container, dict):
return container
if isinstance(container, str):
parsed: Final = safe_json_loads(container)
parsed: Final = _safe_json_loads_obj(container)
return parsed if isinstance(parsed, dict) else None
return None
def _user_requested_modification(container: Any) -> bool:
def _user_requested_modification(container: object) -> bool:
coerced: Final = _coerce_to_dict(container)
if coerced is None:
return False
@ -731,7 +865,7 @@ async def common_checks(
_enforce_user_param_check(general_settings, request, request_body, route)
_global_proxy_budget_check(global_proxy_spend, skip_all_budget_checks, route)
_guardrail_modification_check(request_body, team_object)
_guardrail_modification_check(_typed_request_body(request_body), team_object)
# 10 [OPTIONAL] Organization RBAC checks
organization_role_based_access_check(user_object=user_object, route=route, request_body=request_body)
@ -955,7 +1089,7 @@ async def get_default_end_user_budget(
# Fetch from database
try:
budget_record: Final = await BudgetRepository(prisma_client).table.find_unique(
budget_record: Final = await _dictable_table(BudgetRepository(prisma_client)).find_unique(
where={"budget_id": litellm.max_end_user_budget_id}
)
@ -1007,14 +1141,16 @@ async def get_team_member_default_budget(
cache_key: Final = f"team_member_default_budget:{budget_id}"
cached_budget: Final = await user_api_key_cache.async_get_cache(key=cache_key)
cached_budget: Final = await _budget_cache(user_api_key_cache).async_get_cache(key=cache_key)
if isinstance(cached_budget, LiteLLM_BudgetTable):
return cached_budget
if isinstance(cached_budget, dict):
return LiteLLM_BudgetTable.model_validate(cached_budget)
try:
budget_record: Final = await BudgetRepository(prisma_client).table.find_unique(where={"budget_id": budget_id})
budget_record: Final = await _dictable_table(BudgetRepository(prisma_client)).find_unique(
where={"budget_id": budget_id}
)
if budget_record is None:
verbose_proxy_logger.warning("Team-default member budget not found in database: %s", budget_id)
@ -1171,7 +1307,7 @@ async def get_end_user_object(
# Fetch from database
try:
response: Final = await EndUserRepository(prisma_client).table.find_unique(
response: Final = await _dictable_table(EndUserRepository(prisma_client)).find_unique(
where={"user_id": end_user_id},
include={"litellm_budget_table": True, "object_permission": True},
)
@ -1243,7 +1379,7 @@ async def resolve_and_validate_end_user_id(
return raw_end_user_id
cache_key: Final = f"end_user_validation:{raw_end_user_id}"
cached: Final = await user_api_key_cache.async_get_cache(key=cache_key)
cached: Final = await _raw_cache(user_api_key_cache).async_get_cache(key=cache_key)
if cached == "valid":
return raw_end_user_id
if cached == "invalid":
@ -1345,8 +1481,8 @@ async def get_tag_objects_batch(
if not tag_names:
return {}
tag_objects: Final = {}
uncached_tags: Final = []
tag_objects: Final = dict[str, LiteLLM_TagTable]()
uncached_tags: Final = list[str]()
# Try to get all tags from cache first
for tag_name in tag_names:
@ -1363,7 +1499,7 @@ async def get_tag_objects_batch(
# Batch fetch uncached tags from DB in one query
if uncached_tags:
try:
db_tags: Final = await TagRepository(prisma_client).table.find_many(
db_tags: Final = await _tag_table(TagRepository(prisma_client)).find_many(
where={"tag_name": {"in": uncached_tags}},
include={"litellm_budget_table": True},
)
@ -1457,7 +1593,7 @@ async def get_team_membership(
# else, check db
try:
response: Final = await TeamMembershipRepository(prisma_client).table.find_unique(
response: Final = await _dictable_table(TeamMembershipRepository(prisma_client)).find_unique(
where={"user_id_team_id": {"user_id": user_id, "team_id": team_id}},
include={"litellm_budget_table": True},
)
@ -1524,7 +1660,7 @@ def _should_check_db(key: str, last_db_access_time: LimitedSizeOrderedDict, db_c
return False
def _update_last_db_access_time(key: str, value: Any | None, last_db_access_time: LimitedSizeOrderedDict):
def _update_last_db_access_time(key: str, value: object | None, last_db_access_time: LimitedSizeOrderedDict):
last_db_access_time[key] = (value, time.time())
@ -1545,7 +1681,7 @@ def _get_role_based_permissions(
for role_based_permission in role_based_permissions:
if role_based_permission.role == rbac_role:
return getattr(role_based_permission, key)
return role_based_permission.models if key == "models" else role_based_permission.routes
return None
@ -1586,7 +1722,7 @@ async def _get_fuzzy_user_object(
prisma_client: PrismaClient,
sso_user_id: str | None = None,
user_email: str | None = None,
) -> LiteLLM_UserTable | None:
) -> "_PrismaUserRow | None":
"""
Checks if sso user is in db.
@ -1600,7 +1736,7 @@ async def _get_fuzzy_user_object(
response = None
if sso_user_id is not None:
response = await UserRepository(prisma_client).table.find_unique(
response = await _user_table(UserRepository(prisma_client)).find_unique(
where={"sso_user_id": sso_user_id},
include={"organization_memberships": True},
)
@ -1608,14 +1744,14 @@ async def _get_fuzzy_user_object(
if response is None and user_email is not None:
# Use case-insensitive query to handle emails with different casing
# This matches the pattern used in _check_duplicate_user_email
response = await UserRepository(prisma_client).table.find_first(
response = await _user_table(UserRepository(prisma_client)).find_first(
where={"user_email": {"equals": user_email, "mode": "insensitive"}},
include={"organization_memberships": True},
)
if response is not None and sso_user_id is not None: # update sso_user_id
asyncio.create_task( # background task to update user with sso id
UserRepository(prisma_client).table.update(
_user_table(UserRepository(prisma_client)).update(
where={"user_id": response.user_id},
data={"sso_user_id": sso_user_id},
)
@ -1698,7 +1834,7 @@ async def get_user_object(
)
if should_check_db:
response = await UserRepository(prisma_client).table.find_unique(
response = await _user_table(UserRepository(prisma_client)).find_unique(
where={"user_id": user_id}, include={"organization_memberships": True}
)
@ -1736,7 +1872,7 @@ async def get_user_object(
budget_duration=new_user_params["budget_duration"]
)
response = await UserRepository(prisma_client).table.create(
response = await _user_table(UserRepository(prisma_client)).create(
data=new_user_params,
include={"organization_memberships": True},
)
@ -1802,7 +1938,7 @@ async def get_user_object(
async def _cache_management_object(
key: str,
value: BaseModel | dict[str, Any],
value: BaseModel | Mapping[str, object],
user_api_key_cache: UserApiKeyCache,
proxy_logging_obj: ProxyLogging | None,
*,
@ -1916,8 +2052,10 @@ async def _delete_cache_key_object(
@log_db_metrics
async def _get_team_db_check(team_id: str, prisma_client: PrismaClient, team_id_upsert: bool | None = None):
response = await TeamRepository(prisma_client).table.find_unique(where={"team_id": team_id})
async def _get_team_db_check(
team_id: str, prisma_client: PrismaClient, team_id_upsert: bool | None = None
) -> "_PrismaTeamRow | None":
response = await _team_table(TeamRepository(prisma_client)).find_unique(where={"team_id": team_id})
if response is None and team_id_upsert:
from litellm.proxy.management_endpoints.team_endpoints import new_team
@ -1936,8 +2074,8 @@ async def _get_team_db_check(team_id: str, prisma_client: PrismaClient, team_id_
return response
async def _get_team_object_from_db(team_id: str, prisma_client: PrismaClient):
return await TeamRepository(prisma_client).table.find_unique(where={"team_id": team_id})
async def _get_team_object_from_db(team_id: str, prisma_client: PrismaClient) -> "_PrismaTeamRow | None":
return await _team_table(TeamRepository(prisma_client)).find_unique(where={"team_id": team_id})
async def _get_team_object_from_user_api_key_cache(
@ -2148,7 +2286,7 @@ async def get_access_object(
# Not in cache - fetch from DB
try:
response: Final = await AccessGroupRepository(prisma_client).table.find_unique(
response: Final = await _dictable_table(AccessGroupRepository(prisma_client)).find_unique(
where={"access_group_id": access_group_id}
)
@ -2224,7 +2362,7 @@ async def get_team_object_by_alias(
# Query database by team_alias
try:
teams: Final = await TeamRepository(prisma_client).table.find_many(where={"team_alias": team_alias})
teams: Final = await _team_table(TeamRepository(prisma_client)).find_many(where={"team_alias": team_alias})
if not teams:
raise HTTPException(
@ -2329,7 +2467,9 @@ async def get_org_object_by_alias(
# Query database by organization_alias
try:
orgs = await OrganizationRepository(prisma_client).table.find_many(where={"organization_alias": org_alias})
orgs = await _model_dump_table(OrganizationRepository(prisma_client)).find_many(
where={"organization_alias": org_alias}
)
if not orgs:
raise HTTPException(
@ -2546,7 +2686,7 @@ async def get_jwt_key_mapping_object(
Returns the hashed token (str) if a matching active mapping is found, else None.
"""
mapping: Final = await JWTKeyMappingRepository(prisma_client).table.find_first(
mapping: Final = await _jwt_key_mapping_table(JWTKeyMappingRepository(prisma_client)).find_first(
where={
"jwt_claim_name": jwt_claim_name,
"jwt_claim_value": jwt_claim_value,
@ -2674,7 +2814,7 @@ async def get_object_permission(
# else, check db
try:
response: Final = await ObjectPermissionRepository(prisma_client).table.find_unique(
response: Final = await _dictable_table(ObjectPermissionRepository(prisma_client)).find_unique(
where={"object_permission_id": object_permission_id}
)
@ -2730,7 +2870,7 @@ async def get_managed_vector_store_rows_by_uuids(
if not cache_misses:
return result
rows: Final = await ManagedVectorStoresRepository(prisma_client).table.find_many(
rows: Final = await _vector_store_table(ManagedVectorStoresRepository(prisma_client)).find_many(
where={"vector_store_id": {"in": cache_misses}},
take=len(cache_misses),
)
@ -2804,11 +2944,11 @@ async def get_org_object(
return deserialized_org
# else, check db
try:
query_kwargs: Final[dict[str, Any]] = {"where": {"organization_id": org_id}}
query_kwargs: Final[dict[str, Mapping[str, object]]] = {"where": {"organization_id": org_id}}
if include_budget_table:
query_kwargs["include"] = {"litellm_budget_table": True}
response: Final = await OrganizationRepository(prisma_client).table.find_unique(**query_kwargs)
response: Final = await _model_dump_table(OrganizationRepository(prisma_client)).find_unique(**query_kwargs)
except Exception:
# An operational failure (DB down, timeout, cache fault) is NOT the same fact as a confirmed
# missing row, and relabelling it as "doesn't exist" made every caller unable to tell them
@ -3763,7 +3903,7 @@ async def _virtual_key_soft_budget_check(
)
def _parse_email_list(raw: Any) -> list[str]:
def _parse_email_list(raw: str | Sequence[object] | None) -> list[str]:
"""Parse emails from a list or comma-separated string."""
if isinstance(raw, list):
return [e.strip() for e in raw if isinstance(e, str) and e.strip()]
@ -3773,7 +3913,7 @@ def _parse_email_list(raw: Any) -> list[str]:
def _normalize_alert_emails(
cfg: dict[str, Any] | None,
cfg: Mapping[str, str | Sequence[object] | None] | None,
) -> dict[str, list[str]]:
"""Coerce user-supplied threshold→recipients mapping to Dict[str, List[str]].
@ -3786,8 +3926,8 @@ def _normalize_alert_emails(
def _merge_budget_alert_email_configs(
global_cfg: dict[str, Any] | None,
per_key_cfg: dict[str, Any] | None,
global_cfg: Mapping[str, str | Sequence[object] | None] | None,
per_key_cfg: Mapping[str, str | Sequence[object] | None] | None,
) -> dict[str, list[str]] | None:
"""
Per-threshold additive merge: each threshold's recipient list is the union
@ -4294,7 +4434,7 @@ async def get_project_object(
return deserialized_project
# Fetch from DB
project_row: Final = await ProjectRepository(prisma_client).table.find_unique(
project_row: Final = await _model_dump_table(ProjectRepository(prisma_client)).find_unique(
where={"project_id": project_id},
include={"litellm_budget_table": True},
)
@ -4621,7 +4761,9 @@ async def vector_store_access_check(
#########################################################
# Check if the key can access the vector store
if valid_token is not None and valid_token.object_permission_id is not None:
key_object_permission: Final = await ObjectPermissionRepository(prisma_client).table.find_unique(
key_object_permission: Final = await _object_permission_table(
ObjectPermissionRepository(prisma_client)
).find_unique(
where={"object_permission_id": valid_token.object_permission_id},
)
if key_object_permission is not None:
@ -4633,7 +4775,9 @@ async def vector_store_access_check(
# Check if the team can access the vector store
if team_object is not None and team_object.object_permission_id is not None:
team_object_permission: Final = await ObjectPermissionRepository(prisma_client).table.find_unique(
team_object_permission: Final = await _object_permission_table(
ObjectPermissionRepository(prisma_client)
).find_unique(
where={"object_permission_id": team_object.object_permission_id},
)
if team_object_permission is not None:

113
litellm/proxy/common_request_processing.py
View file

@ -8,7 +8,7 @@ from collections.abc import AsyncGenerator, Callable, Mapping
from datetime import datetime
from functools import lru_cache
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, overload
import anyio
import httpx
@ -312,8 +312,49 @@ def _stream_usage_tracking_updates(
}
def _getattr_object(value: object, name: str, default: object = None) -> object:
return getattr(value, name, default)
class _UpstreamHttpResponse(Protocol):
@property
def status_code(self) -> int: ...
@property
def headers(self) -> httpx.Headers: ...
async def aread(self) -> bytes: ...
def _as_upstream_response(response: _UpstreamHttpResponse) -> _UpstreamHttpResponse:
return response
class _ReadsHeaderValues(Protocol):
def get(self, key: str, default: str = "") -> str: ...
def _as_header_reader(headers: _ReadsHeaderValues) -> _ReadsHeaderValues:
return headers
class _DispatchesSuccessHandlers(Protocol):
async def dispatch_success_handlers(
self,
result: object = None,
start_time: object = None,
end_time: object = None,
cache_hit: object = None,
prefer_async_handlers: bool = False,
) -> None: ...
def _as_success_dispatcher(logging_obj: _DispatchesSuccessHandlers) -> _DispatchesSuccessHandlers:
return logging_obj
def _serialize_http_exception_detail(
detail: Any,
detail: object,
) -> tuple[str, dict | None]:
"""
Convert an HTTPException.detail value into (message, structured_fields)
@ -343,7 +384,7 @@ def _serialize_http_exception_detail(
return str(detail), None
def _collect_response_file_search_vector_store_ids(data: dict[str, Any]) -> set[str]:
def _collect_response_file_search_vector_store_ids(data: Mapping[str, object]) -> set[str]:
vector_store_ids: Final[set[str]] = set()
tools: Final = data.get("tools")
if not isinstance(tools, list):
@ -370,7 +411,7 @@ def _collect_response_file_search_vector_store_ids(data: dict[str, Any]) -> set[
async def _authorize_response_file_search_vector_stores(
data: dict[str, Any],
data: Mapping[str, object],
user_api_key_dict: UserAPIKeyAuth,
) -> None:
vector_store_ids: Final = _collect_response_file_search_vector_store_ids(data)
@ -701,7 +742,7 @@ async def create_response(
# Preserve status code from HTTPException (e.g., guardrail blocks)
error_status: Final = getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR)
raw_detail: Final = getattr(e, "detail", "Error processing stream start")
raw_detail: Final = _getattr_object(e, "detail", "Error processing stream start")
message, structured_fields = _serialize_http_exception_detail(raw_detail)
existing_fields: Final = getattr(e, "provider_specific_fields", None) or {}
@ -712,7 +753,7 @@ async def create_response(
# Match ProxyException.to_dict() shape so streaming and non-streaming
# error frames are byte-identical.
error_obj: Final[dict[str, Any]] = {
error_obj: Final[dict[str, object]] = {
"message": message,
"type": getattr(e, "type", "None"),
"param": getattr(e, "param", "None"),
@ -778,7 +819,7 @@ def _is_azure_model_router_request(model: str) -> bool:
def _override_openai_response_model(
*,
response_obj: Any,
response_obj: object,
requested_model: str,
log_context: str,
return_raw_model_name: bool = False,
@ -973,7 +1014,7 @@ def _log_llm_api_exception(e: Exception) -> None:
async def _cancel_llm_call_on_client_disconnect(
request: Request,
llm_api_call: "asyncio.Future[Any]",
llm_api_call: "asyncio.Future[object]",
disconnect_event: asyncio.Event,
) -> None:
try:
@ -1024,7 +1065,7 @@ class ProxyBaseLLMRequestProcessing:
version: str | None = None,
model_region: str | None = None,
response_cost: float | str | None = None,
hidden_params: dict | None = None,
hidden_params: Mapping[str, object] | None = None,
fastest_response_batch_completion: bool | None = None,
request_data: dict | None = {},
timeout: float | httpx.Timeout | None = None,
@ -1116,7 +1157,7 @@ class ProxyBaseLLMRequestProcessing:
@staticmethod
async def build_litellm_proxy_success_headers_from_llm_response(
*,
response: Any,
response: object,
request_data: dict,
request: Request,
user_api_key_dict: UserAPIKeyAuth,
@ -1907,7 +1948,7 @@ class ProxyBaseLLMRequestProcessing:
_captured_user_api_key_dict: Final = user_api_key_dict
_captured_logging_obj: Final = logging_obj
async def _on_deferred_stream_complete(assembled_response, cache_hit):
async def _on_deferred_stream_complete(assembled_response: object, cache_hit: object) -> None:
await ProxyBaseLLMRequestProcessing._run_deferred_stream_guardrails(
captured_data=_captured_data,
captured_user_api_key_dict=_captured_user_api_key_dict,
@ -2158,7 +2199,7 @@ class ProxyBaseLLMRequestProcessing:
@staticmethod
async def _record_container_owners_from_responses_if_needed(
response: Any,
response: object,
user_api_key_dict: UserAPIKeyAuth,
) -> None:
"""Register code-interpreter containers so follow-up file APIs pass ownership checks."""
@ -2181,7 +2222,7 @@ class ProxyBaseLLMRequestProcessing:
)
@staticmethod
def _extract_completed_responses_response(stream_response: Any) -> Any:
def _extract_completed_responses_response(stream_response: object) -> object:
"""Pull the assembled ``ResponsesAPIResponse`` off a streaming iterator.
``ResponsesAPIStreamingIterator`` stores the terminal stream event
@ -2191,17 +2232,17 @@ class ProxyBaseLLMRequestProcessing:
``ResponsesAPIResponse`` directly. Handle both shapes so the
container-ownership recording path can walk ``.output`` either way.
"""
completed: Final = getattr(stream_response, "completed_response", None)
completed: Final = _getattr_object(stream_response, "completed_response")
if completed is None:
return None
response_obj: Final = getattr(completed, "response", None)
response_obj: Final = _getattr_object(completed, "response")
if response_obj is not None:
return response_obj
return completed
@staticmethod
async def _wrap_responses_stream_for_container_ownership(
original_stream_response: Any,
original_stream_response: object,
wrapped_generator: Any,
user_api_key_dict: UserAPIKeyAuth,
):
@ -2300,12 +2341,13 @@ class ProxyBaseLLMRequestProcessing:
if isinstance(result, Response):
return result
content: Final = await result.aread()
upstream: Final = _as_upstream_response(result)
content: Final = await upstream.aread()
return Response(
content=content,
status_code=result.status_code,
status_code=upstream.status_code,
headers=HttpPassThroughEndpointHelpers.get_response_headers(
headers=result.headers,
headers=upstream.headers,
custom_headers=dict(fastapi_response.headers),
),
)
@ -2436,9 +2478,10 @@ class ProxyBaseLLMRequestProcessing:
HttpPassThroughEndpointHelpers,
)
upstream: Final = _as_upstream_response(response)
try:
response_status: Final[int] = response.status_code
content_type: Final[str] = response.headers.get("content-type", "")
response_status: Final[int] = upstream.status_code
content_type: Final[str] = _as_header_reader(upstream.headers).get("content-type", "")
except AttributeError:
return None
@ -2452,20 +2495,20 @@ class ProxyBaseLLMRequestProcessing:
return None
response_headers: Final = HttpPassThroughEndpointHelpers.get_response_headers(
headers=response.headers,
headers=upstream.headers,
custom_headers=custom_headers,
)
callback_headers: Final = await proxy_logging_obj.post_call_response_headers_hook(
data=self.data,
user_api_key_dict=user_api_key_dict,
response=response,
response=upstream,
request_headers=request_headers,
)
if callback_headers:
response_headers.update(callback_headers)
if is_event_stream:
body_bytes = await response.aread()
body_bytes = await upstream.aread()
modified_bytes: Final = await self._handle_event_stream_allm_passthrough_route(
body_bytes=body_bytes,
proxy_logging_obj=proxy_logging_obj,
@ -2478,7 +2521,7 @@ class ProxyBaseLLMRequestProcessing:
headers=response_headers,
)
body_bytes = await response.aread()
body_bytes = await upstream.aread()
try:
parsed: Final = _json.loads(body_bytes)
except (_json.JSONDecodeError, UnicodeDecodeError):
@ -2567,9 +2610,9 @@ class ProxyBaseLLMRequestProcessing:
async def _run_deferred_stream_guardrails(
captured_data: dict,
captured_user_api_key_dict: "UserAPIKeyAuth",
captured_logging_obj: Any,
captured_logging_obj: LiteLLMLoggingObj,
assembled_response: Any,
cache_hit: Any,
cache_hit: object,
) -> None:
"""
Run non-streaming post-call guardrail hooks on an assembled streaming
@ -2647,7 +2690,7 @@ class ProxyBaseLLMRequestProcessing:
# _is_sync_litellm_request (which only recognizes a subset of
# async markers stored in litellm_params).
asyncio.create_task(
captured_logging_obj.dispatch_success_handlers(
_as_success_dispatcher(captured_logging_obj).dispatch_success_handlers(
_response,
cache_hit=cache_hit,
start_time=None,
@ -2718,7 +2761,7 @@ class ProxyBaseLLMRequestProcessing:
headers = getattr(e, "headers", None) or {}
if not headers:
# Try to get headers from e.response.headers (httpx.Response)
_response: Final = getattr(e, "response", None)
_response: Final = _getattr_object(e, "response")
if _response is not None:
_response_headers: Final = getattr(_response, "headers", None)
if _response_headers:
@ -2750,7 +2793,7 @@ class ProxyBaseLLMRequestProcessing:
raise e
if isinstance(e, HTTPException):
raw_detail: Final = getattr(e, "detail", str(e))
raw_detail: Final = _getattr_object(e, "detail", str(e))
message, structured_fields = _serialize_http_exception_detail(raw_detail)
existing_fields: Final = getattr(e, "provider_specific_fields", None) or {}
if structured_fields:
@ -3043,8 +3086,16 @@ class ProxyBaseLLMRequestProcessing:
request=request,
)
@overload
@staticmethod
def _process_chunk_with_cost_injection(chunk: Any, model_name: str) -> Any:
def _process_chunk_with_cost_injection(chunk: bytes, model_name: str) -> bytes: ...
@overload
@staticmethod
def _process_chunk_with_cost_injection(chunk: object, model_name: str) -> object: ...
@staticmethod
def _process_chunk_with_cost_injection(chunk: object, model_name: str) -> object:
"""
Process a streaming chunk and inject cost information if enabled.

4
litellm/proxy/db/spend_log_tool_index.py
View file

@ -35,7 +35,7 @@ class ToolUsageTransaction:
total_tokens: int
def response_tool_call_names(completion_response: Any) -> tuple[str, ...]:
def response_tool_call_names(completion_response: object) -> tuple[str, ...]:
"""Tool names invoked in a completion response, in call order, for any response
surface get_tool_calls_from_response understands (chat completions, Responses
API output items, Anthropic Messages tool_use blocks). Reads every choice of
@ -59,7 +59,7 @@ def build_tool_usage_transaction(
mcp_namespaced_tool_name: str | None,
spend: float,
total_tokens: int,
completion_response: Any,
completion_response: object,
realtime_tool_calls: Any = None,
) -> ToolUsageTransaction | None:
"""None when the request invoked no tools. Realtime sessions carry invoked

43
litellm/proxy/hooks/parallel_request_limiter.py
View file

@ -20,6 +20,7 @@ from litellm.proxy.auth.auth_utils import (
from litellm.proxy.auth.budget_throttle import throttled_limit
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
from litellm.types.utils import Usage
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
@ -33,6 +34,13 @@ else:
InternalUsageCache = Any
def _response_total_tokens(response_obj: object) -> int:
if not isinstance(response_obj, (ModelResponse, EmbeddingResponse, TextCompletionResponse)):
return 0
response_usage: Final = getattr(response_obj, "usage", None)
return response_usage.total_tokens if isinstance(response_usage, Usage) else 0
class CacheObject(TypedDict):
current_global_requests: dict | None
request_count_api_key: dict | None
@ -480,7 +488,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
) # don't block execution for cache updates
)
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
async def async_log_success_event(self, kwargs, response_obj: object, start_time, end_time):
from litellm.proxy.common_utils.callback_utils import (
get_model_group_from_litellm_kwargs,
)
@ -529,21 +537,18 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
current_minute: Final = datetime.now().strftime("%M")
precise_minute: Final = f"{current_date}-{current_hour}-{current_minute}"
total_tokens = 0
if isinstance(response_obj, (ModelResponse, EmbeddingResponse, TextCompletionResponse)):
total_tokens = response_obj.usage.total_tokens
total_tokens: int = _response_total_tokens(response_obj)
# ------------
# Update usage - API Key
# ------------
values_to_update_in_cache: Final = []
values_to_update_in_cache: Final[list[tuple[str, object]]] = []
if user_api_key is not None:
request_count_api_key = f"{user_api_key}::{precise_minute}::request_count"
current = await self.internal_usage_cache.async_get_cache(
current: dict[str, int] = await self.internal_usage_cache.async_get_cache(
key=request_count_api_key,
litellm_parent_otel_span=litellm_parent_otel_span,
) or {
@ -606,13 +611,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
# Update usage - User
# ------------
if user_api_key_user_id is not None:
total_tokens = 0
if isinstance(
response_obj,
(ModelResponse, EmbeddingResponse, TextCompletionResponse),
):
total_tokens = response_obj.usage.total_tokens
total_tokens = _response_total_tokens(response_obj)
request_count_api_key = f"{user_api_key_user_id}::{precise_minute}::request_count"
@ -638,13 +637,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
# Update usage - Team
# ------------
if user_api_key_team_id is not None:
total_tokens = 0
if isinstance(
response_obj,
(ModelResponse, EmbeddingResponse, TextCompletionResponse),
):
total_tokens = response_obj.usage.total_tokens
total_tokens = _response_total_tokens(response_obj)
request_count_api_key = f"{user_api_key_team_id}::{precise_minute}::request_count"
@ -670,13 +663,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
# Update usage - End User
# ------------
if user_api_key_end_user_id is not None:
total_tokens = 0
if isinstance(
response_obj,
(ModelResponse, EmbeddingResponse, TextCompletionResponse),
):
total_tokens = response_obj.usage.total_tokens
total_tokens = _response_total_tokens(response_obj)
request_count_api_key = f"{user_api_key_end_user_id}::{precise_minute}::request_count"

9
litellm/proxy/image_endpoints/endpoints.py
View file

@ -1,8 +1,10 @@
import asyncio
import io
import traceback
from typing import Final
import orjson
from fastapi import APIRouter, Depends, File, HTTPException, Request, Response, status
from fastapi import APIRouter, Depends, File, HTTPException, Request, Response, UploadFile, status
from fastapi.responses import ORJSONResponse
import litellm
@ -18,11 +20,6 @@ from litellm.types.llms.openai import ChatCompletionUserMessage
router: Final = APIRouter()
import io
from typing import Final
from fastapi import UploadFile
async def uploadfile_to_bytesio(upload: UploadFile) -> io.BytesIO:
"""

18
litellm/proxy/management_endpoints/key_management_endpoints.py
View file

@ -190,6 +190,16 @@ class _PrismaTableActions(Protocol[_PrismaRowT]):
) -> _PrismaRowT | None: ...
class _UserRowLike(Protocol):
user_id: str | None
user_email: str | None
user_alias: str | None
def model_dump(self) -> Mapping[str, object]: ...
def dict(self) -> Mapping[str, object]: ...
class _TxTables(Protocol):
litellm_proxymodeltable: _PrismaTableActions[object]
@ -4225,7 +4235,7 @@ def _transform_verification_tokens_to_deleted_records(
record = deleted_record.model_dump()
# Map org_id to organization_id (model uses org_id, but schema expects organization_id)
org_id_value = record.pop("org_id", None)
org_id_value: object = record.pop("org_id", None)
if org_id_value is not None:
record["organization_id"] = org_id_value
@ -4694,7 +4704,7 @@ async def _execute_virtual_key_regeneration(
grace_period=data.grace_period if data else None,
)
updated_token: Final = await VerificationTokenRepository(prisma_client).table.update(
updated_token: Final[Mapping[str, object] | None] = await VerificationTokenRepository(prisma_client).table.update(
where={"token": hashed_api_key},
data=with_settings_updated_at(jsonified_update_data),
)
@ -5991,7 +6001,9 @@ async def _list_key_helper(
created_by_ids: Final = [key.created_by for key in keys if key.created_by]
all_ids: Final = list(set(user_ids + created_by_ids)) # Remove duplicates
if all_ids:
users: Final = await UserRepository(prisma_client).table.find_many(where={"user_id": {"in": all_ids}})
users: Final[Sequence[_UserRowLike]] = await UserRepository(prisma_client).table.find_many(
where={"user_id": {"in": all_ids}}
)
user_map = {user.user_id: user for user in users}
# Prepare response

2
litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py
View file

@ -92,7 +92,7 @@ class GeminiPassthroughLoggingHandler:
litellm_params={},
api_key="",
request_data={},
encoding=litellm.encoding,
encoding=getattr(litellm, "encoding", None),
)
kwargs = GeminiPassthroughLoggingHandler._create_gemini_response_logging_payload_for_generate_content(
litellm_model_response=litellm_model_response,

2
litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py
View file

@ -327,7 +327,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
optional_params=request_body.get("optional_params", {}),
api_key="",
request_data=request_body,
encoding=litellm.encoding,
encoding=getattr(litellm, "encoding", None),
json_mode=request_body.get("response_format", {}).get("type") == "json_object",
litellm_params=existing_litellm_params,
)

2
litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py
View file

@ -133,7 +133,7 @@ class VertexPassthroughLoggingHandler:
litellm_params={},
api_key="",
request_data={},
encoding=litellm.encoding,
encoding=getattr(litellm, "encoding", None),
)
kwargs = VertexPassthroughLoggingHandler._create_vertex_response_logging_payload_for_generate_content(
litellm_model_response=litellm_model_response,

96
litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
View file

@ -5,10 +5,10 @@ import json
import posixpath
import traceback
from base64 import b64encode
from collections.abc import AsyncGenerator, Mapping
from collections.abc import AsyncGenerator, Callable, Mapping
from datetime import datetime
from itertools import groupby
from typing import Any, Final, cast
from typing import Any, Final, TypedDict, cast
from urllib.parse import urlencode, urlparse
import httpx
@ -92,7 +92,7 @@ router: Final = APIRouter()
pass_through_endpoint_logging: Final = PassThroughEndpointLogging()
# Global registry to track registered pass-through routes and prevent memory leaks
_registered_pass_through_routes: Final[dict[str, dict[str, str | bool | list[str] | dict[str, Any]]]] = {}
_registered_pass_through_routes: Final[dict[str, dict[str, str | bool | list[str] | Mapping[str, object]]]] = {}
def get_response_body(response: httpx.Response) -> dict | None:
@ -1128,15 +1128,22 @@ async def pass_through_request(
else:
# SigV4-signed callers (Bedrock) supply the exact pre-signed bytes;
# otherwise httpx encodes the parsed JSON dict as before.
body_kwargs: Final[dict[str, Any]] = (
{"content": state_raw_body} if state_raw_body is not None else {"json": _parsed_body}
)
req: Final = async_client.build_request(
request.method,
url,
params=requested_query_params,
headers=headers,
**body_kwargs,
req: Final = (
async_client.build_request(
request.method,
url,
params=requested_query_params,
headers=headers,
content=state_raw_body,
)
if state_raw_body is not None
else async_client.build_request(
request.method,
url,
params=requested_query_params,
headers=headers,
json=_parsed_body,
)
)
response = await async_client.send(req, stream=stream)
@ -1584,9 +1591,15 @@ def _update_metadata_with_tags_in_header(request: Request, metadata: dict) -> di
return metadata
class _PassThroughRequestEnvelope(TypedDict, total=False):
query_params: Mapping[str, object] | None
custom_body: Mapping[str, object] | None
stream: bool | None
async def _parse_request_data_by_content_type(
request: Request,
) -> tuple[Any | None, Any | None, Any | None, Any | None]:
) -> tuple[object, object, None, bool | None]:
"""
Parse request data based on content type.
@ -1605,7 +1618,7 @@ async def _parse_request_data_by_content_type(
if "application/json" in content_type:
# ✅ Handle JSON
try:
body = await request.json()
body: _PassThroughRequestEnvelope = await request.json()
query_params_data = body.get("query_params")
custom_body_data = body.get("custom_body")
stream = body.get("stream")
@ -1646,7 +1659,7 @@ async def _parse_request_data_by_content_type(
def create_pass_through_route(
endpoint,
target: str,
custom_headers: Mapping[str, Any] | None = None,
custom_headers: Mapping[str, object] | None = None,
_forward_headers: bool | None = False,
_merge_query_params: bool | None = False,
dependencies: list | None = None,
@ -1656,7 +1669,7 @@ def create_pass_through_route(
is_streaming_request: bool | None = False,
query_params: dict | None = None,
default_query_params: dict | None = None,
guardrails: dict[str, Any] | None = None,
guardrails: dict[str, object] | None = None,
config_file_path: str | None = None,
timeout: float | None = None,
):
@ -1887,7 +1900,7 @@ async def websocket_passthrough_request(
# Initialize tracking variables
start_time: Final = datetime.now()
websocket_messages: Final[list[dict[str, Any]]] = []
websocket_messages: Final[list[dict[str, object]]] = []
litellm_call_id: Final = str(uuid.uuid4())
verbose_proxy_logger.info("WebSocket passthrough (%s): Starting WebSocket connection to %s", endpoint, target)
@ -1980,7 +1993,7 @@ async def websocket_passthrough_request(
)
### CALL HOOKS ### - modify incoming data / reject request before calling the model
websocket_data: dict[str, Any] = {}
websocket_data: dict[str, object] = {}
websocket_data = await proxy_logging_obj.pre_call_hook(
user_api_key_dict=user_api_key_dict,
data=websocket_data,
@ -2009,8 +2022,8 @@ async def websocket_passthrough_request(
await upstream_ws.close()
break
text_data = message.get("text")
bytes_data = message.get("bytes")
text_data: str | None = message.get("text")
bytes_data: bytes | None = message.get("bytes")
if text_data is not None:
# Try to extract model from client setup message for Vertex AI Live
@ -2086,7 +2099,7 @@ async def websocket_passthrough_request(
# Ensure raw_response is bytes before decoding
if isinstance(raw_response, str):
raw_response = raw_response.encode("ascii")
setup_response: Final = json.loads(raw_response.decode("ascii"))
setup_response: Final[Mapping[str, object]] = json.loads(raw_response.decode("ascii"))
verbose_proxy_logger.debug("Setup response: %s", setup_response)
# Extract model and provider from setup response for Vertex AI Live
@ -2129,7 +2142,7 @@ async def websocket_passthrough_request(
await websocket.send_bytes(upstream_message)
# Parse and collect for cost tracking
try:
message_data = json.loads(upstream_message.decode())
message_data: dict[str, object] = json.loads(upstream_message.decode())
websocket_messages.append(message_data)
except (json.JSONDecodeError, UnicodeDecodeError):
pass
@ -2315,7 +2328,8 @@ def _should_buffer_passthrough_response(response: httpx.Response) -> bool:
"""
if response.status_code >= 400:
return True
media_type: Final = response.headers.get("content-type", "").split(";")[0].strip().lower()
content_type_header: Final[str] = response.headers.get("content-type", "")
media_type: Final = content_type_header.split(";")[0].strip().lower()
return media_type in ("", "application/json") or media_type.endswith("+json")
@ -2368,7 +2382,7 @@ async def _relay_passthrough_response_bytes(
)
def _extract_model_from_vertex_ai_setup(setup_response: dict) -> str | None:
def _extract_model_from_vertex_ai_setup(setup_response: Mapping[str, object]) -> str | None:
"""
Extract the model name from Vertex AI Live setup response.
@ -2434,7 +2448,7 @@ class SafeRouteAdder:
def add_api_route_if_not_exists(
app: FastAPI,
path: str,
endpoint: Any,
endpoint: Callable[..., object],
methods: list[str],
dependencies: list | None = None,
) -> bool:
@ -2767,7 +2781,7 @@ def _get_combined_pass_through_endpoints(
async def _register_pass_through_endpoint(
endpoint: dict[str, Any] | PassThroughGenericEndpoint,
endpoint: dict[str, object] | PassThroughGenericEndpoint,
app: FastAPI,
premium_user: bool,
visited_endpoints: set[str],
@ -2783,8 +2797,8 @@ async def _register_pass_through_endpoint(
endpoint_data["id"] = str(uuid.uuid4())
endpoint_id: Final = cast(str, endpoint_data["id"])
target: Final = endpoint_data.get("target")
path: Final = endpoint_data.get("path")
target: Final[str | None] = endpoint_data.get("target")
path: Final[str | None] = endpoint_data.get("path")
if path is None:
raise ValueError("Path is required for pass-through endpoint")
@ -2792,7 +2806,7 @@ async def _register_pass_through_endpoint(
forward_headers: Final = endpoint_data.get("forward_headers")
merge_query_params: Final = endpoint_data.get("merge_query_params")
default_query_params: Final = endpoint_data.get("default_query_params")
auth: Final = endpoint_data.get("auth")
auth: Final[bool | str | None] = endpoint_data.get("auth")
dependencies = None
auth_enforced: Final = auth is not None and str(auth).lower() == "true"
@ -2951,12 +2965,12 @@ def _get_pass_through_endpoints_from_config() -> list[PassThroughGenericEndpoint
if isinstance(endpoint, dict):
endpoint_dict = dict(endpoint)
endpoint_dict["is_from_config"] = True
returned_endpoints.append(PassThroughGenericEndpoint(**endpoint_dict))
returned_endpoints.append(PassThroughGenericEndpoint.model_validate(endpoint_dict))
elif isinstance(endpoint, PassThroughGenericEndpoint):
# Create a copy with is_from_config=True
endpoint_dict = endpoint.model_dump()
endpoint_dict["is_from_config"] = True
returned_endpoints.append(PassThroughGenericEndpoint(**endpoint_dict))
returned_endpoints.append(PassThroughGenericEndpoint.model_validate(endpoint_dict))
except ValidationError as e:
verbose_proxy_logger.warning(
"Skipping malformed pass-through endpoint from config: %s",
@ -2994,11 +3008,11 @@ async def _get_pass_through_endpoints_from_db(
if isinstance(endpoint, dict):
endpoint_dict = dict(endpoint)
endpoint_dict["is_from_config"] = False
returned_endpoints.append(PassThroughGenericEndpoint(**endpoint_dict))
returned_endpoints.append(PassThroughGenericEndpoint.model_validate(endpoint_dict))
elif isinstance(endpoint, PassThroughGenericEndpoint):
endpoint_dict = endpoint.model_dump()
endpoint_dict["is_from_config"] = False
returned_endpoints.append(PassThroughGenericEndpoint(**endpoint_dict))
returned_endpoints.append(PassThroughGenericEndpoint.model_validate(endpoint_dict))
else:
# Find specific endpoint by ID
found_endpoint: Final = _find_endpoint_by_id(pass_through_endpoint_data, endpoint_id)
@ -3009,7 +3023,7 @@ async def _get_pass_through_endpoints_from_db(
else dict(found_endpoint)
)
endpoint_dict["is_from_config"] = False
returned_endpoints.append(PassThroughGenericEndpoint(**endpoint_dict))
returned_endpoints.append(PassThroughGenericEndpoint.model_validate(endpoint_dict))
return returned_endpoints
@ -3191,7 +3205,7 @@ async def update_pass_through_endpoints(
endpoint_dict.pop("is_from_config", None)
# Create updated endpoint object
updated_endpoint: Final = PassThroughGenericEndpoint(**endpoint_dict)
updated_endpoint: Final = PassThroughGenericEndpoint.model_validate(endpoint_dict)
# Update the list
pass_through_endpoint_data[endpoint_index] = endpoint_dict
@ -3212,9 +3226,10 @@ async def update_pass_through_endpoints(
_custom_headers: dict | None = updated_endpoint.headers or {}
_custom_headers = await set_env_variables_in_header(custom_headers=_custom_headers)
route_app: Final[FastAPI] = request.app
if updated_endpoint.include_subpath:
InitPassThroughEndpointHelpers.add_subpath_route(
app=request.app,
app=route_app,
path=updated_endpoint.path,
target=updated_endpoint.target,
custom_headers=_custom_headers,
@ -3231,7 +3246,7 @@ async def update_pass_through_endpoints(
)
else:
InitPassThroughEndpointHelpers.add_exact_path_route(
app=request.app,
app=route_app,
path=updated_endpoint.path,
target=updated_endpoint.target,
custom_headers=_custom_headers,
@ -3297,15 +3312,16 @@ async def create_pass_through_endpoints(
await update_config_general_settings(data=updated_data, user_api_key_dict=user_api_key_dict)
# Return the created endpoint with the generated ID
created_endpoint: Final = PassThroughGenericEndpoint(**data_dict)
created_endpoint: Final = PassThroughGenericEndpoint.model_validate(data_dict)
# Register the new route
_custom_headers: dict | None = created_endpoint.headers or {}
_custom_headers = await set_env_variables_in_header(custom_headers=_custom_headers)
route_app: Final[FastAPI] = request.app
if created_endpoint.include_subpath:
InitPassThroughEndpointHelpers.add_subpath_route(
app=request.app,
app=route_app,
path=created_endpoint.path,
target=created_endpoint.target,
custom_headers=_custom_headers,
@ -3322,7 +3338,7 @@ async def create_pass_through_endpoints(
)
else:
InitPassThroughEndpointHelpers.add_exact_path_route(
app=request.app,
app=route_app,
path=created_endpoint.path,
target=created_endpoint.target,
custom_headers=_custom_headers,

16
litellm/proxy/pass_through_endpoints/success_handler.py
View file

@ -122,7 +122,7 @@ class PassThroughEndpointLogging:
def normalize_llm_passthrough_logging_payload(
self,
httpx_response: httpx.Response,
response_body: dict | None,
response_body: dict | list[dict[str, object]] | None,
request_body: dict,
logging_obj: LiteLLMLoggingObj,
url_route: str,
@ -142,7 +142,7 @@ class PassThroughEndpointLogging:
if self.is_gemini_route(url_route, custom_llm_provider):
gemini_passthrough_logging_handler_result = GeminiPassthroughLoggingHandler.gemini_passthrough_handler(
httpx_response=httpx_response,
response_body=response_body or {},
response_body=response_body if isinstance(response_body, dict) else {},
logging_obj=logging_obj,
url_route=url_route,
result=result,
@ -172,7 +172,7 @@ class PassThroughEndpointLogging:
anthropic_passthrough_logging_handler_result: Final = (
AnthropicPassthroughLoggingHandler.anthropic_passthrough_handler(
httpx_response=httpx_response,
response_body=response_body or {},
response_body=response_body if isinstance(response_body, dict) else {},
logging_obj=logging_obj,
url_route=url_route,
result=result,
@ -189,7 +189,7 @@ class PassThroughEndpointLogging:
elif self.is_cohere_route(url_route):
cohere_passthrough_logging_handler_result = cohere_passthrough_logging_handler.cohere_passthrough_handler(
httpx_response=httpx_response,
response_body=response_body or {},
response_body=response_body if isinstance(response_body, dict) else {},
logging_obj=logging_obj,
url_route=url_route,
result=result,
@ -208,7 +208,7 @@ class PassThroughEndpointLogging:
openai_passthrough_logging_handler_result = OpenAIPassthroughLoggingHandler.openai_passthrough_handler(
httpx_response=httpx_response,
response_body=response_body or {},
response_body=response_body if isinstance(response_body, dict) else {},
logging_obj=logging_obj,
url_route=url_route,
result=result,
@ -224,7 +224,7 @@ class PassThroughEndpointLogging:
elif self.is_cursor_route(url_route, custom_llm_provider):
cursor_passthrough_logging_handler_result = CursorPassthroughLoggingHandler.cursor_passthrough_handler(
httpx_response=httpx_response,
response_body=response_body or {},
response_body=response_body if isinstance(response_body, dict) else {},
logging_obj=logging_obj,
url_route=url_route,
result=result,
@ -266,7 +266,7 @@ class PassThroughEndpointLogging:
async def pass_through_async_success_handler(
self,
httpx_response: httpx.Response,
response_body: dict | None,
response_body: dict | list[dict[str, object]] | None,
logging_obj: LiteLLMLoggingObj,
url_route: str,
result: str,
@ -285,7 +285,7 @@ class PassThroughEndpointLogging:
return
self.assemblyai_passthrough_logging_handler.assemblyai_passthrough_logging_handler(
httpx_response=httpx_response,
response_body=response_body or {},
response_body=response_body if isinstance(response_body, dict) else {},
logging_obj=logging_obj,
url_route=url_route,
result=result,

214
litellm/proxy/proxy_server.py
View file

@ -9,13 +9,14 @@ import random
import re
import secrets
import shutil
import socket
import subprocess
import sys
import threading
import time
import traceback
import warnings
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping, MutableMapping, Sequence
from datetime import datetime, timedelta, timezone
from types import MappingProxyType, UnionType
from typing import (
@ -25,6 +26,8 @@ from typing import (
Literal,
NamedTuple,
Optional,
Protocol,
TypeAlias,
TypedDict,
Union,
cast,
@ -128,6 +131,7 @@ from litellm.utils import (
if TYPE_CHECKING:
from aiohttp import ClientSession
from fastapi.routing import APIRoute
from opentelemetry.trace import Span as _Span
from litellm.integrations.opentelemetry import OpenTelemetry
@ -137,7 +141,7 @@ else:
Span = Any
OpenTelemetry = Any
REALTIME_REQUEST_SCOPE_TEMPLATE: Final[dict[str, Any]] = {
REALTIME_REQUEST_SCOPE_TEMPLATE: Final[dict[str, object]] = {
"type": "http",
"method": "POST",
"path": "/v1/realtime",
@ -599,6 +603,7 @@ from litellm.proxy.utils import (
update_spend,
)
from litellm.proxy.video_endpoints.endpoints import router as video_router
from litellm.repositories.base_repository import SupportsModelDump
from litellm.repositories.credentials_repository import CredentialsRepository
from litellm.router import (
AssistantsTypedDict,
@ -873,6 +878,18 @@ async def proxy_shutdown_event():
cleanup_router_config_variables()
_AiohttpAddrInfo: TypeAlias = tuple[int | socket.AddressFamily, int | socket.SocketKind, int, str, tuple[object, ...]]
class _AiohttpConnectorKwargs(TypedDict, total=False):
keepalive_timeout: float
ttl_dns_cache: int
enable_cleanup_closed: bool
limit: int
limit_per_host: int
socket_factory: Callable[[_AiohttpAddrInfo], socket.socket]
async def _initialize_shared_aiohttp_session():
"""Initialize shared aiohttp session for connection reuse with connection limits."""
try:
@ -882,7 +899,7 @@ async def _initialize_shared_aiohttp_session():
_build_aiohttp_keepalive_socket_factory,
)
connector_kwargs: Final[dict[str, Any]] = {
connector_kwargs: Final[_AiohttpConnectorKwargs] = {
"keepalive_timeout": AIOHTTP_KEEPALIVE_TIMEOUT,
"ttl_dns_cache": AIOHTTP_TTL_DNS_CACHE,
}
@ -1237,7 +1254,7 @@ async def proxy_startup_event(app: FastAPI):
await proxy_shutdown_event()
def _generate_stable_operation_id(route: Any) -> str:
def _generate_stable_operation_id(route: "APIRoute") -> str:
operation_id = re.sub(r"\W", "_", f"{route.name}{route.path_format}")
route_methods: Final = sorted(route.methods or [])
if len(route_methods) == 1:
@ -1496,7 +1513,7 @@ async def openai_exception_handler(request: Request, exc: ProxyException):
def _close_dangling_otel_server_span(request: Request, status_code: int, exc: Exception | None = None) -> None:
parent_otel_span: Final = getattr(request.state, "parent_otel_span", None)
parent_otel_span: Final[_Span | None] = getattr(request.state, "parent_otel_span", None)
if parent_otel_span is None:
return
if open_telemetry_logger is None:
@ -1539,17 +1556,80 @@ async def management_problem_exception_handler(request: Request, exc: Management
return problem_response(exc.problem)
class _ConfigParamRow(Protocol):
param_name: str
param_value: Mapping[str, JsonValue] | None
class _ConfigOverridesRow(Protocol):
config_value: Mapping[str, JsonValue] | None
class _SSOConfigRow(Protocol):
sso_settings: MutableMapping[str, object]
class _UISettingsRow(Protocol):
ui_settings: Mapping[str, object] | str | None
class _InvitationLinkRow(Protocol):
user_id: str
expires_at: datetime
is_accepted: bool
accepted_at: datetime | None
created_by: str
class _UserTableRow(Protocol):
user_id: str
user_email: str | None
user_role: str
class _ModelTableRow(Protocol):
model_id: str | None
created_by: str | None
class _TTFTRow(TypedDict):
api_base: str
model: str
time_to_first_token: float
request_id: str
day: str
class _LatencyRow(TypedDict):
api_base: str | None
model: str
day: str
avg_latency_per_token: float
class _ExceptionRow(TypedDict, total=False):
combined_model_api_base: str
total_exceptions: int
exception_counts: Mapping[str, int]
class _ValidationErrorDetail(TypedDict):
loc: tuple[int | str, ...]
msg: str
@app.exception_handler(RequestValidationError)
async def otel_request_validation_exception_handler(request: Request, exc: RequestValidationError):
if request.url.path.startswith(MANAGEMENT_V1_PREFIX):
_close_dangling_otel_server_span(request, 400, exc=exc)
validation_errors: Final[Sequence[_ValidationErrorDetail]] = exc.errors()
return problem_response(
ProblemDetail(
type=f"{PROBLEM_TYPE_BASE}invalid-query-parameter",
title="Invalid query parameter",
status=400,
detail="; ".join(
f"{'.'.join(str(part) for part in error['loc'][1:])}: {error['msg']}" for error in exc.errors()
f"{'.'.join(str(part) for part in error['loc'][1:])}: {error['msg']}" for error in validation_errors
)
or "The request query parameters are invalid.",
)
@ -2153,13 +2233,14 @@ db_writer_client: AsyncHTTPHandler | None = None
### logger ###
def _resolve_typed_dict_type(typ):
def _resolve_typed_dict_type(typ: object):
"""Resolve the actual TypedDict class from a potentially wrapped type."""
from typing_extensions import _TypedDictMeta
origin: Final = get_origin(typ)
origin: Final[object] = get_origin(typ)
if origin is Union or origin is UnionType: # Check if it's a Union (like Optional)
for arg in get_args(typ):
union_args: Final[tuple[object, ...]] = get_args(typ)
for arg in union_args:
if isinstance(arg, _TypedDictMeta):
return arg
elif isinstance(typ, type) and isinstance(typ, dict):
@ -2167,12 +2248,13 @@ def _resolve_typed_dict_type(typ):
return None
def _resolve_pydantic_type(typ) -> list:
def _resolve_pydantic_type(typ: object) -> list:
"""Resolve the actual TypedDict class from a potentially wrapped type."""
origin: Final = get_origin(typ)
origin: Final[object] = get_origin(typ)
typs: Final = []
if origin is Union or origin is UnionType: # Check if it's a Union (like Optional)
for arg in get_args(typ):
union_args: Final[tuple[object, ...]] = get_args(typ)
for arg in union_args:
if arg is not None and "NoneType" not in str(arg):
typs.append(arg)
elif isinstance(typ, type) and isinstance(typ, BaseModel):
@ -2502,7 +2584,7 @@ async def increment_spend_counters(
increment=cost,
)
key_obj: Final = await user_api_key_cache.async_get_cache(key=hashed_token)
key_obj: Final[object] = await user_api_key_cache.async_get_cache(key=hashed_token)
if key_obj is None:
return
key_budget_limits = getattr(key_obj, "budget_limits", None) or (
@ -2533,7 +2615,7 @@ async def increment_spend_counters(
increment=cost,
)
team_obj: Final = await user_api_key_cache.async_get_cache(key=f"team_id:{scope_team_id}")
team_obj: Final[object] = await user_api_key_cache.async_get_cache(key=f"team_id:{scope_team_id}")
if team_obj is None:
return
team_budget_limits = getattr(team_obj, "budget_limits", None) or (
@ -2827,7 +2909,7 @@ async def _ensure_window_spend_counter_initialized(
async def _is_spend_counter_cache_warm(counter_key: str) -> bool:
if spend_counter_cache.redis_cache is not None:
try:
current_value: Final = await spend_counter_cache.redis_cache.async_get_cache(
current_value: Final[object] = await spend_counter_cache.redis_cache.async_get_cache(
key=counter_key,
)
if current_value is None:
@ -2899,7 +2981,7 @@ async def update_cache(
Put any alerting logic in here.
"""
values_to_update_in_cache: Final[list[tuple[Any, Any]]] = []
values_to_update_in_cache: Final[list[tuple[str, object]]] = []
### UPDATE KEY SPEND ###
async def _update_key_cache(token: str, response_cost: float):
@ -4111,7 +4193,9 @@ class ProxyConfig:
if prisma_client is None or not (general_settings.get("store_model_in_db", False) is True or store_model_in_db):
return
row = await ConfigRepository(prisma_client).table.find_first(where={"param_name": "environment_variables"})
row: Final[_ConfigParamRow | None] = await ConfigRepository(prisma_client).table.find_first(
where={"param_name": "environment_variables"}
)
existing: Final[dict] = dict(row.param_value) if row is not None and row.param_value is not None else {}
to_set: Final = {k: v for k, v in updates.items() if v is not None}
@ -5909,7 +5993,7 @@ class ProxyConfig:
4. Update router settings
"""
if llm_router is not None and prisma_client is not None:
db_router_settings: Final = await ConfigRepository(prisma_client).table.find_first(
db_router_settings: Final[_ConfigParamRow | None] = await ConfigRepository(prisma_client).table.find_first(
where={"param_name": "router_settings"}
)
@ -6394,7 +6478,9 @@ class ProxyConfig:
new_models=new_models, proxy_logging_obj=proxy_logging_obj
)
db_general_settings: Final = await get_config_param(prisma_client, "general_settings")
db_general_settings: Final[_ConfigParamRow | None] = await get_config_param(
prisma_client, "general_settings"
)
# update general settings
if db_general_settings is not None:
@ -6590,7 +6676,7 @@ class ProxyConfig:
"""
try:
sso_settings: Final = await call_with_db_reconnect_retry(
sso_settings: Final[_SSOConfigRow | None] = await call_with_db_reconnect_retry(
prisma_client,
lambda: SSOConfigRepository(prisma_client).table.find_unique(where={"id": "sso_config"}),
reason="init_sso_settings_in_db_lookup_failure",
@ -6621,7 +6707,7 @@ class ProxyConfig:
)
try:
db_record: Final = await call_with_db_reconnect_retry(
db_record: Final[_ConfigOverridesRow | None] = await call_with_db_reconnect_retry(
prisma_client,
lambda: ConfigOverridesRepository(prisma_client).table.find_unique(
where={"config_type": "hashicorp_vault"}
@ -6839,7 +6925,7 @@ class ProxyConfig:
from litellm.types.prompts.init_prompts import PromptSpec
try:
prompts_in_db: Final = await PromptRepository(prisma_client).table.find_many()
prompts_in_db: Final[Sequence[object]] = await PromptRepository(prisma_client).table.find_many()
for prompt in prompts_in_db:
# Convert DB object to dict and create versioned prompt_id
prompt_spec = self._get_prompt_spec_for_db_prompt(db_prompt=prompt)
@ -8451,7 +8537,9 @@ class ProxyStartupEvent:
if prisma_client is None:
return
db_record: Final = await UISettingsRepository(prisma_client).table.find_unique(where={"id": "ui_settings"})
db_record: Final[_UISettingsRow | None] = await UISettingsRepository(prisma_client).table.find_unique(
where={"id": "ui_settings"}
)
if db_record and db_record.ui_settings:
raw: Final = db_record.ui_settings
ui_settings: Final = json.loads(raw) if isinstance(raw, str) else dict(raw)
@ -8600,7 +8688,7 @@ class ProxyStartupEvent:
# but YAML config has False.
if store_model_in_db is not True and prisma_client is not None:
try:
_db_gs_record: Final = await ConfigRepository(prisma_client).table.find_first(
_db_gs_record: Final[_ConfigParamRow | None] = await ConfigRepository(prisma_client).table.find_first(
where={"param_name": "general_settings"}
)
if _db_gs_record is not None and isinstance(_db_gs_record.param_value, dict):
@ -10479,7 +10567,7 @@ async def vertex_ai_live_passthrough_endpoint(
None,
description="Override the Vertex AI region (for example, 'us-central1').",
),
user_api_key_dict=Depends(user_api_key_auth_websocket),
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth_websocket),
):
"""
Vertex AI Live API WebSocket Pass-through Endpoint
@ -10527,7 +10615,7 @@ async def realtime_websocket_endpoint(
None,
description="Comma-separated list of guardrail names to apply to this request.",
),
user_api_key_dict=Depends(user_api_key_auth_websocket),
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth_websocket),
):
requested_protocols: Final = [
p.strip() for p in (websocket.headers.get("sec-websocket-protocol") or "").split(",") if p.strip()
@ -10559,7 +10647,7 @@ async def realtime_websocket_endpoint(
# Only use explicit parameters, not all query params
query_params: Final = cast(RealtimeQueryParams, dict(_realtime_query_params_template(model, intent)))
data: dict[str, Any] = {
data: dict[str, object] = {
"model": route_model,
"websocket": websocket,
"query_params": query_params, # Only explicit params
@ -11692,7 +11780,7 @@ async def _check_if_model_is_user_added(
id = model.get("model_info", {}).get("id", None)
if id is None:
continue
db_model = await ModelRepository(prisma_client).table.find_unique(where={"model_id": id})
db_model: _ModelTableRow | None = await ModelRepository(prisma_client).table.find_unique(where={"model_id": id})
if db_model is not None:
if db_model.created_by == user_api_key_dict.user_id:
filtered_models.append(model)
@ -11876,7 +11964,7 @@ async def get_all_team_models(
team_db_objects_typed: list[LiteLLM_TeamTable] = []
if user_teams == "*":
team_db_objects = await TeamRepository(prisma_client).table.find_many()
team_db_objects: Sequence[SupportsModelDump] = await TeamRepository(prisma_client).table.find_many()
team_db_objects_typed = [
LiteLLM_TeamTable.model_validate(team_db_object.model_dump()) for team_db_object in team_db_objects
]
@ -11955,7 +12043,7 @@ async def _populate_team_access_on_models(
user_teams = "*"
direct_access_models = llm_router.get_model_ids(exclude_team_models=True) # has access to all models
elif user_api_key_dict.user_id is not None:
user_db_object: Final = await UserRepository(prisma_client).table.find_unique(
user_db_object: Final[SupportsModelDump | None] = await UserRepository(prisma_client).table.find_unique(
where={"user_id": user_api_key_dict.user_id}
)
if user_db_object is not None:
@ -12511,7 +12599,9 @@ def _team_models_resolve_to_names(team_models: list[str], access_groups: dict[st
async def _load_team_object_for_model_filter(team_id: str, prisma_client: PrismaClient) -> LiteLLM_TeamTable | None:
"""Load team row from DB; returns None if missing or on error."""
try:
team_db_object: Final = await TeamRepository(prisma_client).table.find_unique(where={"team_id": team_id})
team_db_object: Final[SupportsModelDump | None] = await TeamRepository(prisma_client).table.find_unique(
where={"team_id": team_id}
)
if team_db_object is None:
verbose_proxy_logger.warning("Team %s not found in database", team_id)
return None
@ -12560,7 +12650,7 @@ async def _gather_team_accessible_model_ids(
try:
if team_object.models and SpecialModelNames.all_proxy_models.value not in team_object.models:
_resolved_names: Final = _team_models_resolve_to_names(team_object.models, access_groups)
db_models: Final = await ModelRepository(prisma_client).table.find_many(
db_models: Final[Sequence[_ModelTableRow]] = await ModelRepository(prisma_client).table.find_many(
where={"model_name": {"in": _resolved_names}}
)
for db_model in db_models:
@ -13022,7 +13112,9 @@ async def model_streaming_metrics(
"""
_all_api_bases: Final = set()
db_response: Final = await prisma_client.db.query_raw(sql_query, _selected_model_group, startTime, endTime)
db_response: Final[Sequence[_TTFTRow] | None] = await prisma_client.db.query_raw(
sql_query, _selected_model_group, startTime, endTime
)
_daily_entries: dict = {} # {"Jun 23": {"model1": 0.002, "model2": 0.003}}
if db_response is not None:
for model_data in db_response:
@ -13144,7 +13236,7 @@ async def model_metrics(
avg_latency_per_token DESC;
"""
_all_api_bases: Final = set()
db_response: Final = await prisma_client.db.query_raw(
db_response: Final[Sequence[_LatencyRow] | None] = await prisma_client.db.query_raw(
sql_query, _selected_model_group, startTime, endTime, api_key, customer
)
_daily_entries: dict = {} # {"Jun 23": {"model1": 0.002, "model2": 0.003}}
@ -13335,7 +13427,9 @@ async def model_metrics_exceptions(
ORDER BY total_exceptions DESC
LIMIT 200;
"""
db_response: Final = await prisma_client.db.query_raw(sql_query, startTime, endTime, _selected_model_group, api_key)
db_response: Final[Sequence[_ExceptionRow] | None] = await prisma_client.db.query_raw(
sql_query, startTime, endTime, _selected_model_group, api_key
)
response: Final[list[dict]] = []
exception_types: Final = set()
@ -14497,7 +14591,9 @@ async def onboarding(invite_link: str, request: Request):
detail={"error": CommonProxyErrors.db_not_connected_error.value},
)
invite_obj: Final = await InvitationLinkRepository(prisma_client).table.find_unique(where={"id": invite_link})
invite_obj: Final[_InvitationLinkRow | None] = await InvitationLinkRepository(prisma_client).table.find_unique(
where={"id": invite_link}
)
if invite_obj is None:
raise HTTPException(status_code=401, detail={"error": "Invitation link does not exist in db."})
#### CHECK IF EXPIRED
@ -14515,7 +14611,9 @@ async def onboarding(invite_link: str, request: Request):
)
### GET USER OBJECT ###
user_obj: Final = await UserRepository(prisma_client).table.find_unique(where={"user_id": invite_obj.user_id})
user_obj: Final[_UserTableRow | None] = await UserRepository(prisma_client).table.find_unique(
where={"user_id": invite_obj.user_id}
)
if user_obj is None:
raise HTTPException(status_code=401, detail={"error": "User does not exist in db."})
@ -14687,7 +14785,9 @@ async def claim_onboarding_link(data: InvitationClaim, request: Request):
detail={"error": CommonProxyErrors.db_not_connected_error.value},
)
invite_obj = await InvitationLinkRepository(prisma_client).table.find_unique(where={"id": data.invitation_link})
invite_obj: Final[_InvitationLinkRow | None] = await InvitationLinkRepository(prisma_client).table.find_unique(
where={"id": data.invitation_link}
)
if invite_obj is None:
raise HTTPException(status_code=401, detail={"error": "Invitation link does not exist in db."})
#### CHECK IF EXPIRED
@ -14742,7 +14842,7 @@ async def claim_onboarding_link(data: InvitationClaim, request: Request):
)
### UPDATE USER OBJECT ###
user_obj: Final = await tx.litellm_usertable.update(
user_obj: Final[_UserTableRow | None] = await tx.litellm_usertable.update(
where={"user_id": invite_obj.user_id}, data={"password": hashed_pw}
)
@ -14968,7 +15068,7 @@ async def new_invitation(data: InvitationNew, user_api_key_dict: UserAPIKeyAuth
detail={"error": "You can only create invitations for users in your organization or team."},
)
response: Final = await create_invitation_for_user(
response: Final[object] = await create_invitation_for_user(
data=data,
user_api_key_dict=user_api_key_dict,
)
@ -15010,7 +15110,9 @@ async def invitation_info(invitation_id: str, user_api_key_dict: UserAPIKeyAuth
detail={"error": f"{CommonProxyErrors.not_allowed_access.value}, your role={user_api_key_dict.user_role}"},
)
response: Final = await InvitationLinkRepository(prisma_client).table.find_unique(where={"id": invitation_id})
response: Final[object] = await InvitationLinkRepository(prisma_client).table.find_unique(
where={"id": invitation_id}
)
if response is None:
raise HTTPException(
@ -15058,7 +15160,7 @@ async def invitation_update(
)
current_time: Final = litellm.utils.get_utc_datetime()
response: Final = await InvitationLinkRepository(prisma_client).table.update(
response: Final[object] = await InvitationLinkRepository(prisma_client).table.update(
where={"id": data.invitation_id},
data={
"id": data.invitation_id,
@ -15124,7 +15226,9 @@ async def invitation_delete(
# Org admins can only delete invitations they created
if is_other_admin and not is_proxy_admin:
invitation = await InvitationLinkRepository(prisma_client).table.find_unique(where={"id": data.invitation_id})
invitation: Final[_InvitationLinkRow | None] = await InvitationLinkRepository(prisma_client).table.find_unique(
where={"id": data.invitation_id}
)
if invitation is None:
raise HTTPException(
status_code=400,
@ -15136,7 +15240,9 @@ async def invitation_delete(
detail={"error": "Organization admins can only delete invitations they created."},
)
response: Final = await InvitationLinkRepository(prisma_client).table.delete(where={"id": data.invitation_id})
response: Final[object] = await InvitationLinkRepository(prisma_client).table.delete(
where={"id": data.invitation_id}
)
if response is None:
raise HTTPException(
@ -15174,7 +15280,9 @@ async def update_config(
raise Exception("No DB Connected")
async def _read_section(param_name: str) -> dict:
row: Final = await ConfigRepository(prisma_client).table.find_first(where={"param_name": param_name})
row: Final[_ConfigParamRow | None] = await ConfigRepository(prisma_client).table.find_first(
where={"param_name": param_name}
)
if row is None or row.param_value is None:
return {}
return dict(row.param_value)
@ -15197,7 +15305,7 @@ async def update_config(
if config_info.general_settings is not None:
existing = await _read_section("general_settings")
before_general_settings: Final = copy.deepcopy(existing)
updates = config_info.general_settings.dict(exclude_none=True)
updates: Mapping[str, JsonValue] = config_info.general_settings.dict(exclude_none=True)
for k, v in updates.items():
if k == "alert_to_webhook_url":
if "alerting" not in existing:
@ -15637,7 +15745,7 @@ async def get_config_general_settings(
)
## get general settings from db
db_general_settings: Final = await ConfigRepository(prisma_client).table.find_first(
db_general_settings: Final[_ConfigParamRow | None] = await ConfigRepository(prisma_client).table.find_first(
where={"param_name": "general_settings"}
)
### pop the value
@ -15826,12 +15934,12 @@ async def get_config_list(
is_full_admin: Final = user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN
## get general settings from db
db_general_settings: Final = await ConfigRepository(prisma_client).table.find_first(
db_general_settings: Final[_ConfigParamRow | None] = await ConfigRepository(prisma_client).table.find_first(
where={"param_name": "general_settings"}
)
if db_general_settings is not None and db_general_settings.param_value is not None:
db_general_settings_dict = dict(db_general_settings.param_value)
db_general_settings_dict: Mapping[str, JsonValue] = dict(db_general_settings.param_value)
else:
db_general_settings_dict = {}
@ -15922,7 +16030,7 @@ async def get_config_list(
)
return_val.append(_response_obj)
db_litellm_settings_row: Final = await ConfigRepository(prisma_client).table.find_first(
db_litellm_settings_row: Final[_ConfigParamRow | None] = await ConfigRepository(prisma_client).table.find_first(
where={"param_name": "litellm_settings"}
)
db_litellm_settings: Final[dict] = (
@ -15999,7 +16107,7 @@ async def delete_config_general_settings(
)
## get general settings from db
db_general_settings: Final = await ConfigRepository(prisma_client).table.find_first(
db_general_settings: Final[_ConfigParamRow | None] = await ConfigRepository(prisma_client).table.find_first(
where={"param_name": "general_settings"}
)
### pop the value
@ -16566,7 +16674,7 @@ async def reload_anthropic_beta_headers(
last_anthropic_beta_headers_reload = current_time.isoformat()
# Set force reload flag in database for other pods, preserving existing interval_hours
existing_beta_config: Final = await ConfigRepository(prisma_client).table.find_unique(
existing_beta_config: Final[_ConfigParamRow | None] = await ConfigRepository(prisma_client).table.find_unique(
where={"param_name": "anthropic_beta_headers_reload_config"}
)
existing_beta_interval = None
@ -17154,7 +17262,7 @@ async def _is_mcp_access_group_cached(name: str) -> bool:
)
cache_key: Final = f"mcp_access_group_exists:{name}"
cached: Final = await user_api_key_cache.async_get_cache(key=cache_key)
cached: Final[object] = await user_api_key_cache.async_get_cache(key=cache_key)
if cached is not None:
return bool(cached)
result: Final = bool(await MCPRequestHandler._get_mcp_servers_from_access_groups([name]))

5
litellm/proxy/rerank_endpoints/endpoints.py
View file

@ -1,5 +1,8 @@
#### Rerank Endpoints #####
import asyncio
from typing import Final
import orjson
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
from fastapi.responses import ORJSONResponse
@ -10,8 +13,6 @@ from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
router: Final = APIRouter()
import asyncio
from typing import Final
@router.post(

2
litellm/proxy/response_api_endpoints/endpoints.py
View file

@ -95,7 +95,7 @@ def _normalize_tool_dialect(
def _is_chat_completions_body(data: Mapping[str, Any]) -> bool:
messages: Final = data.get("messages")
if isinstance(messages, list) and len(messages) > 0:
if isinstance(messages, list) and messages:
return True
return "messages" in data and "input" not in data

101
litellm/proxy/utils.py
View file

@ -15,7 +15,7 @@ from dataclasses import dataclass, field
from datetime import date, datetime, timedelta, timezone
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, Union, cast, overload
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, TypeVar, Union, cast, overload
from litellm import _custom_logger_compatible_callbacks_literal
from litellm.constants import (
@ -164,23 +164,26 @@ if TYPE_CHECKING:
from mcp.types import CallToolResult
from opentelemetry.trace import Span as _Span
from prisma.client import TransactionManager
from prisma.types import HttpConfig
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.models.team import LiteLLM_TeamTableCachedObj
from litellm.proxy.db.autorouter_session_rollup import AutoRouterTurnTransaction
from litellm.proxy.db.spend_log_tool_index import ToolUsageTransaction
Span = _Span | Any
Span = _Span | object
else:
Span = Any
_T: Final = TypeVar("_T")
unified_guardrail: Final = UnifiedLLMGuardrails()
NON_OPENAI_STREAM_GUARDRAIL_TRANSLATION_CALL_TYPES: "frozenset[CallTypes]" = frozenset({CallTypes.anthropic_messages})
def print_verbose(print_statement):
def print_verbose(print_statement: object):
"""
Prints the given `print_statement` to the console if `litellm.set_verbose` is True.
Also logs the `print_statement` at the debug level using `verbose_proxy_logger`.
@ -228,10 +231,10 @@ class InternalUsageCache:
async def async_get_cache(
self,
key,
key: str,
litellm_parent_otel_span: Span | None,
local_only: bool = False,
**kwargs,
**kwargs: object,
) -> Any:
return await self.dual_cache.async_get_cache(
key=key,
@ -242,11 +245,11 @@ class InternalUsageCache:
async def async_set_cache(
self,
key,
value,
key: str,
value: object,
litellm_parent_otel_span: Span | None,
local_only: bool = False,
**kwargs,
**kwargs: object,
) -> None:
return await self.dual_cache.async_set_cache(
key=key,
@ -258,10 +261,10 @@ class InternalUsageCache:
async def async_batch_set_cache(
self,
cache_list: list,
cache_list: list[tuple[str, object]],
litellm_parent_otel_span: Span | None,
local_only: bool = False,
**kwargs,
**kwargs: object,
) -> None:
return await self.dual_cache.async_set_cache_pipeline(
cache_list=cache_list,
@ -272,19 +275,19 @@ class InternalUsageCache:
async def async_batch_get_cache(
self,
keys: list,
keys: Sequence[str | None],
parent_otel_span: Span | None = None,
local_only: bool = False,
):
return await self.dual_cache.async_batch_get_cache(
keys=keys,
keys=list(keys),
parent_otel_span=parent_otel_span,
local_only=local_only,
)
async def async_increment_cache(
self,
key,
key: str,
value: float,
litellm_parent_otel_span: Span | None,
local_only: bool = False,
@ -300,10 +303,10 @@ class InternalUsageCache:
def set_cache(
self,
key,
value,
key: str,
value: object,
local_only: bool = False,
**kwargs,
**kwargs: object,
) -> None:
return self.dual_cache.set_cache(
key=key,
@ -314,9 +317,9 @@ class InternalUsageCache:
def get_cache(
self,
key,
key: str,
local_only: bool = False,
**kwargs,
**kwargs: object,
) -> Any:
return self.dual_cache.get_cache(
key=key,
@ -339,7 +342,7 @@ def _accepts_litellm_call_info(cb: CustomLogger) -> bool:
return _CALLBACK_ACCEPTS_CALL_INFO[key]
def _enrich_http_exception_with_guardrail_context(exc: BaseException, callback: Any) -> None:
def _enrich_http_exception_with_guardrail_context(exc: BaseException, callback: object) -> None:
"""
If `exc` is an HTTPException with a dict `detail`, mutate it in place to
add `guardrail_name` and `guardrail_mode` taken from the callback instance.
@ -392,7 +395,7 @@ class _CallbackCapabilities:
# Resolved CustomLogger callbacks in original order. Pre-resolving once
# avoids the per-request ``get_custom_logger_compatible_class`` walk for
# every string entry in ``litellm.callbacks``.
resolved_callbacks: tuple[Any, ...] = field(default_factory=tuple)
resolved_callbacks: tuple[object, ...] = field(default_factory=tuple)
class ProxyLogging:
@ -678,7 +681,7 @@ class ProxyLogging:
return synthetic_data
def _convert_llm_result_to_mcp_response(self, llm_result, request_obj) -> Any | None:
def _convert_llm_result_to_mcp_response(self, llm_result, request_obj) -> MCPPreCallResponseObject | None:
"""
Convert LLM guardrail result back to MCP response format.
"""
@ -804,7 +807,7 @@ class ProxyLogging:
verbose_proxy_logger.error("Error in manual argument parsing: %s", e)
return None
def _convert_llm_result_to_mcp_during_response(self, llm_result, request_obj) -> Any | None:
def _convert_llm_result_to_mcp_during_response(self, llm_result, request_obj) -> MCPDuringCallResponseObject | None:
"""
Convert LLM guardrail result back to MCP during call response format.
"""
@ -850,7 +853,7 @@ class ProxyLogging:
self,
response: MCPPreCallResponseObject,
original_request: MCPPreCallRequestObject,
) -> dict[str, Any]:
) -> Mapping[str, object]:
"""
Parse the response from the pre_mcp_tool_call_hook
@ -953,8 +956,8 @@ class ProxyLogging:
data: dict,
user_api_key_dict: UserAPIKeyAuth | None,
call_type: CallTypesLiteral,
response: Any | None = None,
) -> Any:
response: LLMResponseTypes | None = None,
) -> object:
"""
Execute a single guardrail's hook.
@ -1008,8 +1011,8 @@ class ProxyLogging:
data: dict,
user_api_key_dict: UserAPIKeyAuth | None,
call_type: CallTypesLiteral,
response: Any | None = None,
) -> Any:
response: LLMResponseTypes | None = None,
) -> object:
"""
Execute a guardrail using the router's load balancing.
@ -1144,8 +1147,8 @@ class ProxyLogging:
self,
data: dict,
litellm_logging_obj: Any,
prompt_id: Any,
prompt_version: Any,
prompt_id: str,
prompt_version: int | None,
call_type: CallTypesLiteral,
) -> None:
"""Process prompt template if applicable."""
@ -1366,8 +1369,8 @@ class ProxyLogging:
return None
litellm_logging_obj: Final = cast(Optional["LiteLLMLoggingObj"], data.get("litellm_logging_obj", None))
prompt_id: Final = data.get("prompt_id", None)
prompt_version: Final = data.get("prompt_version", None)
prompt_id: Final[str | None] = data.get("prompt_id", None)
prompt_version: Final[int | None] = data.get("prompt_version", None)
## PROMPT TEMPLATE CHECK ##
@ -1448,7 +1451,7 @@ class ProxyLogging:
if call_type == "call_mcp_tool" and user_api_key_dict is None:
continue
response = await _callback.async_pre_call_hook(
response: Exception | str | Mapping[str, object] | None = await _callback.async_pre_call_hook(
user_api_key_dict=user_api_key_dict,
cache=self.call_details["user_api_key_cache"],
data=data,
@ -1616,7 +1619,7 @@ class ProxyLogging:
break
@staticmethod
async def _run_guardrail_with_metrics(callback: Any, coro: Awaitable[Any], hook_type: str) -> Any:
async def _run_guardrail_with_metrics(callback: object, coro: Awaitable[_T], hook_type: str) -> _T:
"""
Await `coro`, recording its latency and status to the
`litellm_guardrail_latency_seconds` metric under `hook_type`, and
@ -1648,8 +1651,8 @@ class ProxyLogging:
@staticmethod
async def _wrap_streaming_iterator_with_enrichment(
callback: Any, gen: AsyncGenerator[Any, None]
) -> AsyncGenerator[Any, None]:
callback: object, gen: AsyncGenerator[_T, None]
) -> AsyncGenerator[_T, None]:
"""
Yield from `gen`; if iteration raises an HTTPException with dict detail,
enrich the detail with the originating callback's `guardrail_name` and
@ -1694,11 +1697,11 @@ class ProxyLogging:
has_guardrail = False
has_pre_call_override = False
iterator_overrides: Final[list[tuple[Any, str]]] = [] # (callback, kind)
resolved_callbacks: Final[list[Any]] = []
resolved_callbacks: Final[list[CustomLogger]] = []
for callback in callbacks:
if isinstance(callback, str):
resolved: Any = litellm.litellm_core_utils.litellm_logging.get_custom_logger_compatible_class(
resolved = litellm.litellm_core_utils.litellm_logging.get_custom_logger_compatible_class(
cast(_custom_logger_compatible_callbacks_literal, callback)
)
else:
@ -2543,7 +2546,7 @@ class ProxyLogging:
self,
data: dict,
user_api_key_dict: UserAPIKeyAuth,
response: Any,
response: object,
request_headers: dict[str, str] | None = None,
) -> dict[str, str]:
"""
@ -2599,7 +2602,7 @@ class ProxyLogging:
return merged_headers
@staticmethod
def _build_litellm_call_info(data: dict, response: Any) -> dict[str, Any]:
def _build_litellm_call_info(data: dict, response: object) -> dict[str, object]:
"""
Build a normalized dict of routing metadata from response._hidden_params
and data, abstracting away the metadata vs litellm_metadata split.
@ -2876,7 +2879,7 @@ _DEPRECATED_KEY_CACHE_TTL_SECONDS: Final = 60
async def _lookup_deprecated_key(
db: Any,
db: PrismaWrapper | RoutingPrismaWrapper,
hashed_token: str,
) -> str | None:
"""
@ -2944,7 +2947,7 @@ def _config_cache_key(param_name: str) -> str:
return f"litellm_config:param:{param_name}"
def _pack_config_row(row: Any) -> dict[str, Any]:
def _pack_config_row(row: Any) -> dict[str, object]:
return {"param_name": row.param_name, "param_value": row.param_value}
@ -2956,7 +2959,7 @@ def _unpack_config_row(cached: Any) -> _ConfigRow | None:
return None
async def get_config_param(prisma_client: Any, param_name: str) -> Any | None:
async def get_config_param(prisma_client: "PrismaClient", param_name: str) -> Any | None:
"""Cached read of a LiteLLM_Config row; returns row, _ConfigRow shim, or None."""
cache_key: Final = _config_cache_key(param_name)
cached: Final = await litellm_config_cache.async_get_cache(cache_key)
@ -2964,7 +2967,7 @@ async def get_config_param(prisma_client: Any, param_name: str) -> Any | None:
return _unpack_config_row(cached)
row: Final = await prisma_client.get_generic_data(key="param_name", value=param_name, table_name="config")
cache_value: Final[Any] = _pack_config_row(row) if row is not None else _CONFIG_CACHE_MISS
cache_value: Final[Mapping[str, object] | str] = _pack_config_row(row) if row is not None else _CONFIG_CACHE_MISS
await litellm_config_cache.async_set_cache(cache_key, cache_value, ttl=LITELLM_CONFIG_CACHE_TTL_SECONDS)
return row
@ -2979,7 +2982,7 @@ async def invalidate_config_param(param_name: str) -> None:
await publish_config_param_change(param_name)
async def prefetch_config_params(prisma_client: Any, param_names: list[str]) -> None:
async def prefetch_config_params(prisma_client: "PrismaClient | None", param_names: list[str]) -> None:
"""Batch-load LiteLLM_Config rows into the cache with one find_many."""
if not param_names:
return
@ -2994,7 +2997,7 @@ async def prefetch_config_params(prisma_client: Any, param_names: list[str]) ->
by_name: Final = {row.param_name: row for row in rows}
for name in param_names:
row = by_name.get(name)
cache_value: Any = _pack_config_row(row) if row is not None else _CONFIG_CACHE_MISS
cache_value: Mapping[str, object] | str = _pack_config_row(row) if row is not None else _CONFIG_CACHE_MISS
await litellm_config_cache.async_set_cache(
_config_cache_key(name), cache_value, ttl=LITELLM_CONFIG_CACHE_TTL_SECONDS
)
@ -3021,7 +3024,7 @@ class PrismaClient:
self,
database_url: str,
proxy_logging_obj: ProxyLogging,
http_client: Any | None = None,
http_client: "HttpConfig | None" = None,
):
## init logging object
self.proxy_logging_obj = proxy_logging_obj
@ -3313,7 +3316,7 @@ class PrismaClient:
async def get_generic_data(
self,
key: str,
value: Any,
value: object,
table_name: Literal["users", "keys", "config", "spend"],
):
"""
@ -5498,7 +5501,7 @@ class ProxyUpdateSpend:
prisma_client: PrismaClient,
db_writer_client: AsyncHTTPHandler | None,
proxy_logging_obj: ProxyLogging,
logs_to_process: list[dict[str, Any]] | None = None,
logs_to_process: list[dict[str, object]] | None = None,
):
BATCH_SIZE: Final = 1000 # Preferred size of each batch to write to the database
MAX_LOGS_PER_INTERVAL: Final = 10000 # Maximum number of logs to flush in a single interval
@ -6729,7 +6732,7 @@ def model_dump_with_preserved_fields(
obj: Any,
preserve_fields: list[str] | None = None,
exclude_unset: bool = True,
) -> dict[str, Any]:
) -> dict[str, object]:
"""
Serialize a Pydantic model to a dictionary while preserving specific fields
even if they are None.

121
litellm/responses/main.py
View file

@ -1,6 +1,6 @@
import asyncio
import contextvars
from collections.abc import Coroutine, Iterable
from collections.abc import Coroutine, Iterable, Mapping
from functools import partial
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast
@ -53,6 +53,7 @@ from litellm.utils import (
)
if TYPE_CHECKING:
from fastapi import WebSocket
from mcp.types import Tool as MCPTool
else:
MCPTool = Any
@ -66,7 +67,7 @@ litellm_completion_transformation_handler: Final = LiteLLMCompletionTransformati
#################################################
def _has_file_search_tool(tools: Any | None) -> bool:
def _has_file_search_tool(tools: Iterable[Mapping[str, object]] | None) -> bool:
"""Return True if any tool in the list has type 'file_search'."""
if not tools:
return False
@ -132,7 +133,7 @@ async def aresponses_api_with_mcp(
instructions: str | None = None,
max_output_tokens: int | None = None,
prompt: PromptObject | None = None,
metadata: dict[str, Any] | None = None,
metadata: dict[str, object] | None = None,
parallel_tool_calls: bool | None = None,
previous_response_id: str | None = None,
reasoning: Reasoning | None = None,
@ -148,9 +149,9 @@ async def aresponses_api_with_mcp(
user: str | None = None,
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: dict[str, Any] | None = None,
extra_query: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
extra_headers: dict[str, object] | None = None,
extra_query: dict[str, object] | None = None,
extra_body: dict[str, object] | None = None,
timeout: float | httpx.Timeout | None = None,
# LiteLLM specific params,
custom_llm_provider: str | None = None,
@ -397,7 +398,7 @@ async def aresponses(
instructions: str | None = None,
max_output_tokens: int | None = None,
prompt: PromptObject | None = None,
metadata: dict[str, Any] | None = None,
metadata: dict[str, object] | None = None,
parallel_tool_calls: bool | None = None,
previous_response_id: str | None = None,
reasoning: Reasoning | None = None,
@ -416,9 +417,9 @@ async def aresponses(
safety_identifier: str | None = None,
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: dict[str, Any] | None = None,
extra_query: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
extra_headers: dict[str, object] | None = None,
extra_query: dict[str, object] | None = None,
extra_body: dict[str, object] | None = None,
timeout: float | httpx.Timeout | None = None,
# LiteLLM specific params,
custom_llm_provider: str | None = None,
@ -564,9 +565,9 @@ def _apply_prompt_management_to_responses_call(
custom_llm_provider: str | None,
litellm_logging_obj: LiteLLMLoggingObj | None,
kwargs: dict[str, Any],
local_vars: dict[str, Any],
local_vars: dict[str, object],
) -> tuple[str | ResponseInputParam, str, str | None]:
async_merged: Final = kwargs.pop("_async_prompt_merged_params", None)
async_merged: Final[Mapping[str, object] | None] = kwargs.pop("_async_prompt_merged_params", None)
if async_merged is not None:
for key, value in async_merged.items():
local_vars[key] = value
@ -633,7 +634,7 @@ def _normalize_openai_chat_completions_responses_model(model: str) -> tuple[str,
return f"openai/{remainder}", True
def _pop_use_chat_completions_api_kw(kwargs: dict[str, Any]) -> bool:
def _pop_use_chat_completions_api_kw(kwargs: dict[str, object]) -> bool:
"""Pop use_chat_completions_api; True when the chat-completions bridge is requested."""
use_cc: Final = kwargs.pop("use_chat_completions_api", None)
return bool(use_cc)
@ -643,7 +644,7 @@ def _resolve_model_provider_for_responses(
model: str,
custom_llm_provider: str | None,
litellm_params: GenericLiteLLMParams,
local_vars: dict[str, Any],
local_vars: dict[str, object],
) -> tuple[str, str | None]:
if custom_llm_provider is not None and not litellm_params.custom_llm_provider:
litellm_params.custom_llm_provider = custom_llm_provider
@ -668,7 +669,7 @@ def _apply_managed_file_id_mapping(
input: str | ResponseInputParam,
tools: Iterable[ToolParam] | None,
kwargs: dict[str, Any],
local_vars: dict[str, Any],
local_vars: dict[str, object],
) -> tuple[str | ResponseInputParam, Iterable[ToolParam] | None]:
model_file_id_mapping: Final = kwargs.get("model_file_id_mapping")
model_info_id = kwargs.get("model_info", {}).get("id") if isinstance(kwargs.get("model_info"), dict) else None
@ -706,7 +707,7 @@ def _responses_try_dispatch_mcp_gateway(
instructions: str | None,
max_output_tokens: int | None,
prompt: PromptObject | None,
metadata: dict[str, Any] | None,
metadata: dict[str, object] | None,
parallel_tool_calls: bool | None,
previous_response_id: str | None,
reasoning: Reasoning | None,
@ -719,9 +720,9 @@ def _responses_try_dispatch_mcp_gateway(
top_p: float | None,
truncation: Literal["auto", "disabled"] | None,
user: str | None,
extra_headers: dict[str, Any] | None,
extra_query: dict[str, Any] | None,
extra_body: dict[str, Any] | None,
extra_headers: dict[str, object] | None,
extra_query: dict[str, object] | None,
extra_body: dict[str, object] | None,
timeout: float | httpx.Timeout | None,
custom_llm_provider: str | None,
kwargs: dict[str, Any],
@ -778,7 +779,7 @@ def _responses_try_dispatch_emulated_file_search(
instructions: str | None,
max_output_tokens: int | None,
prompt: PromptObject | None,
metadata: dict[str, Any] | None,
metadata: dict[str, object] | None,
parallel_tool_calls: bool | None,
previous_response_id: str | None,
reasoning: Reasoning | None,
@ -795,14 +796,14 @@ def _responses_try_dispatch_emulated_file_search(
safety_identifier: str | None,
text_format: type[BaseModel] | dict | None,
allowed_openai_params: list[str] | None,
extra_headers: dict[str, Any] | None,
extra_query: dict[str, Any] | None,
extra_body: dict[str, Any] | None,
extra_headers: dict[str, object] | None,
extra_query: dict[str, object] | None,
extra_body: dict[str, object] | None,
timeout: float | httpx.Timeout | None,
custom_llm_provider: str | None,
kwargs: dict[str, Any],
_is_async: bool,
) -> Any | None:
) -> ResponsesAPIResponse | Coroutine[object, object, ResponsesAPIResponse] | None:
"""Return a response when emulated file_search handles the call; otherwise None."""
if not _has_file_search_tool(tools) or not (
responses_api_provider_config is None
@ -864,7 +865,7 @@ def responses(
instructions: str | None = None,
max_output_tokens: int | None = None,
prompt: PromptObject | None = None,
metadata: dict[str, Any] | None = None,
metadata: dict[str, object] | None = None,
parallel_tool_calls: bool | None = None,
previous_response_id: str | None = None,
reasoning: Reasoning | None = None,
@ -883,9 +884,9 @@ def responses(
safety_identifier: str | None = None,
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: dict[str, Any] | None = None,
extra_query: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
extra_headers: dict[str, object] | None = None,
extra_query: dict[str, object] | None = None,
extra_body: dict[str, object] | None = None,
timeout: float | httpx.Timeout | None = None,
# LiteLLM specific params,
allowed_openai_params: list[str] | None = None,
@ -1148,9 +1149,9 @@ async def adelete_responses(
response_id: str,
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: dict[str, Any] | None = None,
extra_query: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
extra_headers: dict[str, object] | None = None,
extra_query: dict[str, object] | None = None,
extra_body: dict[str, object] | None = None,
timeout: float | httpx.Timeout | None = None,
# LiteLLM specific params,
custom_llm_provider: str | None = None,
@ -1209,14 +1210,14 @@ def delete_responses(
response_id: str,
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: dict[str, Any] | None = None,
extra_query: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
extra_headers: dict[str, object] | None = None,
extra_query: dict[str, object] | None = None,
extra_body: dict[str, object] | None = None,
timeout: float | httpx.Timeout | None = None,
# LiteLLM specific params,
custom_llm_provider: str | None = None,
**kwargs,
) -> DeleteResponseResult | Coroutine[Any, Any, DeleteResponseResult]:
) -> DeleteResponseResult | Coroutine[object, object, DeleteResponseResult]:
"""
Synchronous version of the DELETE Responses API
@ -1299,9 +1300,9 @@ async def aget_responses(
response_id: str,
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: dict[str, Any] | None = None,
extra_query: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
extra_headers: dict[str, object] | None = None,
extra_query: dict[str, object] | None = None,
extra_body: dict[str, object] | None = None,
timeout: float | httpx.Timeout | None = None,
# LiteLLM specific params,
custom_llm_provider: str | None = None,
@ -1374,14 +1375,14 @@ def get_responses(
response_id: str,
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: dict[str, Any] | None = None,
extra_query: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
extra_headers: dict[str, object] | None = None,
extra_query: dict[str, object] | None = None,
extra_body: dict[str, object] | None = None,
timeout: float | httpx.Timeout | None = None,
# LiteLLM specific params,
custom_llm_provider: str | None = None,
**kwargs,
) -> ResponsesAPIResponse | Coroutine[Any, Any, ResponsesAPIResponse]:
) -> ResponsesAPIResponse | Coroutine[object, object, ResponsesAPIResponse]:
"""
Fetch a response by its ID.
@ -1481,7 +1482,7 @@ async def alist_input_items(
include: list[str] | None = None,
limit: int = 20,
order: Literal["asc", "desc"] = "desc",
extra_headers: dict[str, Any] | None = None,
extra_headers: dict[str, object] | None = None,
timeout: float | httpx.Timeout | None = None,
custom_llm_provider: str | None = None,
**kwargs,
@ -1537,11 +1538,11 @@ def list_input_items(
include: list[str] | None = None,
limit: int = 20,
order: Literal["asc", "desc"] = "desc",
extra_headers: dict[str, Any] | None = None,
extra_headers: dict[str, object] | None = None,
timeout: float | httpx.Timeout | None = None,
custom_llm_provider: str | None = None,
**kwargs,
) -> dict | Coroutine[Any, Any, dict]:
) -> dict | Coroutine[object, object, dict]:
"""List input items for a response"""
local_vars: Final = locals()
try:
@ -1612,9 +1613,9 @@ async def acancel_responses(
response_id: str,
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: dict[str, Any] | None = None,
extra_query: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
extra_headers: dict[str, object] | None = None,
extra_query: dict[str, object] | None = None,
extra_body: dict[str, object] | None = None,
timeout: float | httpx.Timeout | None = None,
# LiteLLM specific params,
custom_llm_provider: str | None = None,
@ -1673,14 +1674,14 @@ def cancel_responses(
response_id: str,
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: dict[str, Any] | None = None,
extra_query: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
extra_headers: dict[str, object] | None = None,
extra_query: dict[str, object] | None = None,
extra_body: dict[str, object] | None = None,
timeout: float | httpx.Timeout | None = None,
# LiteLLM specific params,
custom_llm_provider: str | None = None,
**kwargs,
) -> ResponsesAPIResponse | Coroutine[Any, Any, ResponsesAPIResponse]:
) -> ResponsesAPIResponse | Coroutine[object, object, ResponsesAPIResponse]:
"""
Synchronous version of the POST Responses API
@ -1766,9 +1767,9 @@ async def acompact_responses(
previous_response_id: str | None = None,
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: dict[str, Any] | None = None,
extra_query: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
extra_headers: dict[str, object] | None = None,
extra_query: dict[str, object] | None = None,
extra_body: dict[str, object] | None = None,
timeout: float | httpx.Timeout | None = None,
# LiteLLM specific params,
custom_llm_provider: str | None = None,
@ -1844,14 +1845,14 @@ def compact_responses(
previous_response_id: str | None = None,
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: dict[str, Any] | None = None,
extra_query: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
extra_headers: dict[str, object] | None = None,
extra_query: dict[str, object] | None = None,
extra_body: dict[str, object] | None = None,
timeout: float | httpx.Timeout | None = None,
# LiteLLM specific params,
custom_llm_provider: str | None = None,
**kwargs,
) -> ResponsesAPIResponse | Coroutine[Any, Any, ResponsesAPIResponse]:
) -> ResponsesAPIResponse | Coroutine[object, object, ResponsesAPIResponse]:
"""
Synchronous version of the POST Compact Responses API
@ -1975,7 +1976,7 @@ def _build_litellm_metadata_for_ws(kwargs: dict) -> dict:
@client
async def _aresponses_websocket(
model: str,
websocket: Any,
websocket: "WebSocket",
api_base: str | None = None,
api_key: str | None = None,
timeout: float | None = None,

6
litellm/responses/mcp/chat_completions_handler.py
View file

@ -233,7 +233,7 @@ async def acompletion_with_mcp(
self.follow_up_iterator = None
self.follow_up_exhausted = False
async def __aiter__(self):
def __aiter__(self):
return self
def _add_mcp_list_tools_to_chunk(self, chunk: ModelResponseStream) -> ModelResponseStream:
@ -497,12 +497,12 @@ async def acompletion_with_mcp(
# Create a wrapper class that delegates to our custom iterator
# We'll use a simple approach: just replace the __aiter__ method
class MCPStreamWrapper(CustomStreamWrapper):
def __init__(self, original_wrapper, custom_iterator):
def __init__(self, original_wrapper: CustomStreamWrapper, custom_iterator: MCPStreamingIterator):
# Initialize with the same parameters as original wrapper
super().__init__(
completion_stream=None,
model=getattr(original_wrapper, "model", "unknown"),
logging_obj=getattr(original_wrapper, "logging_obj", None),
logging_obj=original_wrapper.logging_obj,
custom_llm_provider=getattr(original_wrapper, "custom_llm_provider", None),
stream_options=getattr(original_wrapper, "stream_options", None),
make_call=getattr(original_wrapper, "make_call", None),

32
litellm/responses/streaming_iterator.py
View file

@ -9,7 +9,7 @@ from collections.abc import Awaitable, Callable, Mapping
from datetime import datetime
from functools import lru_cache
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, runtime_checkable
import httpx
from openai._streaming import SSEDecoder
@ -313,8 +313,10 @@ class BaseResponsesAPIStreamingIterator:
if encrypted_content and isinstance(encrypted_content, str):
model_id: Final = _model_id_from_metadata(self.litellm_metadata)
if model_id:
wrapped_content = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
encrypted_content, model_id
wrapped_content: Final = (
ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
encrypted_content, model_id
)
)
setattr(item, "encrypted_content", wrapped_content)
@ -336,7 +338,9 @@ class BaseResponsesAPIStreamingIterator:
usage_obj: Final[ResponseAPIUsage | None] = getattr(response_obj, "usage", None)
if usage_obj is not None:
try:
cost: float | None = self.logging_obj._response_cost_calculator(result=response_obj)
cost: Final[float | None] = self.logging_obj._response_cost_calculator(
result=response_obj
)
if cost is not None:
setattr(usage_obj, "cost", cost)
except Exception:
@ -1029,6 +1033,16 @@ class CachedResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
return evt
@runtime_checkable
class _HasModelDump(Protocol):
def model_dump(self, *, exclude_none: bool = ...) -> Mapping[str, object]: ...
@runtime_checkable
class _HasModelDumpJson(Protocol):
def model_dump_json(self, *, exclude_none: bool = ...) -> str: ...
def _dump_response_object(obj: Any) -> dict[str, Any]:
if hasattr(obj, "model_dump"):
return obj.model_dump()
@ -1358,7 +1372,7 @@ class ResponsesWebSocketStreaming:
# response.create frame to prevent deployment-substitution attacks.
self.authorized_model: str | None = authorized_model
def _should_store_event(self, event_obj: dict[str, object]) -> bool:
def _should_store_event(self, event_obj: Mapping[str, object]) -> bool:
return event_obj.get("type") in RESPONSES_WS_LOGGED_EVENT_TYPES
def _store_event(self, event: str | bytes | dict[str, object]) -> None:
@ -1636,7 +1650,7 @@ class ResponsesWebSocketStreaming:
metadata: Final = self.request_data.get("metadata")
raw_pii_tokens: Final = metadata.get("pii_tokens") if _is_json_object(metadata) else None
pii_tokens: Final[dict[str, str]] = raw_pii_tokens if _is_str_mapping(raw_pii_tokens) else {}
pii_tokens: Final[Mapping[str, str]] = raw_pii_tokens if _is_str_mapping(raw_pii_tokens) else {}
if not pii_tokens:
return response_str
@ -1883,11 +1897,11 @@ class ManagedResponsesWebSocketHandler:
def _serialize_chunk(chunk: Any) -> str | None:
"""Serialize a streaming chunk to a JSON string for WebSocket transmission."""
try:
if hasattr(chunk, "model_dump_json"):
if isinstance(chunk, _HasModelDumpJson):
return chunk.model_dump_json(exclude_none=True)
if hasattr(chunk, "model_dump"):
if isinstance(chunk, _HasModelDump):
return json.dumps(chunk.model_dump(exclude_none=True), default=str)
if isinstance(chunk, dict):
if _is_json_object(chunk):
return json.dumps(chunk, default=str)
return json.dumps(str(chunk))
except Exception as exc:

71
litellm/router.py
View file

@ -23,7 +23,7 @@ from collections import defaultdict
from collections.abc import AsyncGenerator, Callable, Generator, Mapping, Sequence
from functools import lru_cache
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypeVar, Union, cast
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypeAlias, TypeVar, Union, cast
import anyio
import httpx
@ -258,6 +258,14 @@ else:
QualityRouter = Any
PreRoutingHookResponse = Any
RouterStrategySelector: TypeAlias = (
LeastBusyLoggingHandler
| LowestCostLoggingHandler
| LowestLatencyLoggingHandler
| LowestTPMLoggingHandler
| LowestTPMLoggingHandler_v2
)
def _cost_value_as_float(value: str | float | None) -> float | None:
if value is None:
@ -730,7 +738,7 @@ class Router:
routing_strategy_args=routing_strategy_args,
)
self._init_routing_groups(self._routing_groups_input)
self._override_selectors: dict[str, Any] = {}
self._override_selectors: dict[str, RouterStrategySelector | None] = {}
self._override_selectors_lock = threading.Lock()
self.access_groups = None
## USAGE TRACKING ##
@ -923,13 +931,13 @@ class Router:
strategy: RoutingStrategy | str,
routing_strategy_args: dict,
register_callbacks: bool = True,
) -> Any | None:
) -> RouterStrategySelector | None:
"""
Constructs a strategy selector for a given strategy.
Returns None for `simple-shuffle` (no selector needed) and unknown
strategies.
"""
selector: Any | None = None
selector: RouterStrategySelector | None = None
match self._normalize_strategy(strategy):
case RoutingStrategy.LEAST_BUSY.value:
selector = LeastBusyLoggingHandler(router_cache=self.cache)
@ -966,7 +974,7 @@ class Router:
return selector
def _unregister_router_selectors(self, selectors: list[Any]) -> None:
def _unregister_router_selectors(self, selectors: Sequence[object]) -> None:
"""
Drop router-owned strategy selectors from litellm's global callback
lists by identity. Used before re-init (`routing_strategy_init` /
@ -1023,13 +1031,14 @@ class Router:
`"default"` group, whose selectors are the `self.<strategy>_logger`
attributes set up in `routing_strategy_init`.
"""
self._unregister_router_selectors(
[sel for selectors in getattr(self, "_group_selectors", {}).values() for sel in selectors.values()]
group_selectors: Final[Mapping[str, Mapping[str, RouterStrategySelector]]] = getattr(
self, "_group_selectors", {}
)
self._unregister_router_selectors([sel for selectors in group_selectors.values() for sel in selectors.values()])
self._routing_groups: dict[str, RoutingGroup] = {}
self._model_to_group: dict[str, str] = {}
self._group_selectors: dict[str, dict[str, Any]] = {}
self._group_selectors: dict[str, dict[str, RouterStrategySelector]] = {}
if not groups_input:
return
@ -1107,7 +1116,7 @@ class Router:
return None
return strategy
def _get_override_strategy_selector(self, strategy: str) -> Any | None:
def _get_override_strategy_selector(self, strategy: str) -> RouterStrategySelector | None:
"""
Returns the selector for a per-request strategy override.
@ -1128,7 +1137,9 @@ class Router:
)
return self._override_selectors[strategy]
def _get_routing_context(self, model: str, request_kwargs: dict | None = None) -> tuple[str | None, Any | None]:
def _get_routing_context(
self, model: str, request_kwargs: dict | None = None
) -> tuple[str | None, RouterStrategySelector | None]:
"""
Resolves the routing strategy and selector to use for the given model.
@ -1951,7 +1962,7 @@ class Router:
return silent_kwargs
def _silent_experiment_completion(self, silent_model: str, messages: list[Any], **kwargs):
def _silent_experiment_completion(self, silent_model: str, messages: Sequence[Mapping[str, str]], **kwargs):
"""
Run a silent experiment in the background (thread).
"""
@ -2301,7 +2312,7 @@ class Router:
# in __init__ rather than declaring it as a class field, so
# static narrowing doesn't expose it. Mirror the sync path
# (_completion_streaming_iterator) and pull via getattr.
chat: Final = getattr(built, "usage", None) if built is not None else None
chat: Final[object | None] = getattr(built, "usage", None) if built is not None else None
if chat is not None:
# getattr-with-default because the test path may
# substitute a SimpleNamespace lacking some fields;
@ -2395,7 +2406,7 @@ class Router:
# ResponseOutputMessageParam, ...) — annotating as List[Dict[str, Any]]
# rejects the list() spread of input_val. We cast the combined list to
# ResponseInputParam at the return.
base: list[Any]
base: list[object]
if isinstance(input_val, str):
base = [
{
@ -2408,7 +2419,7 @@ class Router:
base = list(input_val)
else:
base = []
continuation: Final[list[Any]] = [
continuation: Final[list[object]] = [
{
"type": "message",
"role": "developer",
@ -2785,7 +2796,7 @@ class Router:
return SyncFallbackStreamWrapper(stream_with_fallbacks())
async def _silent_experiment_acompletion(self, silent_model: str, messages: list[Any], **kwargs):
async def _silent_experiment_acompletion(self, silent_model: str, messages: Sequence[Mapping[str, str]], **kwargs):
"""
Run a silent experiment in the background.
"""
@ -3041,7 +3052,7 @@ class Router:
pass
def _stamp_failed_deployment_id_with_effective_model_info(
self, exception: Exception, deployment: Mapping[str, Any], kwargs: Mapping[str, Any]
self, exception: Exception, deployment: Mapping[str, object], kwargs: Mapping[str, object]
) -> None:
# A client-side-credential call gets a dynamic deployment id generated inside
# _update_kwargs_with_deployment and stamped into kwargs["model_info"]; stamping
@ -3564,8 +3575,8 @@ class Router:
model: str,
priority: int,
original_function: Callable,
args: tuple[Any, ...],
kwargs: dict[str, Any],
args: tuple[object, ...],
kwargs: dict[str, object],
):
parent_otel_span: Final = _get_parent_otel_span_from_kwargs(kwargs)
### FLOW ITEM ###
@ -4653,7 +4664,7 @@ class Router:
# fallback to the original reference for any non-picklable value.
# The original_generic_function is preserved so the per-attempt
# helper knows which underlying API to call on fallback.
fallback_kwargs: Final[dict[str, Any]] = kwargs.copy()
fallback_kwargs: Final[dict[str, object]] = kwargs.copy()
if isinstance(fallback_kwargs.get("litellm_metadata"), dict):
fallback_kwargs["litellm_metadata"] = safe_deep_copy(fallback_kwargs["litellm_metadata"])
if isinstance(fallback_kwargs.get("metadata"), dict):
@ -5700,7 +5711,7 @@ class Router:
def sync_wrapper(
custom_llm_provider: str | None = None,
client: Any | None = None,
client: object | None = None,
**kwargs,
):
return self._generic_api_call_with_fallbacks(original_function=original_function, **kwargs)
@ -5716,7 +5727,7 @@ class Router:
def vector_store_sync_wrapper(
custom_llm_provider: str | None = None,
client: Any | None = None,
client: object | None = None,
**kwargs,
):
if custom_llm_provider and "custom_llm_provider" not in kwargs:
@ -5738,7 +5749,7 @@ class Router:
def vector_store_file_sync_wrapper(
custom_llm_provider: str | None = None,
client: Any | None = None,
client: object | None = None,
**kwargs,
):
return original_function(
@ -5759,7 +5770,7 @@ class Router:
def managed_agents_sync_wrapper(
custom_llm_provider: str | None = None,
client: Any | None = None,
client: object | None = None,
**kwargs,
):
if custom_llm_provider and "custom_llm_provider" not in kwargs:
@ -7144,7 +7155,9 @@ class Router:
except Exception as e:
raise e
async def async_deployment_callback_on_failure(self, kwargs, completion_response: Any | None, start_time, end_time):
async def async_deployment_callback_on_failure(
self, kwargs, completion_response: object | None, start_time, end_time
):
"""
Update RPM usage for a deployment
"""
@ -7845,7 +7858,7 @@ class Router:
continue
if self._has_registered_strategy(self.adaptive_routers, model_name, tagged.tags):
continue
adaptive_router = complexity_router._ensure_adaptive_router()
adaptive_router: AdaptiveRouter | None = complexity_router._ensure_adaptive_router()
if adaptive_router is not None:
self.adaptive_routers[model_name] = [
*self.adaptive_routers.get(model_name, []),
@ -8133,7 +8146,7 @@ class Router:
self.provider_default_deployment_ids.append(deployment.model_info.id)
_team_id: Final = deployment.model_info.get("team_id")
_team_public_model_name: Final = deployment.model_info.get("team_public_model_name")
_team_public_model_name: Final[str | None] = deployment.model_info.get("team_public_model_name")
if _team_id is not None and _team_public_model_name is not None and "*" in _team_public_model_name:
if _team_id not in self.team_pattern_routers:
self.team_pattern_routers[_team_id] = PatternMatchRouter()
@ -9487,7 +9500,7 @@ class Router:
async def set_response_headers(
self,
response: Any,
response: object,
model_group: str | None = None,
request_kwargs: dict | None = None,
) -> Any:
@ -11322,7 +11335,7 @@ class Router:
@staticmethod
def _redact_prompt_text_if_needed(
request_kwargs: Mapping[str, Any],
request_kwargs: Mapping[str, object],
routing_decision: StandardLoggingRoutingDecision,
) -> StandardLoggingRoutingDecision:
"""Drop verbatim prompt text from the record when message logging is redacted.
@ -11684,7 +11697,7 @@ class Router:
flag. Used by credential-lookup helpers so passthrough file / batch endpoints
cannot bypass the pause by resolving credentials directly.
"""
model_info: Final = getattr(deployment, "model_info", None)
model_info: Final[object | None] = getattr(deployment, "model_info", None)
if model_info is None:
return False
return getattr(model_info, "blocked", None) is True

2
litellm/types/completion.py
View file

@ -217,7 +217,7 @@ class _CompletionDispatchContext:
headers: dict
hf_model_name: str | None
kwargs: dict
litellm_params: dict
litellm_params: dict[str, object]
logger_fn: Callable | None
logging: LiteLLMLoggingObj
max_retries: int | None

123
litellm/utils.py
View file

@ -234,9 +234,11 @@ except (ImportError, AttributeError, TypeError):
# Convert to str (if necessary)
claude_json_str = json.dumps(json_data)
import importlib.metadata
from collections.abc import Callable, Iterable, Mapping
from collections.abc import Callable, Iterable, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Union, cast, get_args
from litellm import utils as litellm_utils
# These are lazy loaded via __getattr__
from litellm.llms.base_llm.base_utils import (
BaseLLMModelInfo,
@ -263,6 +265,7 @@ if TYPE_CHECKING:
map_finish_reason,
process_response_headers,
)
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
from litellm.litellm_core_utils.dot_notation_indexing import (
delete_nested_value,
is_nested_path,
@ -351,6 +354,24 @@ if TYPE_CHECKING:
)
from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
from litellm.llms.bedrock.common_utils import BedrockModelInfo
from litellm.llms.bedrock.embed.amazon_nova_transformation import (
AmazonNovaEmbeddingConfig,
)
from litellm.llms.bedrock.embed.amazon_titan_g1_transformation import (
AmazonTitanG1Config,
)
from litellm.llms.bedrock.embed.amazon_titan_multimodal_transformation import (
AmazonTitanMultimodalEmbeddingG1Config,
)
from litellm.llms.bedrock.embed.amazon_titan_v2_transformation import (
AmazonTitanV2Config,
)
from litellm.llms.bedrock.embed.cohere_transformation import (
BedrockCohereEmbeddingConfig,
)
from litellm.llms.bedrock.embed.twelvelabs_marengo_transformation import (
TwelveLabsMarengoEmbeddingConfig,
)
from litellm.llms.cohere.common_utils import CohereModelInfo
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.mistral.ocr.transformation import MistralOCRConfig
@ -574,7 +595,7 @@ def get_request_guardrails(kwargs: dict[str, Any]) -> list[str]:
return applied_guardrails
def get_applied_guardrails(kwargs: dict[str, Any]) -> list[str]:
def get_applied_guardrails(kwargs: dict[str, object]) -> list[str]:
"""
- Add 'default_on' guardrails to the list
- Add request guardrails to the list
@ -601,7 +622,7 @@ def load_credentials_from_list(kwargs: dict):
credential_name: Final = kwargs.get("litellm_credential_name")
if credential_name and litellm.credential_list:
credential_accessor: Final = CredentialAccessor.get_credential_values(credential_name)
credential_accessor: Final[Mapping[str, object]] = CredentialAccessor.get_credential_values(credential_name)
for key, value in credential_accessor.items():
if key not in kwargs:
kwargs[key] = value
@ -789,7 +810,7 @@ def function_setup(
function_id: Final[str | None] = kwargs["id"] if "id" in kwargs else None
## LAZY LOAD COROUTINE CHECKER ##
get_coroutine_checker_fn: Final = getattr(sys.modules[__name__], "get_coroutine_checker")
get_coroutine_checker_fn: Final = litellm_utils.get_coroutine_checker
coroutine_checker: Final = get_coroutine_checker_fn()
## DYNAMIC CALLBACKS ##
@ -925,7 +946,7 @@ def function_setup(
elif kwargs.get("messages", None):
messages = kwargs["messages"]
### PRE-CALL RULES ###
Rules: Final = getattr(sys.modules[__name__], "Rules")
Rules: Final = litellm_utils.Rules
if (
Rules.has_pre_call_rules()
and isinstance(messages, list)
@ -1033,7 +1054,7 @@ def function_setup(
)
contents_param: Final = args[1] if len(args) > 1 else kwargs.get("contents")
model_param: Final = args[0] if len(args) > 0 else kwargs.get("model", "")
model_param: Final[str] = args[0] if len(args) > 0 else kwargs.get("model", "")
if contents_param:
adapter: Final = GoogleGenAIAdapter()
@ -1078,7 +1099,7 @@ def function_setup(
)
## check if metadata is passed in
litellm_params: Final[dict[str, Any]] = {"api_base": ""}
litellm_params: Final[dict[str, object]] = {"api_base": ""}
if "metadata" in kwargs:
litellm_params["metadata"] = kwargs["metadata"]
if "litellm_metadata" in kwargs and isinstance(kwargs["litellm_metadata"], dict):
@ -1154,9 +1175,11 @@ def _get_wrapper_num_retries(kwargs: dict[str, Any], exception: Exception) -> tu
if num_retries is None:
num_retries = litellm.num_retries
if kwargs.get("retry_policy", None):
get_num_retries_from_retry_policy: Final = getattr(sys.modules[__name__], "get_num_retries_from_retry_policy")
reset_retry_policy: Final = getattr(sys.modules[__name__], "reset_retry_policy")
retry_policy_num_retries: Final = get_num_retries_from_retry_policy(
get_num_retries_from_retry_policy: Final[Callable[..., int | None]] = getattr(
sys.modules[__name__], "get_num_retries_from_retry_policy"
)
reset_retry_policy: Final = litellm_utils.reset_retry_policy
retry_policy_num_retries: Final[int | None] = get_num_retries_from_retry_policy(
exception=exception,
retry_policy=kwargs.get("retry_policy"),
)
@ -1167,7 +1190,7 @@ def _get_wrapper_num_retries(kwargs: dict[str, Any], exception: Exception) -> tu
return num_retries, kwargs
def _get_wrapper_timeout(kwargs: dict[str, Any], exception: Exception) -> float | int | httpx.Timeout | None:
def _get_wrapper_timeout(kwargs: dict[str, object], exception: Exception) -> float | int | httpx.Timeout | None:
"""
Get the timeout from the kwargs
Used for the wrapper functions.
@ -1179,7 +1202,7 @@ def _get_wrapper_timeout(kwargs: dict[str, Any], exception: Exception) -> float
def check_coroutine(value) -> bool:
get_coroutine_checker: Final = getattr(sys.modules[__name__], "get_coroutine_checker")
get_coroutine_checker: Final = litellm_utils.get_coroutine_checker
return get_coroutine_checker().is_async_callable(value)
@ -1207,7 +1230,7 @@ async def async_pre_call_deployment_hook(kwargs: dict[str, Any], call_type: str)
async def async_post_call_success_deployment_hook(
request_data: dict, response: Any, call_type: CallTypes | None
request_data: dict, response: object, call_type: CallTypes | None
) -> Any | None:
"""
Allow modifying / reviewing the response just after it's received from the deployment.
@ -1317,7 +1340,7 @@ def post_call_processing(
def client(original_function):
Rules: Final = getattr(sys.modules[__name__], "Rules")
Rules: Final = litellm_utils.Rules
rules_obj: Final = Rules()
@wraps(original_function)
@ -1551,10 +1574,10 @@ def client(original_function):
if call_type == CallTypes.completion.value:
num_retries = kwargs.get("num_retries", None) or litellm.num_retries or None
if kwargs.get("retry_policy", None):
get_num_retries_from_retry_policy = getattr(
get_num_retries_from_retry_policy: Callable[..., int | None] = getattr(
sys.modules[__name__], "get_num_retries_from_retry_policy"
)
reset_retry_policy = getattr(sys.modules[__name__], "reset_retry_policy")
reset_retry_policy = litellm_utils.reset_retry_policy
num_retries = get_num_retries_from_retry_policy(
exception=e,
retry_policy=kwargs.get("retry_policy"),
@ -1593,7 +1616,7 @@ def client(original_function):
get_num_retries_from_retry_policy = getattr(
sys.modules[__name__], "get_num_retries_from_retry_policy"
)
reset_retry_policy = getattr(sys.modules[__name__], "reset_retry_policy")
reset_retry_policy = litellm_utils.reset_retry_policy
num_retries = get_num_retries_from_retry_policy(
exception=e,
retry_policy=kwargs.get("retry_policy"),
@ -1939,7 +1962,7 @@ def client(original_function):
if not _is_streaming_response_for_correlation(result):
_restore_correlation_context_if_supported(logging_obj)
get_coroutine_checker: Final = getattr(sys.modules[__name__], "get_coroutine_checker")
get_coroutine_checker: Final = litellm_utils.get_coroutine_checker
is_coroutine: Final = get_coroutine_checker().is_async_callable(original_function)
# Return the appropriate wrapper based on the original function type
@ -1992,7 +2015,7 @@ _STREAMING_CALL_TYPES: Final = frozenset(
def _is_streaming_request(
kwargs: dict[str, Any],
kwargs: dict[str, object],
call_type: CallTypes | str,
) -> bool:
"""
@ -2323,7 +2346,7 @@ def supports_response_schema(model: str, custom_llm_provider: str | None = None)
"""
## GET LLM PROVIDER ##
try:
get_llm_provider: Final = getattr(sys.modules[__name__], "get_llm_provider")
get_llm_provider: Final = litellm_utils.get_llm_provider
model, custom_llm_provider, _, _ = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
except Exception as e:
verbose_logger.debug(
@ -2700,7 +2723,7 @@ _CACHE_PRICING_FIELDS: Final = (
)
def _resolve_builtin_model_cost_entry(key: str, provider: str) -> dict[str, Any] | None:
def _resolve_builtin_model_cost_entry(key: str, provider: str) -> dict[str, object] | None:
"""Best-effort lookup of a built-in ``model_cost`` entry for a custom key
whose shape ``get_model_info`` cannot resolve (repeated provider prefixes
like ``bedrock/bedrock/bedrock/us.anthropic.claude-sonnet-4-6`` or region
@ -2992,7 +3015,7 @@ def get_optional_params_transcription(
passed_params.pop("OPENAI_TRANSCRIPTION_PARAMS")
custom_llm_provider = passed_params.pop("custom_llm_provider")
drop_params = passed_params.pop("drop_params")
special_params: Final = passed_params.pop("kwargs")
special_params: Final[Mapping[str, object]] = passed_params.pop("kwargs")
for k, v in special_params.items():
passed_params[k] = v
@ -3101,7 +3124,7 @@ def get_optional_params_image_gen(
provider_config = passed_params.pop("provider_config", None)
drop_params = passed_params.pop("drop_params", None)
additional_drop_params = passed_params.pop("additional_drop_params", None)
special_params: Final = passed_params.pop("kwargs")
special_params: Final[Mapping[str, object]] = passed_params.pop("kwargs")
for k, v in special_params.items():
if (
k.startswith("aws_")
@ -3133,7 +3156,7 @@ def get_optional_params_image_gen(
default_params=default_params,
additional_drop_params=additional_drop_params,
)
optional_params: dict[str, Any] = {}
optional_params: dict[str, object] = {}
## raise exception if non-default value passed for non-openai/azure embedding calls
def _check_valid_arg(supported_params):
@ -3365,7 +3388,14 @@ def get_optional_params_embeddings(
elif custom_llm_provider == "bedrock":
# if dimensions is in non_default_params -> pass it for model=bedrock/amazon.titan-embed-text-v2
if "amazon.titan-embed-text-v1" in model:
object: Any = litellm.AmazonTitanG1Config()
object: (
AmazonTitanG1Config
| AmazonTitanMultimodalEmbeddingG1Config
| AmazonTitanV2Config
| BedrockCohereEmbeddingConfig
| TwelveLabsMarengoEmbeddingConfig
| AmazonNovaEmbeddingConfig
) = litellm.AmazonTitanG1Config()
elif "amazon.titan-embed-image-v1" in model:
object = litellm.AmazonTitanMultimodalEmbeddingG1Config()
elif "amazon.titan-embed-text-v2:0" in model:
@ -4949,7 +4979,7 @@ def get_max_tokens(model: str) -> int | None:
response.raise_for_status() # Raise an exception for bad responses (4xx or 5xx)
# Parse the JSON response
config_json: Final = response.json()
config_json: Final[Mapping[str, int]] = response.json()
# Extract and return the max_position_embeddings
max_position_embeddings: Final = config_json.get("max_position_embeddings")
if max_position_embeddings is not None:
@ -4965,7 +4995,7 @@ def get_max_tokens(model: str) -> int | None:
return litellm.model_cost[model]["max_output_tokens"]
elif "max_tokens" in litellm.model_cost[model]:
return litellm.model_cost[model]["max_tokens"]
get_llm_provider: Final = getattr(sys.modules[__name__], "get_llm_provider")
get_llm_provider: Final = litellm_utils.get_llm_provider
model, custom_llm_provider, _, _ = get_llm_provider(model=model)
if custom_llm_provider == "huggingface":
max_tokens: Final = _get_max_position_embeddings(model_name=model)
@ -5253,7 +5283,7 @@ def _get_potential_model_names(model: str, custom_llm_provider: str | None) -> P
if custom_llm_provider is None:
# Get custom_llm_provider
try:
get_llm_provider: Final = getattr(sys.modules[__name__], "get_llm_provider")
get_llm_provider: Final = litellm_utils.get_llm_provider
split_model, custom_llm_provider, _, _ = get_llm_provider(model=model)
except Exception:
split_model = model
@ -5297,7 +5327,7 @@ def _get_max_position_embeddings(model_name: str) -> int | None:
response.raise_for_status() # Raise an exception for bad responses (4xx or 5xx)
# Parse the JSON response
config_json: Final = response.json()
config_json: Final[Mapping[str, int]] = response.json()
# Extract and return the max_position_embeddings
max_position_embeddings: Final = config_json.get("max_position_embeddings")
@ -6067,7 +6097,7 @@ def validate_environment(
}
## EXTRACT LLM PROVIDER - if model name provided
try:
get_llm_provider: Final = getattr(sys.modules[__name__], "get_llm_provider")
get_llm_provider: Final = litellm_utils.get_llm_provider
_, custom_llm_provider, _, _ = get_llm_provider(model=model)
except Exception:
custom_llm_provider = None
@ -6544,7 +6574,7 @@ def _get_retry_after_from_exception_header(
# <http-date>". See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After#syntax for
# details.
if response_headers is not None:
retry_header: Final = response_headers.get("retry-after")
retry_header: Final[str] = response_headers.get("retry-after")
try:
retry_after = int(retry_header)
except Exception:
@ -6635,7 +6665,7 @@ def register_prompt_template(
complete_model: Final = model
potential_models: Final = [complete_model]
try:
get_llm_provider: Final = getattr(sys.modules[__name__], "get_llm_provider")
get_llm_provider: Final = litellm_utils.get_llm_provider
model = get_llm_provider(model=model)[0]
potential_models.append(model)
except Exception:
@ -7277,7 +7307,7 @@ def _get_base_model_from_metadata(model_call_details=None):
return _base_model
metadata: Final = litellm_params.get("metadata") or {}
_get_base_model_from_litellm_call_metadata = getattr(
_get_base_model_from_litellm_call_metadata: Callable[..., str | None] = getattr(
sys.modules[__name__], "_get_base_model_from_litellm_call_metadata"
)
base_model_from_metadata: Final = _get_base_model_from_litellm_call_metadata(metadata=metadata)
@ -7970,7 +8000,7 @@ class ProviderConfigManager:
@staticmethod
def _get_cohere_config(model: str) -> BaseConfig:
"""Get Cohere config based on route."""
CohereModelInfo: Final = getattr(sys.modules[__name__], "CohereModelInfo")
CohereModelInfo: Final = litellm_utils.CohereModelInfo
route: Final = CohereModelInfo.get_cohere_route(model)
if route == "v2":
return litellm.CohereV2ChatConfig()
@ -9007,7 +9037,7 @@ class ProviderConfigManager:
return ReductoParseLegacyConfig()
return None
MistralOCRConfig: Final = getattr(sys.modules[__name__], "MistralOCRConfig")
MistralOCRConfig: Final = litellm_utils.MistralOCRConfig
PROVIDER_TO_CONFIG_MAP: Final = {
litellm.LlmProviders.MISTRAL: MistralOCRConfig,
}
@ -9286,13 +9316,14 @@ def extract_duration_from_srt_or_vtt(srt_or_vtt_content: str) -> float | None:
# Regular expression to match timestamps in the format "hh:mm:ss,ms" or "hh:mm:ss.ms"
timestamp_pattern: Final = r"(\d{2}):(\d{2}):(\d{2})[.,](\d{3})"
timestamps: Final = re.findall(timestamp_pattern, srt_or_vtt_content)
timestamps: Final[Sequence[tuple[str, str, str, str]]] = re.findall(timestamp_pattern, srt_or_vtt_content)
if not timestamps:
return None
# Convert timestamps to seconds and find the max (end time)
durations: Final = []
match: tuple[str, str, str, str]
for match in timestamps:
hours, minutes, seconds, milliseconds = map(int, match)
total_seconds = hours * 3600 + minutes * 60 + seconds + milliseconds / 1000.0
@ -9339,11 +9370,11 @@ def _add_path_to_api_base(api_base: str, ending_path: str) -> str:
return str(modified_url.copy_with(params=original_url.params))
def get_standard_openai_params(params: dict) -> dict:
def get_standard_openai_params(params: Mapping[str, object]) -> dict:
return {k: v for k, v in params.items() if k in litellm.OPENAI_CHAT_COMPLETION_PARAMS and v is not None}
def get_non_default_completion_params(kwargs: dict) -> dict:
def get_non_default_completion_params(kwargs: Mapping[str, object]) -> dict:
openai_params: Final = litellm.OPENAI_CHAT_COMPLETION_PARAMS
default_params: Final = openai_params + all_litellm_params
non_default_params: Final = {
@ -9353,7 +9384,7 @@ def get_non_default_completion_params(kwargs: dict) -> dict:
return non_default_params
def peek_reasoning_summary_aliases(optional_params: dict) -> Any | None:
def peek_reasoning_summary_aliases(optional_params: dict) -> object | None:
"""Read AI-SDK-style reasoning summary from optional_params or nested extra_body.
Uses key membership (not ``or`` chains) so falsy values like ``""`` are not skipped.
@ -9373,7 +9404,7 @@ def peek_reasoning_summary_aliases(optional_params: dict) -> Any | None:
def strip_reasoning_summary_aliases_from_optional_params(
optional_params: dict,
) -> tuple[dict, Any | None]:
) -> tuple[dict, object | None]:
"""Copy optional_params; remove reasoningSummary aliases from top-level and extra_body."""
op: Final = dict(optional_params)
rs_val = op.pop("reasoningSummary", None)
@ -9405,7 +9436,7 @@ def get_non_default_transcription_params(kwargs: dict) -> dict:
def add_openai_metadata(
metadata: Mapping[str, Any] | None,
metadata: Mapping[str, object] | None,
) -> dict[str, str] | None:
"""
Add metadata to openai optional parameters, excluding hidden params.
@ -9439,7 +9470,7 @@ def add_openai_metadata(
return visible_metadata.copy()
def get_requester_metadata(metadata: dict):
def get_requester_metadata(metadata: Mapping[str, object]):
if not metadata:
return None
@ -9499,7 +9530,7 @@ def return_raw_request(endpoint: CallTypes, kwargs: dict) -> RawRequestTypedDict
)
def jsonify_tools(tools: list[Any]) -> list[dict]:
def jsonify_tools(tools: Sequence[object]) -> list[dict]:
"""
Fixes https://github.com/BerriAI/litellm/issues/9321
@ -9525,9 +9556,9 @@ def get_empty_usage() -> Usage:
def should_run_mock_completion(
mock_response: Any | None,
mock_tool_calls: Any | None,
mock_timeout: Any | None,
mock_response: object | None,
mock_tool_calls: object | None,
mock_timeout: object | None,
) -> bool:
if mock_response or mock_tool_calls or mock_timeout:
return True

20
ruff-strict-budget.json
View file

@ -1,21 +1,21 @@
{
"ANN001": {
"limit": 3106
"limit": 3058
},
"ANN002": {
"limit": 71
},
"ANN003": {
"limit": 832
"limit": 827
},
"ANN201": {
"limit": 2023
"limit": 2022
},
"ANN202": {
"limit": 860
"limit": 855
},
"ANN204": {
"limit": 713
"limit": 712
},
"ANN205": {
"limit": 114
@ -24,7 +24,7 @@
"limit": 133
},
"ANN401": {
"limit": 1491
"limit": 1384
},
"ASYNC230": {
"limit": 11
@ -39,7 +39,7 @@
"limit": 505
},
"B009": {
"limit": 79
"limit": 64
},
"B010": {
"limit": 190
@ -171,7 +171,7 @@
"limit": 3
},
"RET504": {
"limit": 177
"limit": 176
},
"RUF012": {
"limit": 241
@ -201,7 +201,7 @@
"limit": 58
},
"SIM102": {
"limit": 322
"limit": 321
},
"SIM103": {
"limit": 119
@ -234,7 +234,7 @@
"limit": 5
},
"TID251": {
"limit": 1226
"limit": 1224
},
"TRY002": {
"limit": 524

10
type-discipline-budget.json
View file

@ -1,9 +1,9 @@
{
"LIT001": {
"limit": 23039
"limit": 23003
},
"LIT002": {
"limit": 27154
"limit": 27146
},
"LIT003": {
"limit": 269
@ -15,19 +15,19 @@
"limit": 0
},
"LIT006": {
"limit": 1078
"limit": 1077
},
"LIT007": {
"limit": 0
},
"LIT008": {
"limit": 951
"limit": 950
},
"LIT009": {
"limit": 0
},
"LIT010": {
"limit": 16742
"limit": 16731
},
"LIT011": {
"limit": 5596