scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-11 08:38:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 280

refactor(typing): clear about 1,100 basedpyright Any errors across 303 files (#49986)

Browse source 
* refactor(typing): clear basedpyright Any errors (wave 1)

* refactor(typing): clear basedpyright Any errors (wave 2)

* refactor(typing): clear basedpyright Any errors (wave 3)

* refactor(typing): clear basedpyright Any errors (wave 4)

* refactor(typing): clear basedpyright Any errors (wave 5)

* refactor(typing): clear basedpyright Any errors (wave 6)

* fix(typing): keep the cached-plan fallback's direct query_first calls so span naming resolves

* fix(logging): keep callbacks running when litellm_params is a params model

should_run_callback and the sync success and failure handlers read litellm_params through any object with a get method again, as they did before this branch, so a logging hook that returns a GenericLiteLLMParams no longer stops sync logging for that request

Adds behavior tests for the changed lines codecov listed as not run

* fix: read the regional uplift for every data residency again and reuse one dict TypeAdapter

The typed lookup only read the eu and us multipliers, so a deployment that set regional_processing_uplift_multiplier_fedramp stopped applying it. /model/update and the model write paths now share one module-level TypeAdapter instead of building one per call.

* fix: bill regional uplift multipliers float() accepts, bytes included

* fix: raise the in operator's own error from one shared json_contains, and name the bad uplift type
This commit is contained in:
Mateo Wang 2026-11-06 13:31:15 -08:00 • committed by GitHub
parent 2d2290067c
commit 3f38730d2c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
303 changed files with 5111 additions and 1411 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
litellm/a2a_protocol/card_resolver.py
View file

@ -4,13 +4,14 @@ Custom A2A Card Resolver for LiteLLM.
Extends the A2A SDK's card resolver to support multiple well-known paths.
"""
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Protocol, runtime_checkable
from litellm._logging import verbose_logger
from litellm.a2a_protocol.exceptions import A2AAgentCardDiscoveryError
from litellm.constants import LOCALHOST_URL_PATTERNS
from litellm.litellm_core_utils.core_helpers import has_attributes
if TYPE_CHECKING:
from a2a.types import AgentCard
@ -130,15 +131,34 @@ def normalize_agent_card_interfaces(agent_card: "AgentCard", protocol_version: s
return normalized
class _UrlCarrier(Protocol):
url: str
class _InterfacesCarrier(Protocol):
@property
def supported_interfaces(self) -> Sequence[_UrlCarrier]: ...
def _url_of(value: object) -> str | None:
return value.url if has_attributes(value, _UrlCarrier, "url") else None
def _supported_interfaces(agent_card: object) -> Sequence[_UrlCarrier] | None:
if has_attributes(agent_card, _InterfacesCarrier, "supported_interfaces"):
return agent_card.supported_interfaces
return None
def get_agent_card_url(agent_card: "AgentCard") -> str | None:
"""Return the agent endpoint URL from the resolved SDK card."""
url: Final = getattr(agent_card, "url", None)
url: Final = _url_of(agent_card)
if url:
return url
interfaces: Final = getattr(agent_card, "supported_interfaces", None)
interfaces: Final = _supported_interfaces(agent_card)
if interfaces:
return getattr(interfaces[0], "url", None)
return _url_of(interfaces[0])
return None
@ -168,16 +188,16 @@ def fix_agent_card_url(agent_card: "AgentCard", base_url: str) -> "AgentCard":
Returns:
The agent card with the URL fixed if necessary
"""
card_url: Final = getattr(agent_card, "url", None)
card_url: Final = _url_of(agent_card)
if card_url and is_localhost_or_internal_url(card_url):
# Normalize base_url to ensure it ends with /
fixed_url: Final = base_url.rstrip("/") + "/"
agent_card.url = fixed_url
interfaces: Final = getattr(agent_card, "supported_interfaces", None)
interfaces: Final = _supported_interfaces(agent_card)
if interfaces:
interface_url: Final = getattr(interfaces[0], "url", None)
interface_url: Final = _url_of(interfaces[0])
if interface_url and is_localhost_or_internal_url(interface_url):
interfaces[0].url = base_url.rstrip("/") + "/"
@ -194,6 +214,9 @@ class LiteLLMA2ACardResolver(_A2ACardResolver):
- /agentCard/v1.0
"""
if TYPE_CHECKING:
base_url: str
async def get_agent_card(
self,
relative_card_path: str | None = None,

5
litellm/a2a_protocol/exception_mapping_utils.py
View file

@ -4,7 +4,7 @@ A2A Protocol Exception Mapping Utils.
Maps A2A SDK exceptions to LiteLLM A2A exception types.
"""
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Final
from litellm._logging import verbose_logger
from litellm.a2a_protocol.card_resolver import (
@ -21,6 +21,7 @@ from litellm.constants import CONNECTION_ERROR_PATTERNS
if TYPE_CHECKING:
from a2a.client import Client as A2AClientType
from a2a.types import AgentCard
_A2A_KIND_DIALECT_HINT: Final = (
@ -165,7 +166,7 @@ def map_a2a_exception(
async def handle_a2a_localhost_retry(
error: A2ALocalhostURLError,
agent_card: Any,
agent_card: "AgentCard | None",
a2a_client: "A2AClientType",
is_streaming: bool = False,
) -> "A2AClientType":

7
litellm/a2a_protocol/litellm_completion_bridge/handler.py
View file

@ -10,7 +10,7 @@ A2A Streaming Events (in order):
4. Status update (kind: "status-update") - Final status "completed" with final=true
"""
from collections.abc import AsyncIterator, Callable, Coroutine, Mapping
from collections.abc import AsyncIterator, Mapping
from types import MappingProxyType
from typing import Any, Final
@ -126,10 +126,7 @@ class A2ACompletionBridgeHandler:
@staticmethod
async def _acompletion(completion_params: Mapping[str, object]) -> ModelResponse | CustomStreamWrapper:
acompletion_fn: Final[Callable[..., Coroutine[object, object, ModelResponse | CustomStreamWrapper]]] = vars(
litellm
)["acompletion"]
return await acompletion_fn(**completion_params)
return await litellm.acompletion(**completion_params)
@staticmethod
async def handle_non_streaming(

15
litellm/a2a_protocol/main.py
View file

@ -14,7 +14,7 @@ import datetime
import uuid
from collections.abc import AsyncIterator, Coroutine, Mapping
from types import MappingProxyType, ModuleType
from typing import TYPE_CHECKING, Final, Optional, cast
from typing import TYPE_CHECKING, Final, Optional, Protocol, cast
from typing_extensions import TypedDict, Unpack
@ -24,6 +24,7 @@ from litellm.a2a_protocol.streaming_iterator import A2AStreamingIterator
from litellm.a2a_protocol.utils import A2ARequestUtils
from litellm.constants import DEFAULT_A2A_AGENT_TIMEOUT
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
@ -36,7 +37,6 @@ if TYPE_CHECKING:
from a2a.client import Client as A2AClientType
from a2a.client import ClientCallContext as A2ACallContextType
from a2a.compat.v0_3.types import (
AgentCard,
Message,
SendMessageRequest,
SendMessageResponse,
@ -45,6 +45,7 @@ if TYPE_CHECKING:
SendStreamingMessageSuccessResponse,
Task,
)
from a2a.types import AgentCard
from a2a.types.a2a_pb2 import SendMessageRequest as CoreSendMessageRequest
from a2a.types.a2a_pb2 import StreamResponse as CoreStreamResponse
@ -95,6 +96,12 @@ class _SendMessageOptions(TypedDict, total=False, extra_items=object):
pass
class _StreamingRequestConverter(Protocol):
def to_core_send_message_request(
self, compat_req: "SendStreamingMessageRequest", /
) -> "CoreSendMessageRequest": ...
def _set_usage_on_logging_obj(
kwargs: Mapping[str, object],
prompt_tokens: int,
@ -375,7 +382,9 @@ async def _stream_messages(
"The 'a2a' package is required for A2A agent invocation. Install it with: pip install a2a-sdk"
)
pb_request: Final[CoreSendMessageRequest] = _a2a_conversions.to_core_send_message_request(request)
pb_request: Final[CoreSendMessageRequest] = require_attributes(
_a2a_conversions, _StreamingRequestConverter, "to_core_send_message_request"
).to_core_send_message_request(request)
async for event in a2a_client.send_message(pb_request, context=_get_a2a_call_context(a2a_client)):
compat_chunk = _to_compat_stream_response(event, request_id=request.id)
yield SendStreamingMessageResponse(root=compat_chunk)

25
litellm/caching/caching.py
View file

@ -13,10 +13,10 @@ import json
import logging
import time
import traceback
from collections.abc import Generator, Mapping
from collections.abc import Awaitable, Callable, Generator, Mapping
from contextlib import contextmanager
from enum import Enum
from typing import Any, Final, Literal, cast
from typing import Any, Final, Literal, Protocol, cast
from pydantic import BaseModel, ConfigDict, TypeAdapter
@ -25,6 +25,7 @@ from litellm._internal_context import current_service_target, service_target
from litellm._logging import verbose_logger
from litellm.constants import CACHED_STREAMING_CHUNK_DELAY
from litellm.integrations.otel.runtime import phase_span
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.litellm_core_utils.model_param_helper import (
ModelParamHelper,
embeds_thought_signature_in_tool_call_id,
@ -52,6 +53,16 @@ _REDIS_STARTUP_NODES_ADAPTER: Final = TypeAdapter(
)
class _SupportsAsyncPing(Protocol):
@property
def ping(self) -> Callable[[], Awaitable[object]] | None: ...
class _SupportsAsyncDeleteCacheKeys(Protocol):
@property
def delete_cache_keys(self) -> Callable[[object], Awaitable[object]] | None: ...
def _keyed_provider_params_in_extra_body(kwargs: Mapping[str, object]) -> str:
raw_extra_body: Final = kwargs.get("extra_body")
if not isinstance(raw_extra_body, dict):
@ -1032,14 +1043,16 @@ class Cache:
cache_key, cached_data, kwargs = self._add_cache_logic(result=result, **kwargs)
await self.cache.batch_cache_write(cache_key, cached_data, **kwargs)
async def ping(self):
cache_ping: Final = getattr(self.cache, "ping")
async def ping(self) -> object:
cache_ping: Final = require_attributes(self.cache, _SupportsAsyncPing, "ping").ping
if cache_ping:
return await cache_ping()
return None
async def delete_cache_keys(self, keys):
cache_delete_cache_keys: Final = getattr(self.cache, "delete_cache_keys")
async def delete_cache_keys(self, keys: object) -> object:
cache_delete_cache_keys: Final = require_attributes(
self.cache, _SupportsAsyncDeleteCacheKeys, "delete_cache_keys"
).delete_cache_keys
if cache_delete_cache_keys:
return await cache_delete_cache_keys(keys)
return None

39
litellm/caching/caching_handler.py
View file

@ -22,6 +22,7 @@ from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable,
from typing import TYPE_CHECKING, Any, Final, Optional, TypeVar
from pydantic import ConfigDict, SkipValidation, TypeAdapter, ValidationError
from typing_extensions import TypedDict
import litellm
from litellm._internal_context import is_internal_call, post_response_phase
@ -90,6 +91,10 @@ class CachingHandlerResponse(LiteLLMBaseModel):
in_memory_cache_obj: Final = InMemoryCache()
class _CacheCallKwargs(TypedDict, total=False, extra_items=object):
pass
def _drop_logging_obj_from_kwargs(request_kwargs: dict[str, object]) -> dict[str, object]:
"""
The caching handler is stored on the Logging object
@ -455,7 +460,7 @@ class LLMCachingHandler:
if litellm.cache is not None and self._is_call_type_supported_by_cache(original_function=original_function):
args = args or ()
# Now that we confirmed caching will happen, prepare kwargs
new_kwargs: Final = kwargs.copy()
new_kwargs: Final = dict[str, object](kwargs)
new_kwargs.update(
convert_args_to_kwargs(
self.original_function,
@ -470,7 +475,7 @@ class LLMCachingHandler:
print_verbose("Checking Sync Cache")
logging_obj.reset_caching_details()
with response_cache_phase("get"):
cached_result = litellm.cache.get_cache(**new_kwargs)
cached_result = litellm.cache.get_cache(**_CacheCallKwargs(**new_kwargs))
if is_response_without_output(cached_result):
verbose_logger.debug("LiteLLM Cache: cached response has no output, treating it as a miss")
return CachingHandlerResponse(cached_result=None)
@ -480,7 +485,7 @@ class LLMCachingHandler:
pass
else:
call_type = original_function.__name__
cached_result = self._convert_cached_result_to_model_response(
converted_result: Final = self._convert_cached_result_to_model_response(
cached_result=cached_result,
call_type=call_type,
kwargs=kwargs,
@ -508,16 +513,16 @@ class LLMCachingHandler:
logging_obj=logging_obj,
model=f"{custom_llm_provider}/{model}",
kwargs=kwargs,
cached_result=cached_result,
cached_result=converted_result,
is_async=False,
custom_llm_provider=custom_llm_provider,
)
if not is_internal_call.get() and not _should_defer_streaming_cache_hit_callbacks(
cached_result=cached_result
cached_result=converted_result
):
logging_obj.handle_sync_cache_hit_success_callbacks(
result=cached_result,
result=converted_result,
start_time=start_time,
end_time=end_time,
cache_hit=cache_hit,
@ -527,8 +532,8 @@ class LLMCachingHandler:
or self.request_kwargs.get("cache_key")
or litellm.cache.get_cache_key(**self.request_kwargs)
)
_set_cached_hidden_param(cached_result, "cache_key", cache_key)
return CachingHandlerResponse(cached_result=cached_result)
_set_cached_hidden_param(converted_result, "cache_key", cache_key)
return CachingHandlerResponse(cached_result=converted_result)
return CachingHandlerResponse(cached_result=cached_result)
_sync_get_cache = sync_get_cache
@ -950,22 +955,14 @@ class LLMCachingHandler:
def _convert_cached_result_to_model_response(
self,
cached_result: Any,
cached_result: object,
call_type: str,
kwargs: dict[str, object],
logging_obj: LiteLLMLoggingObj,
model: str,
args: tuple[object, ...],
custom_llm_provider: str | None = None,
) -> (
ModelResponse
| TextCompletionResponse
| EmbeddingResponse
| RerankResponse
| TranscriptionResponse
| CustomStreamWrapper
| None
):
) -> object:
"""
Internal method to process the cached result
@ -1141,7 +1138,7 @@ class LLMCachingHandler:
self,
result: object,
original_function: Callable,
kwargs: dict[str, Any],
kwargs: Mapping[str, object],
args: tuple[object, ...] | None = None,
):
"""
@ -1169,7 +1166,7 @@ class LLMCachingHandler:
return
cache: Final = litellm.cache
new_kwargs: Final = kwargs.copy()
new_kwargs: Final = dict(kwargs)
new_kwargs.update(
convert_args_to_kwargs(
original_function,
@ -1206,7 +1203,7 @@ class LLMCachingHandler:
)
)
else:
create_cache_write_task(lambda: cache.async_add_cache(result, **new_kwargs))
create_cache_write_task(lambda: cache.async_add_cache(result, **_CacheCallKwargs(**new_kwargs)))
def sync_set_cache(
self,

27
litellm/caching/redis_cache.py
View file

@ -40,7 +40,7 @@ from litellm.constants import (
REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION,
REDIS_TIMEOUT_LOG_INTERVAL,
)
from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs
from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs, has_attributes
from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
from litellm.redis.store_settings import RedisStoreSettings, breaker_settings
from litellm.types.caching import (
@ -97,6 +97,22 @@ class _AsyncRedisCommands(Protocol):
def eval(self, script: str, numkeys: int, *keys_and_args: str | bytes | float) -> Awaitable[object]: ...
class _AsyncLuaScript(Protocol):
def __call__(
self, keys: Sequence[str], args: Sequence[str | bytes | int | float], client: object
) -> Awaitable[object]: ...
class _SupportsRegisterScript(Protocol):
def register_script(self, script: str, /) -> _AsyncLuaScript: ...
class _SupportsScriptLoad(Protocol):
def script_load(self, script: str, /) -> object: ...
def evalsha(self, sha: object, numkeys: int, /, *keys_and_args: str | bytes | float) -> Awaitable[object]: ...
_GENERIC_CALLER_MODULES: Final = frozenset(
{
__name__,
@ -844,6 +860,9 @@ class RedisCache(BaseCache):
def _async_commands(self) -> _AsyncRedisCommands:
return self.init_async_client()
def _async_client_object(self) -> object:
return self.init_async_client()
def check_and_fix_namespace(self, key: str) -> str:
"""
Make sure each key starts with the given namespace
@ -1117,8 +1136,8 @@ class RedisCache(BaseCache):
Kept separate from async_register_script so each loop caches its own
executor; see that method for why the binding must be per loop.
"""
_redis_client: Final[Any] = self.init_async_client()
if hasattr(_redis_client, "register_script"):
_redis_client: Final = self._async_client_object()
if has_attributes(_redis_client, _SupportsRegisterScript, "register_script"):
registered_script: Final = _redis_client.register_script(script)
async def standalone_executor(
@ -1131,7 +1150,7 @@ class RedisCache(BaseCache):
return standalone_executor
if hasattr(_redis_client, "script_load"):
if has_attributes(_redis_client, _SupportsScriptLoad, "script_load"):
script_sha: Final = _redis_client.script_load(script)
async def cluster_executor(

5
litellm/caching/redis_semantic_cache.py
View file

@ -43,6 +43,7 @@ if TYPE_CHECKING:
_CACHE_RESPONSE_ADAPTER: Final[TypeAdapter[str | bytes | None]] = TypeAdapter(
str | bytes | None, config=ConfigDict(strict=True, hide_input_in_errors=True)
)
_DECODED_CACHE_VALUE: Final = TypeAdapter(object)
class RedisSemanticCache(BaseCache):
@ -407,10 +408,10 @@ class RedisSemanticCache(BaseCache):
# Convert string representation to Python object
try:
return json.loads(response_text)
return _DECODED_CACHE_VALUE.validate_python(json.loads(response_text))
except json.JSONDecodeError:
try:
return ast.literal_eval(response_text)
return _DECODED_CACHE_VALUE.validate_python(ast.literal_eval(response_text))
except (ValueError, SyntaxError) as e:
print_verbose(f"Error parsing cached response: {e}")
return None

9
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -39,7 +39,7 @@ from litellm.litellm_core_utils.allowed_tools_tool_choice import (
parse_chat_allowed_tools_tool_choice,
responses_allowed_tools_tool_choice,
)
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, iter_items
from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
prompt_cache_breakpoint_with_readable_mode,
@ -349,6 +349,9 @@ _TOOL_SEARCH_REPLAY_ITEM_TYPES: Final = frozenset({"reasoning", *_TOOL_SEARCH_IT
_TOOL_SEARCH_ITEMS_KEY: Final = "tool_search_items"
_DECODED_JSON: Final = TypeAdapter(object)
def _json_field(value: object, key: str) -> object:
return cast(Mapping[str, object], value).get(key) if isinstance(value, dict) else None # cast-ok: untyped JSON
@ -857,8 +860,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
# Handle message items with output_text content
if item_type == "message":
content_list: Final = item.get("content", [])
for content_item in content_list:
content_list: Final = _DECODED_JSON.validate_python(item.get("content", []))
for content_item in iter_items(content_list):
if isinstance(content_item, dict):
content_type = content_item.get("type")
if content_type == "output_text":

33
litellm/cost_calculator.py
View file

@ -8,7 +8,7 @@ from datetime import datetime
from functools import lru_cache
from itertools import accumulate
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, cast
from httpx import Response
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
@ -24,7 +24,12 @@ from litellm.constants import (
REALTIME_METERED_USAGE_KEY,
)
from litellm.litellm_core_utils.audio_utils.utils import normalized_audio_duration_seconds
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, billed_usage_cost
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
attribute_or_none,
billed_usage_cost,
require_attributes,
)
from litellm.litellm_core_utils.cost_discount import resolve_cost_discount
from litellm.litellm_core_utils.decisions_pricing import (
decisions_pricing_fields,
@ -1188,6 +1193,10 @@ def _model_contains_known_llm_provider(model: str) -> bool:
return _provider_prefix in LlmProvidersSet
class _SupportsGet(Protocol):
def get(self, key: str, /) -> object: ...
def _get_response_model(completion_response: object) -> str | None:
"""
Extract the model name from a completion response object.
@ -1219,7 +1228,7 @@ _GEMINI_TRAFFIC_TYPE_TO_SERVICE_TIER: Final[dict] = {
}
def _map_traffic_type_to_service_tier(traffic_type: str | None) -> str | None:
def _map_traffic_type_to_service_tier(traffic_type: object) -> str | None:
"""
Map a Gemini usageMetadata.trafficType value to a LiteLLM service_tier string.
@ -1273,11 +1282,13 @@ def _served_service_tier(completion_response: object, usage_object: Usage | None
usage_tier: Final = _extract_service_tier(usage_object)
if isinstance(usage_tier, str):
return usage_tier
hidden_params: Final = getattr(completion_response, "_hidden_params", None)
hidden_params: Final[object] = getattr(completion_response, "_hidden_params", None)
if hidden_params is None:
return None
provider_specific: Final = hidden_params.get("provider_specific_fields") or {}
raw_traffic_type: Final = provider_specific.get("traffic_type")
provider_specific: Final[object] = (
require_attributes(hidden_params, _SupportsGet, "get").get("provider_specific_fields") or {}
)
raw_traffic_type: Final = require_attributes(provider_specific, _SupportsGet, "get").get("traffic_type")
if not raw_traffic_type:
return None
return _map_traffic_type_to_service_tier(raw_traffic_type) or "default"
@ -1341,8 +1352,16 @@ def get_usage_object(
return None
class _DumpedUsageFields(TypedDict, total=False, extra_items=object):
pass
def _dumped_usage_fields(usage_model: BaseModel) -> Mapping[str, object]:
return usage_model.model_dump()
def _usage_from_model(usage_model: BaseModel) -> Usage:
return Usage(**usage_model.model_dump())
return Usage(**_DumpedUsageFields(**_dumped_usage_fields(usage_model)))
def _decisions_prompt_tokens_details(usage: DecisionsUsage | OpenAIDecisionUsage) -> PromptTokensDetailsWrapper:

22
litellm/exceptions.py
View file

@ -12,10 +12,11 @@
import datetime
import enum
from collections.abc import Mapping, Sequence
from typing import Any, Final
from typing import Any, Final, Protocol, SupportsIndex, SupportsInt, runtime_checkable
import httpx
import openai
from typing_extensions import Buffer
import litellm
from litellm.types.utils import LiteLLMCommonStrings
@ -1250,6 +1251,19 @@ class BlockedPiiEntityError(Exception):
super().__init__(self.message)
@runtime_checkable
class _SupportsTrunc(Protocol):
def __trunc__(self) -> int: ...
def _status_code_int(value: object) -> int:
if isinstance(value, (str, Buffer, SupportsInt, SupportsIndex, _SupportsTrunc)):
return int(value)
raise TypeError(
f"int() argument must be a string, a bytes-like object or a real number, not '{type(value).__name__}'"
)
class MidStreamFallbackError(ServiceUnavailableError):
def __init__(
self,
@ -1265,8 +1279,8 @@ class MidStreamFallbackError(ServiceUnavailableError):
is_pre_first_chunk: bool = False,
emitted_disqualifying_content: bool = False,
):
original_status: Final = getattr(original_exception, "status_code", None)
self.status_code = int(original_status) if original_status is not None else 503
original_status: Final[object] = getattr(original_exception, "status_code", None)
self.status_code = _status_code_int(original_status) if original_status is not None else 503
self.message = f"litellm.MidStreamFallbackError: {message}"
self.model = model
self.llm_provider = llm_provider
@ -1309,7 +1323,7 @@ class MidStreamFallbackError(ServiceUnavailableError):
)
# Restore the propagated status and original response/request objects
self.status_code = int(original_status) if original_status is not None else 503
self.status_code = _status_code_int(original_status) if original_status is not None else 503
self.response = _saved_response
self.request = _saved_request
self.message = _saved_message

47
litellm/harness/endpoint.py
View file

@ -14,10 +14,11 @@ import contextlib
import itertools
import json
import logging
import math
import secrets
from collections.abc import AsyncIterable, AsyncIterator, Mapping
from collections.abc import AsyncIterable, AsyncIterator, Callable, Mapping
from dataclasses import dataclass
from types import MappingProxyType, ModuleType
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Protocol
import httpx
@ -36,6 +37,7 @@ from litellm.constants import (
from litellm.harness.context import GatewayTarget
from litellm.harness.errors import HarnessError, HarnessInstallFailed
from litellm.harness.types import Harness, Usage
from litellm.litellm_core_utils.core_helpers import has_attributes
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
from litellm.types.llms.custom_http import httpxSpecialProvider
@ -45,7 +47,7 @@ if TYPE_CHECKING:
from starlette.requests import Request
from starlette.responses import JSONResponse, Response, StreamingResponse
from starlette.routing import Route
from uvicorn import Server
from uvicorn import Config, Server
verbose_logger: Final = logging.getLogger("LiteLLM")
_JSON_VALUE_ADAPTER: Final = TypeAdapter(object, config=ConfigDict(strict=True))
@ -110,9 +112,22 @@ class _ResponsesModule(Protocol):
StreamingResponse: type[StreamingResponse]
class _UvicornModule(Protocol):
Config: type[Config]
Server: type[Server]
class _CapturesSignals(Protocol):
capture_signals: Callable[[], contextlib.AbstractContextManager[None]]
class _InstallsSignalHandlers(Protocol):
install_signal_handlers: Callable[[], None]
@dataclass(frozen=True)
class _ServerDeps:
uvicorn: ModuleType
uvicorn: _UvicornModule
applications: _ApplicationsModule
routing: _RoutingModule
responses: _ResponsesModule
@ -438,6 +453,14 @@ def _noop() -> None:
return None
def _signal_capture(server: object) -> _CapturesSignals | None:
return server if has_attributes(server, _CapturesSignals, "capture_signals") else None
def _signal_handler_installer(server: object) -> _InstallsSignalHandlers | None:
return server if has_attributes(server, _InstallsSignalHandlers, "install_signal_handlers") else None
class ModelEndpoint:
"""Local HTTP endpoint for one harness session. Use as an async context manager."""
@ -521,7 +544,7 @@ class ModelEndpoint:
await asyncio.sleep(DEFAULT_POLLING_INTERVAL)
def _build_server(self, deps: _ServerDeps) -> Server:
config = deps.uvicorn.Config(
config: Final = deps.uvicorn.Config(
self._build_app(deps),
host=HARNESS_ENDPOINT_HOST,
port=0,
@ -529,14 +552,16 @@ class ModelEndpoint:
log_level="warning",
access_log=False,
lifespan="off",
timeout_graceful_shutdown=HARNESS_PROCESS_KILL_GRACE_SECONDS,
timeout_graceful_shutdown=math.ceil(HARNESS_PROCESS_KILL_GRACE_SECONDS),
)
server = deps.uvicorn.Server(config)
server: Final = deps.uvicorn.Server(config)
# Never touch the host process's signal handlers.
if hasattr(server, "capture_signals"):
server.capture_signals = contextlib.nullcontext
if hasattr(server, "install_signal_handlers"):
server.install_signal_handlers = _noop
signal_capture: Final = _signal_capture(server)
if signal_capture is not None:
signal_capture.capture_signals = contextlib.nullcontext
handler_installer: Final = _signal_handler_installer(server)
if handler_installer is not None:
handler_installer.install_signal_handlers = _noop
return server
def _build_app(self, deps: _ServerDeps) -> Starlette:

21
litellm/images/utils.py
View file

@ -6,12 +6,25 @@ from pathlib import Path
from typing import Any, Final, Protocol, cast, get_type_hints, runtime_checkable
import litellm
from litellm.litellm_core_utils.core_helpers import has_attributes
from litellm.litellm_core_utils.token_counter import get_image_type
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.types.files import FILE_MIME_TYPES, FileType
from litellm.types.images.main import ImageEditOptionalRequestParams
class _ReadsBytes(Protocol):
def read(self, size: int, /) -> bytes: ...
class _Tells(Protocol):
def tell(self) -> int: ...
class _Seeks(Protocol):
def seek(self, offset: int, /) -> object: ...
class ImageEditRequestUtils:
@staticmethod
def get_optional_params_image_edit(
@ -114,12 +127,12 @@ class ImageEditRequestUtils:
bytes_data = image_data[:100]
else:
# For other types, try to read if possible
if hasattr(image_data, "read"):
current_pos = getattr(image_data, "tell", lambda: 0)()
if hasattr(image_data, "seek"):
if has_attributes(image_data, _ReadsBytes, "read"):
current_pos = image_data.tell() if has_attributes(image_data, _Tells, "tell") else 0
if has_attributes(image_data, _Seeks, "seek"):
image_data.seek(0)
bytes_data = image_data.read(100)
if hasattr(image_data, "seek"):
if has_attributes(image_data, _Seeks, "seek"):
image_data.seek(current_pos)
else:
return FILE_MIME_TYPES[FileType.PNG] # Default fallback

4
litellm/integrations/SlackAlerting/user_spend_alerts.py
View file

@ -53,11 +53,13 @@ async def fetch_user_spend_rows(
today: datetime.date,
baseline_days: int,
) -> tuple[UserSpendRow, ...]:
from litellm.proxy.utils import query_raw_rows
today_str: Final = today.strftime("%Y-%m-%d")
month_start_str: Final = today.replace(day=1).strftime("%Y-%m-%d")
baseline_start_str: Final = (today - datetime.timedelta(days=max(baseline_days, 1))).strftime("%Y-%m-%d")
return USER_SPEND_ROWS_ADAPTER.validate_python(
await prisma_client.db.query_raw(USER_SPEND_QUERY, today_str, month_start_str, baseline_start_str)
await query_raw_rows(prisma_client.db, USER_SPEND_QUERY, today_str, month_start_str, baseline_start_str)
)

23
litellm/integrations/argilla.py
View file

@ -7,11 +7,12 @@ import json
import os
import random
import types
from collections.abc import Mapping
from typing import Final
from collections.abc import Mapping, Sequence
from typing import Final, Protocol
import httpx
from pydantic import BaseModel
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
@ -29,6 +30,22 @@ from litellm.types.integrations.argilla import (
from litellm.types.utils import StandardLoggingPayload
class _ArgillaDataset(TypedDict):
id: ReadOnly[str]
class _ArgillaDatasetsBody(TypedDict, total=False):
items: ReadOnly[Sequence[_ArgillaDataset]]
class _ArgillaDatasetsResponse(Protocol):
def json(self) -> _ArgillaDatasetsBody: ...
def _datasets_body(response: _ArgillaDatasetsResponse) -> _ArgillaDatasetsBody:
return response.json()
def is_serializable(value):
non_serializable_types: Final = (
types.CoroutineType,
@ -102,7 +119,7 @@ class ArgillaLogger(CustomBatchLogger):
url=f"{_credentials_base_url}/api/v1/me/datasets?name={_credentials_dataset_name}",
headers={"X-Argilla-Api-Key": _credentials_api_key},
)
json_response: Final = dataset_response.json()
json_response: Final = _datasets_body(dataset_response)
if (
"items" in json_response
and isinstance(json_response["items"], list)

23
litellm/integrations/arize/_utils.py
View file

@ -1,6 +1,6 @@
import json
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Final, Protocol
from pydantic import ConfigDict, TypeAdapter
from typing_extensions import ReadOnly, TypedDict, override
@ -10,6 +10,7 @@ from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes impor
BaseLLMObsOTELAttributes,
safe_set_attribute,
)
from litellm.litellm_core_utils.core_helpers import iter_items, require_attributes, subscript
from litellm.litellm_core_utils.redact_messages import (
should_redact_message_logging,
)
@ -34,32 +35,40 @@ from litellm.integrations._types.open_inference import (
_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True))
class _SupportsGetWithDefault(Protocol):
def get(self, key: str, default: object = None, /) -> object: ...
class ArizeOTELAttributes(BaseLLMObsOTELAttributes):
@staticmethod
@override
def set_messages(span: "Span", kwargs: dict[str, Any]):
def set_messages(span: "Span", kwargs: Mapping[str, object]):
messages: Final = kwargs.get("messages")
# for /chat/completions
# https://docs.arize.com/arize/large-language-models/tracing/semantic-conventions
if messages:
last_message: Final = messages[-1]
last_message: Final = subscript(messages, -1)
safe_set_attribute(
span,
SpanAttributes.INPUT_VALUE,
last_message.get("content", ""),
require_attributes(last_message, _SupportsGetWithDefault, "get").get("content", ""),
)
# LLM_INPUT_MESSAGES shows up under `input_messages` tab on the span page.
for idx, msg in enumerate(messages):
for idx, msg in enumerate(iter_items(messages)):
prefix = f"{SpanAttributes.LLM_INPUT_MESSAGES}.{idx}"
# Set the role per message.
safe_set_attribute(span, f"{prefix}.{MessageAttributes.MESSAGE_ROLE}", msg.get("role"))
safe_set_attribute(
span,
f"{prefix}.{MessageAttributes.MESSAGE_ROLE}",
require_attributes(msg, _SupportsGetWithDefault, "get").get("role"),
)
# Set the content per message.
safe_set_attribute(
span,
f"{prefix}.{MessageAttributes.MESSAGE_CONTENT}",
msg.get("content", ""),
require_attributes(msg, _SupportsGetWithDefault, "get").get("content", ""),
)
# Additive: emit structured tool_calls / multimodal content

18
litellm/integrations/azure_sentinel/azure_sentinel.py
View file

@ -18,10 +18,11 @@ import time
import traceback
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final, TypeVar
from typing import Final, Protocol, TypeVar
from urllib.parse import urlparse
import httpx
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_logger
from litellm.integrations.batch_utils import (
@ -52,6 +53,19 @@ MONITOR_SCOPE_BY_AUTHORITY_HOST: Final[Mapping[str, str]] = MappingProxyType(
)
class _OAuthTokenBody(TypedDict, total=False):
access_token: ReadOnly[str]
expires_in: ReadOnly[int]
class _OAuthTokenResponse(Protocol):
def json(self) -> _OAuthTokenBody: ...
def _oauth_token_body(response: _OAuthTokenResponse) -> _OAuthTokenBody:
return response.json()
class AzureSentinelLogger(CustomBatchLogger):
"""
Logger that sends LiteLLM logs to Azure Sentinel via Azure Monitor Logs Ingestion API
@ -226,7 +240,7 @@ class AzureSentinelLogger(CustomBatchLogger):
if response.status_code != 200:
raise Exception(f"Failed to get OAuth2 token: {response.status_code} - {response.text}")
token_response: Final = response.json()
token_response: Final = _oauth_token_body(response)
self.oauth_token = token_response.get("access_token")
expires_in: Final = token_response.get("expires_in", 3600)

32
litellm/integrations/bitbucket/bitbucket_client.py
View file

@ -5,7 +5,7 @@ BitBucket API client for fetching .prompt files from BitBucket repositories.
import base64
import urllib.parse
from collections.abc import Mapping
from typing import Final, TypedDict
from typing import Final, Protocol, TypedDict
from typing_extensions import NotRequired, ReadOnly
@ -36,6 +36,30 @@ class BitBucketFileMetadata(TypedDict):
last_modified: ReadOnly[str | None]
class _SrcListingResponse(Protocol):
def json(self) -> BitBucketSrcListing: ...
class _RepositoryInfoResponse(Protocol):
def json(self) -> Mapping[str, object]: ...
class _BranchListingResponse(Protocol):
def json(self) -> BitBucketBranchListing: ...
def _src_listing(response: _SrcListingResponse) -> BitBucketSrcListing:
return response.json()
def _repository_info(response: _RepositoryInfoResponse) -> Mapping[str, object]:
return response.json()
def _branch_listing(response: _BranchListingResponse) -> BitBucketBranchListing:
return response.json()
def _sanitize_file_path(file_path: str) -> str:
"""Reject path traversal and URL-encode each path segment."""
if "#" in file_path or "?" in file_path:
@ -166,7 +190,7 @@ class BitBucketClient:
response: Final = self.http_handler.get(url, headers=self.headers)
response.raise_for_status()
data: Final[BitBucketSrcListing] = response.json()
data: Final = _src_listing(response)
return [
file_path
for item in data.get("values", [])
@ -201,7 +225,7 @@ class BitBucketClient:
try:
response: Final = self.http_handler.get(url, headers=self.headers)
response.raise_for_status()
return response.json()
return _repository_info(response)
except Exception as e:
raise Exception(f"Failed to get repository info: {e}")
@ -231,7 +255,7 @@ class BitBucketClient:
response: Final = self.http_handler.get(url, headers=self.headers)
response.raise_for_status()
data: Final[BitBucketBranchListing] = response.json()
data: Final = _branch_listing(response)
return data.get("values", [])
except Exception as e:
raise Exception(f"Failed to get branches: {e}")

13
litellm/integrations/bitbucket/bitbucket_prompt_manager.py
View file

@ -4,12 +4,13 @@ Fetches .prompt files from BitBucket repositories and provides team-based access
"""
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Any, Final, Protocol
from jinja2 import DictLoader, select_autoescape
from jinja2.sandbox import ImmutableSandboxedEnvironment
from litellm.integrations.custom_prompt_management import CustomPromptManagement
from litellm.litellm_core_utils.core_helpers import require_attributes
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -26,6 +27,10 @@ from litellm.types.utils import StandardCallbackDynamicParams
from .bitbucket_client import BitBucketClient
class _SupportsGetWithDefault(Protocol):
def get(self, key: str, default: object = None, /) -> object: ...
class BitBucketPromptTemplate:
"""
Represents a prompt template loaded from BitBucket.
@ -35,7 +40,7 @@ class BitBucketPromptTemplate:
self,
template_id: str,
content: str,
metadata: dict[str, Any],
metadata: Mapping[str, object],
model: str | None = None,
):
self.template_id = template_id
@ -44,7 +49,9 @@ class BitBucketPromptTemplate:
self.model = model or metadata.get("model")
self.temperature = metadata.get("temperature")
self.max_tokens = metadata.get("max_tokens")
self.input_schema = metadata.get("input", {}).get("schema", {})
self.input_schema = require_attributes(metadata.get("input", {}), _SupportsGetWithDefault, "get").get(
"schema", {}
)
self.optional_params = {k: v for k, v in metadata.items() if k not in ["model", "input", "content"]}
def __repr__(self):

40
litellm/integrations/clickhouse/clickhouse_spend_logger.py
View file

@ -7,10 +7,10 @@ so `response_id` is always the raw provider response id (cache-hit suffix stripp
import json
import re
from collections.abc import Iterator, Mapping
from collections.abc import Iterable, Iterator, Mapping
from math import isfinite
from types import MappingProxyType
from typing import Any, Final
from typing import Any, Final, Protocol
from pydantic import JsonValue, TypeAdapter, ValidationError
@ -19,6 +19,7 @@ from litellm._logging import verbose_logger
from litellm.integrations.clickhouse.clickhouse_batch_logger import ClickHouseBatchLogger
from litellm.integrations.clickhouse.context import is_lens_analysis
from litellm.integrations.clickhouse.schema import SPEND_LOGS_TABLE
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id
from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
@ -40,6 +41,14 @@ _INTERNAL_METADATA_KEYS: Final = frozenset(
)
class _SupportsGet(Protocol):
def get(self, key: str, /) -> object: ...
class _SupportsItems(Protocol):
def items(self) -> Iterable[tuple[object, object]]: ...
def _request_metadata_fields(value: object) -> Iterator[tuple[str, JsonValue]]:
if value is None:
return
@ -85,22 +94,22 @@ def _json_mapping(value: Mapping[str, object]) -> str:
return _json(dict(value))
def _find_traceparent(metadata: Mapping[str, Any], kwargs: Mapping[str, Any]) -> tuple[str, str]:
def _find_traceparent(metadata: Mapping[str, object], kwargs: Mapping[str, object]) -> tuple[str, str]:
custom_headers = metadata.get("requester_custom_headers") or MappingProxyType({})
proxy_request = (kwargs.get("litellm_params") or MappingProxyType({})).get(
proxy_request = require_attributes(kwargs.get("litellm_params") or MappingProxyType({}), _SupportsGet, "get").get(
"proxy_server_request"
) or MappingProxyType({})
request_headers = proxy_request.get("headers") or MappingProxyType({})
request_headers = require_attributes(proxy_request, _SupportsGet, "get").get("headers") or MappingProxyType({})
for headers in (custom_headers, request_headers):
for name, value in headers.items():
for name, value in require_attributes(headers, _SupportsItems, "items").items():
if str(name).lower() == "traceparent":
return parse_traceparent(value)
return "", ""
def _cache_tokens(usage: Mapping[str, Any]) -> tuple[int, int]:
def _cache_tokens(usage: Mapping[str, object]) -> tuple[int, int]:
"""(cache_read, cache_write) from a Usage dict: OpenAI prompt_tokens_details first, Anthropic fields as fallback."""
details = usage.get("prompt_tokens_details") or MappingProxyType({})
details = require_attributes(usage.get("prompt_tokens_details") or MappingProxyType({}), _SupportsGet, "get")
cache_read = _int(details.get("cached_tokens")) or _int(usage.get("cache_read_input_tokens"))
cache_write = (
_int(details.get("cache_write_tokens"))
@ -116,10 +125,17 @@ def _request_tags(value: object) -> list[str]:
return [str(tag) for tag in value]
def _session_id(payload: StandardLoggingPayload | SpendLogPayload, kwargs: Mapping[str, Any]) -> str:
def _session_id(payload: StandardLoggingPayload | SpendLogPayload, kwargs: Mapping[str, object]) -> str:
"""Mirrors proxy `_get_session_id_for_spend_log`: explicit session id, else the payload trace id."""
request_metadata = (kwargs.get("litellm_params") or MappingProxyType({})).get("metadata") or MappingProxyType({})
return str(payload.get("session_id") or request_metadata.get("session_id") or payload.get("trace_id") or "")
request_metadata = require_attributes(
kwargs.get("litellm_params") or MappingProxyType({}), _SupportsGet, "get"
).get("metadata") or MappingProxyType({})
return str(
payload.get("session_id")
or require_attributes(request_metadata, _SupportsGet, "get").get("session_id")
or payload.get("trace_id")
or ""
)
def _is_trace_ingest(payload: StandardLoggingPayload) -> bool:
@ -128,7 +144,7 @@ def _is_trace_ingest(payload: StandardLoggingPayload) -> bool:
def spend_log_row_from_payload(
payload: StandardLoggingPayload | SpendLogPayload, kwargs: Mapping[str, Any]
payload: StandardLoggingPayload | SpendLogPayload, kwargs: Mapping[str, object]
) -> SpendLogRecord:
metadata: Mapping[str, Any] = payload.get("metadata") or MappingProxyType({})
hidden_params: Mapping[str, Any] = payload.get("hidden_params") or MappingProxyType({})

13
litellm/integrations/custom_guardrail.py
View file

@ -7,7 +7,7 @@ from collections.abc import Iterable, Mapping, Sequence
from datetime import datetime
from itertools import chain
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, cast, get_args
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, Protocol, cast, get_args
import httpx
from pydantic import TypeAdapter
@ -20,6 +20,7 @@ from litellm.litellm_core_utils.core_helpers import (
get_metadata_variable_name_from_kwargs,
get_or_create_metadata_bucket,
redact_nested_match_and_regex_keys,
require_attributes,
)
from litellm.secret_managers.main import str_to_bool
from litellm.types.guardrails import (
@ -167,18 +168,22 @@ def _strict_guardrail_modes_enabled() -> bool:
return True if parsed is None else parsed
def get_session_id_from_request_data(request_data: dict[str, Any]) -> str | None:
class _SupportsGet(Protocol):
def get(self, key: str, /) -> object: ...
def get_session_id_from_request_data(request_data: Mapping[str, object]) -> str | None:
"""Extract session_id from request data (litellm_session_id or metadata)."""
session_id = request_data.get("litellm_session_id")
if session_id:
return str(session_id)
metadata: Final = request_data.get("metadata") or {}
metadata: Final = require_attributes(request_data.get("metadata") or {}, _SupportsGet, "get")
session_id = metadata.get("session_id")
if session_id:
return str(session_id)
litellm_metadata: Final = request_data.get("litellm_metadata") or {}
litellm_metadata: Final = require_attributes(request_data.get("litellm_metadata") or {}, _SupportsGet, "get")
session_id = litellm_metadata.get("session_id")
if session_id:
return str(session_id)

25
litellm/integrations/email_alerting.py
View file

@ -3,17 +3,29 @@ Functions for sending Email Alerts
"""
import os
from typing import Final
from collections.abc import Awaitable, Mapping, Sequence
from typing import TYPE_CHECKING, Final
from litellm._logging import verbose_logger, verbose_proxy_logger
from litellm.repositories.team_repository import TeamRepository
from litellm.types.integrations.slack_alerting import WebhookEvent
if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
# we use this for the email header, please send a test email if you change this. verify it looks good on email
LITELLM_LOGO_URL: Final = "https://litellm-listing.s3.amazonaws.com/litellm_logo.png"
LITELLM_SUPPORT_CONTACT: Final = "support@berri.ai"
def _query_raw_rows(
client: "PrismaClient", query: str, *args: object
) -> Awaitable[Sequence[Mapping[str, object]] | None]:
from litellm.proxy.utils import query_raw_rows
return query_raw_rows(client.db, query, *args)
async def get_all_team_member_emails(team_id: str | None = None) -> list:
verbose_logger.debug("Email Alerting: Getting all team members for team_id=%s", team_id)
if team_id is None:
@ -51,17 +63,18 @@ async def get_all_team_member_emails(team_id: str | None = None) -> list:
WHERE user_id = ANY($1::TEXT[]);
"""
_result: Final = await prisma_client.db.query_raw(sql_query, _team_member_user_ids)
_result: Final = await _query_raw_rows(prisma_client, sql_query, _team_member_user_ids)
verbose_logger.debug("Email Alerting: Got all Emails for team, emails=%s", _result)
if _result is None:
return []
emails: Final = []
for user in _result:
if user and isinstance(user, dict) and user.get("user_email", None) is not None:
emails.append(user.get("user_email"))
emails: Final = [
user.get("user_email")
for user in _result
if user and isinstance(user, dict) and user.get("user_email", None) is not None
]
return emails

4
litellm/integrations/focus/database.py
View file

@ -123,7 +123,9 @@ class FocusLiteLLMDatabase:
ORDER BY ordinal_position;
"""
try:
columns_response: Final = await client.db.query_raw(info_query)
from litellm.proxy.utils import query_raw_rows
columns_response: Final = await query_raw_rows(client.db, info_query)
return {"columns": columns_response, "table_name": "LiteLLM_DailyUserSpend"}
except Exception as exc:
raise RuntimeError(f"Error getting table info: {exc}") from exc

13
litellm/integrations/galileo.py
View file

@ -15,6 +15,7 @@ from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import require_attributes, subscript
from litellm.litellm_core_utils.prompt_templates.common_utils import (
convert_content_list_to_str,
get_content_from_model_response,
@ -60,6 +61,10 @@ class _JsonResponse(Protocol):
def json(self) -> object: ...
class _SupportsGet(Protocol):
def get(self, key: str, /) -> object: ...
def _login_access_token(response: _GalileoLoginResponse) -> str:
"""Read the bearer token out of a Galileo login response body."""
return response.json()["access_token"]
@ -463,13 +468,13 @@ class GalileoObserve(CustomLogger):
pass
@staticmethod
def _build_prompt(kwargs: Mapping[str, Any]) -> dict[str, object]:
optional_params: Final[Mapping[str, object]] = kwargs.get("optional_params", {}) or {}
def _build_prompt(kwargs: Mapping[str, object]) -> dict[str, object]:
optional_params: Final = require_attributes(kwargs.get("optional_params", {}) or {}, _SupportsGet, "get")
prompt: Final[dict[str, object]] = {"messages": kwargs.get("messages")}
if optional_params.get("functions") is not None:
prompt["functions"] = optional_params["functions"]
prompt["functions"] = subscript(optional_params, "functions")
if optional_params.get("tools") is not None:
prompt["tools"] = optional_params["tools"]
prompt["tools"] = subscript(optional_params, "tools")
return prompt
@staticmethod

14
litellm/integrations/gcs_bucket/gcs_bucket.py
View file

@ -5,7 +5,7 @@ import os
import time
from collections.abc import Mapping
from datetime import datetime, timedelta, timezone
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Any, Final, Protocol
from litellm._logging import verbose_logger
from litellm._uuid import uuid
@ -16,7 +16,7 @@ from litellm.integrations.gcs_bucket.gcs_bucket_base import GCSBucketBase
from litellm.litellm_core_utils.cloud_storage_security import (
sanitize_cloud_object_component,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.core_helpers import as_str_mapping, require_attributes
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
from litellm.types.integrations.gcs_bucket import *
@ -28,6 +28,10 @@ else:
VertexBase = Any
class _SupportsGetWithDefault(Protocol):
def get(self, key: str, default: str, /) -> object: ...
class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
def __init__(self, bucket_name: str | None = None) -> None:
from litellm.proxy.proxy_server import premium_user
@ -298,7 +302,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
await self._send_individual_logs(items_to_process)
def _get_object_name(
self, kwargs: Mapping[str, object], logging_payload: StandardLoggingPayload, response_obj: Any
self, kwargs: Mapping[str, object], logging_payload: StandardLoggingPayload, response_obj: object
) -> str:
"""
Get the object name to use for the current payload
@ -313,7 +317,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
return f"{current_date}/custom-{uuid.uuid4().hex}-{safe_log_id}"
if logging_payload.get("error_str", None) is not None:
return self._generate_failure_object_name(request_date_str=current_date)
raw_response_id: Final[object] = response_obj.get("id", "")
raw_response_id: Final[object] = require_attributes(response_obj, _SupportsGetWithDefault, "get").get("id", "")
response_id: Final = raw_response_id if isinstance(raw_response_id, str) else ""
return self._generate_success_object_name(
request_date_str=current_date,
@ -360,7 +364,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
def _generate_success_object_name(
self,
request_date_str: str,
response_id: str,
response_id: object,
) -> str:
return f"{request_date_str}/{response_id}"

2
litellm/integrations/generic_prompt_management/generic_prompt_manager.py
View file

@ -132,7 +132,7 @@ class GenericPromptManager(CustomPromptManagement):
async def async_fetch_prompt_from_api(
self, prompt_id: str | None, prompt_spec: PromptSpec | None
) -> dict[str, Any]:
) -> dict[str, object]:
"""
Fetch a prompt from the API asynchronously.
"""

24
litellm/integrations/humanloop.py
View file

@ -7,10 +7,12 @@ https://humanloop.com/
from typing import Final, cast
import httpx
from pydantic import TypeAdapter
from typing_extensions import TypedDict
import litellm
from litellm.caching import DualCache
from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping, decoded_json_object, subscript
from litellm.llms.custom_httpx.http_handler import get_httpx_client
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
@ -19,6 +21,8 @@ from litellm.types.utils import StandardCallbackDynamicParams
from .custom_logger import CustomLogger
_DECODED_JSON: Final = TypeAdapter(object)
class PromptManagementClient(TypedDict):
prompt_id: str
@ -79,18 +83,20 @@ class HumanLoopPromptManager(DualCache):
raise Exception(f"Error getting prompt from Humanloop: {e.response.text}")
json_response: Final = response.json()
template_message: Final = json_response["template"]
if isinstance(template_message, dict):
template_messages = [template_message]
elif isinstance(template_message, list):
template_messages = template_message
decoded_response: Final = _DECODED_JSON.validate_python(json_response)
template_message: Final = subscript(decoded_response, "template")
template_mapping: Final = as_str_mapping(template_message)
template_list: Final = as_object_list(template_message)
if template_mapping is not None:
template_messages = [template_mapping]
elif template_list is not None:
template_messages = template_list
else:
raise ValueError(f"Invalid template message type: {type(template_message)}")
template_model: Final = json_response["model"]
optional_params: Final = {}
for k, v in json_response.items():
if k in litellm.OPENAI_CHAT_COMPLETION_PARAMS:
optional_params[k] = v
optional_params: Final = {
k: v for k, v in decoded_json_object(decoded_response).items() if k in litellm.OPENAI_CHAT_COMPLETION_PARAMS
}
return PromptManagementClient(
prompt_id=humanloop_prompt_id,
prompt_template=cast(list[AllMessageValues], template_messages),

14
litellm/integrations/lago.py
View file

@ -3,7 +3,7 @@
import json
import os
from typing import Final, Literal
from typing import Final, Literal, Protocol
import httpx
@ -11,6 +11,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.llms.custom_httpx.http_handler import (
HTTPHandler,
get_async_httpx_client,
@ -18,6 +19,10 @@ from litellm.llms.custom_httpx.http_handler import (
)
class _SupportsGet(Protocol):
def get(self, key: str, default: object = None, /) -> object: ...
def get_utc_datetime():
import datetime as dt
from datetime import datetime
@ -70,10 +75,11 @@ class LagoLogger(CustomLogger):
if (
isinstance(response_obj, litellm.ModelResponse) or isinstance(response_obj, litellm.EmbeddingResponse)
) and hasattr(response_obj, "usage"):
response_usage: Final = require_attributes(response_obj["usage"], _SupportsGet, "get")
usage = {
"prompt_tokens": response_obj["usage"].get("prompt_tokens", 0),
"completion_tokens": response_obj["usage"].get("completion_tokens", 0),
"total_tokens": response_obj["usage"].get("total_tokens"),
"prompt_tokens": response_usage.get("prompt_tokens", 0),
"completion_tokens": response_usage.get("completion_tokens", 0),
"total_tokens": response_usage.get("total_tokens"),
}
litellm_params: Final = kwargs.get("litellm_params", {}) or {}

5
litellm/integrations/langfuse/langfuse_otel.py
View file

@ -7,6 +7,7 @@ from itertools import chain
from typing import TYPE_CHECKING, Any, Final, Optional, Protocol, runtime_checkable
from opentelemetry import trace
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
@ -33,6 +34,8 @@ from litellm.types.integrations.langfuse_otel import (
)
from litellm.types.utils import StandardCallbackDynamicParams
_DECODED_JSON: Final = TypeAdapter(object)
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
@ -643,7 +646,7 @@ def _tool_call_arguments(arguments: object) -> object:
if not isinstance(arguments, str):
return arguments
try:
return json.loads(arguments)
return _DECODED_JSON.validate_python(json.loads(arguments))
except json.JSONDecodeError:
return {}

13
litellm/integrations/langfuse/langfuse_otel_attributes.py
View file

@ -7,7 +7,7 @@ Relevant Issue: https://github.com/BerriAI/litellm/issues/13764
import json
from collections.abc import Mapping
from datetime import datetime
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Final, Protocol
from pydantic import BaseModel
from typing_extensions import override
@ -16,6 +16,7 @@ from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes impor
BaseLLMObsOTELAttributes,
safe_set_attribute,
)
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponsesAPIResponse
from litellm.types.utils import (
EmbeddingResponse,
@ -30,6 +31,10 @@ if TYPE_CHECKING:
from opentelemetry.trace import Span
class _SupportsGet(Protocol):
def get(self, key: str, /) -> object: ...
def serialize_langfuse_datetime(value: datetime) -> str:
return json.dumps(value.astimezone().isoformat())
@ -87,11 +92,11 @@ def get_output_content_by_type(
class LangfuseLLMObsOTELAttributes(BaseLLMObsOTELAttributes):
@staticmethod
@override
def set_messages(span: "Span", kwargs: dict[str, Any]):
def set_messages(span: "Span", kwargs: Mapping[str, object]):
prompt: Final = {"messages": kwargs.get("messages")}
optional_params: Final = kwargs.get("optional_params", {})
functions: Final = optional_params.get("functions")
tools: Final = optional_params.get("tools")
functions: Final = require_attributes(optional_params, _SupportsGet, "get").get("functions")
tools: Final = require_attributes(optional_params, _SupportsGet, "get").get("tools")
if functions is not None:
prompt["functions"] = functions
if tools is not None:

25
litellm/integrations/mlflow.py
View file

@ -1,9 +1,11 @@
import json
import threading
from typing import TYPE_CHECKING, Any, Final
from collections.abc import Sequence
from typing import TYPE_CHECKING, Any, Final, Protocol
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import attribute_or_none, iter_items, require_attributes
if TYPE_CHECKING:
from litellm.types.utils import StandardLoggingPayload
@ -11,6 +13,25 @@ else:
StandardLoggingPayload = Any
class _ChoiceWithMessage(Protocol):
@property
def message(self) -> object: ...
class _ExcludeNoneDumpable(Protocol):
def model_dump(self, *, exclude_none: bool) -> object: ...
def _dumped_choice_message(choice: object) -> object:
message: Final = require_attributes(choice, _ChoiceWithMessage, "message").message
return require_attributes(message, _ExcludeNoneDumpable, "model_dump").model_dump(exclude_none=True)
def _output_messages(response_obj: object) -> Sequence[object]:
choices: Final = attribute_or_none(response_obj, "choices") if hasattr(response_obj, "choices") else ()
return [_dumped_choice_message(c) for c in iter_items(choices)]
class MlflowLogger(CustomLogger):
def __init__(self):
from mlflow.tracking import MlflowClient
@ -62,7 +83,7 @@ class MlflowLogger(CustomLogger):
inputs: Final = self._construct_input(kwargs)
input_messages: Final = inputs.get("messages", [])
output_messages: Final = [c.message.model_dump(exclude_none=True) for c in getattr(response_obj, "choices", [])]
output_messages: Final = _output_messages(response_obj)
if messages := [*input_messages, *output_messages]:
set_span_chat_messages(span, messages)
if tools := inputs.get("tools"):

15
litellm/integrations/openmeter.py
View file

@ -4,13 +4,13 @@
import json
import os
from collections.abc import Mapping
from typing import Final
from typing import Final, Protocol
import httpx
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.core_helpers import as_str_mapping, require_attributes
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
from litellm.llms.custom_httpx.http_handler import (
HTTPHandler,
@ -19,6 +19,10 @@ from litellm.llms.custom_httpx.http_handler import (
)
class _SupportsGet(Protocol):
def get(self, key: str, default: object = None, /) -> object: ...
def get_utc_datetime():
import datetime as dt
from datetime import datetime
@ -60,10 +64,11 @@ class OpenMeterLogger(CustomLogger):
if (
isinstance(response_obj, litellm.ModelResponse) or isinstance(response_obj, litellm.EmbeddingResponse)
) and hasattr(response_obj, "usage"):
response_usage: Final = require_attributes(response_obj["usage"], _SupportsGet, "get")
usage = {
"prompt_tokens": response_obj["usage"].get("prompt_tokens", 0),
"completion_tokens": response_obj["usage"].get("completion_tokens", 0),
"total_tokens": response_obj["usage"].get("total_tokens"),
"prompt_tokens": response_usage.get("prompt_tokens", 0),
"completion_tokens": response_usage.get("completion_tokens", 0),
"total_tokens": response_usage.get("total_tokens"),
}
# OPENMETER_TRUST_REQUEST_USER (default "true"): when set to "false",

83
litellm/integrations/opentelemetry.py
View file

@ -6,7 +6,7 @@ from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field
from datetime import datetime
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, cast
from typing import TYPE_CHECKING, Any, Final, Protocol, cast
from pydantic import ConfigDict, TypeAdapter
from typing_extensions import ReadOnly, TypedDict
@ -35,6 +35,7 @@ from litellm.integrations.otel.model.metadata import flatten_metadata
from litellm.integrations.otel.model.semconv import LiteLLM, Metric
from litellm.integrations.otel.plumbing.otlp_tls import resolve_otlp_http_tls
from litellm.integrations.otel.routing import routing_decision_attributes
from litellm.litellm_core_utils.core_helpers import iter_items, require_attributes
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.secret_redaction import redact_string
@ -47,7 +48,6 @@ from litellm.secret_managers.main import get_secret_bool, str_to_bool
from litellm.types.integrations.otel_span_attributes import SpanAttributes
from litellm.types.services import ServiceLoggerPayload
from litellm.types.utils import (
ChatCompletionMessageToolCall,
CostBreakdown,
Function,
LLMResponseTypes,
@ -130,6 +130,22 @@ def _text_as_stored_on_span(span: Span, key: str, text: str) -> str:
_MAX_DYNAMIC_TRACER_PROVIDERS: Final = 256
class _SupportsGet(Protocol):
def get(self, key: str, /) -> object: ...
def _response_field(response_obj: object, key: str) -> object:
if not response_obj:
return None
return require_attributes(response_obj, _SupportsGet, "get").get(key)
def _billed_usage(response_obj: object, call_type: str | None, litellm_params: Mapping[str, object] | None) -> object:
if not response_obj or is_unbilled_non_inference_call_from_params(call_type, litellm_params, response_obj):
return None
return _response_field(response_obj, "usage")
class _CacheHitView(TypedDict, total=False):
cache_hit: ReadOnly[object]
saved_cache_cost: ReadOnly[object]
@ -2407,17 +2423,17 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
@staticmethod
def _tool_calls_kv_pair(
tool_calls: list[ChatCompletionMessageToolCall],
tool_calls: Iterable[object],
) -> dict[str, object]:
kv_pairs: Final[dict[str, object]] = {}
for idx, tool_call in enumerate(tool_calls):
_function = tool_call.get("function")
_function: object = require_attributes(tool_call, _SupportsGet, "get").get("function")
if not _function:
continue
keys = Function.__annotations__.keys()
for key in keys:
_value = _function.get(key)
_value = require_attributes(_function, _SupportsGet, "get").get(key)
if _value:
kv_pairs[f"{SpanAttributes.LLM_COMPLETIONS.value}.{idx}.function_call.{key}"] = _value
@ -2584,7 +2600,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
# but Embeddings and Image-gen responses do not. Fall back to
# the litellm call ID so every call type can be correlated
# across LiteLLM UI, Phoenix traces, and provider logs (Issue #8).
response_id = (response_obj.get("id") if response_obj else None) or standard_logging_payload.get("id")
response_id = _response_field(response_obj, "id") or standard_logging_payload.get("id")
if response_id:
self.safe_set_attribute(
span=span,
@ -2601,42 +2617,37 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
)
# The model used to generate the response.
if response_obj and response_obj.get("model"):
response_model: Final = _response_field(response_obj, "model")
if response_model:
self.safe_set_attribute(
span=span,
key=SpanAttributes.LLM_RESPONSE_MODEL.value,
value=response_obj.get("model"),
value=response_model,
)
self._set_service_tier_attributes(span=span, standard_logging_payload=standard_logging_payload)
usage: Final = (
response_obj.get("usage")
if response_obj
and not is_unbilled_non_inference_call_from_params(
kwargs.get("call_type"), litellm_params, response_obj
)
else None
)
usage: Final = _billed_usage(response_obj, kwargs.get("call_type"), litellm_params)
if usage:
usage_fields: Final = require_attributes(usage, _SupportsGet, "get")
self.safe_set_attribute(
span=span,
key=SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS.value,
value=usage.get("total_tokens"),
value=usage_fields.get("total_tokens"),
)
# The number of tokens used in the LLM response (completion).
self.safe_set_attribute(
span=span,
key=SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS.value,
value=usage.get("completion_tokens"),
value=usage_fields.get("completion_tokens"),
)
# The number of tokens used in the LLM prompt.
self.safe_set_attribute(
span=span,
key=SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS.value,
value=usage.get("prompt_tokens"),
value=usage_fields.get("prompt_tokens"),
)
########################################################################
@ -2712,7 +2723,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
########## LLM Response Attributes ##########
#############################################
if response_obj is not None:
if response_obj.get("choices"):
response_view: Final = require_attributes(response_obj, _SupportsGet, "get")
if response_view.get("choices"):
transformed_choices: Final = self._transform_choices_to_otel_semantic_conventions(
response_obj.get("choices")
)
@ -2722,10 +2734,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
value=safe_dumps(transformed_choices),
)
finish_reasons: Final = []
for idx, choice in enumerate(response_obj.get("choices")):
if choice.get("finish_reason"):
finish_reasons.append(choice.get("finish_reason"))
choice_views: Final = [
require_attributes(choice, _SupportsGet, "get")
for choice in iter_items(response_view.get("choices"))
]
finish_reasons: Final = [
choice.get("finish_reason") for choice in choice_views if choice.get("finish_reason")
]
if finish_reasons:
self.safe_set_attribute(
@ -2734,11 +2749,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
value=safe_dumps(finish_reasons),
)
for idx, choice in enumerate(response_obj.get("choices")):
for choice in choice_views:
if choice.get("finish_reason"):
message = choice.get("message")
if tool_calls := tool_calls_with_server_tool_calls(message):
kv_pairs = OpenTelemetry._tool_calls_kv_pair(list(tool_calls))
tool_calls = tool_calls_with_server_tool_calls(choice.get("message"))
if tool_calls:
kv_pairs = OpenTelemetry._tool_calls_kv_pair(tool_calls)
for key, value in kv_pairs.items():
self.safe_set_attribute(
span=span,
@ -2746,13 +2761,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
value=value,
)
elif response_obj.get("output"):
elif response_view.get("output"):
# Responses API: ResponsesAPIResponse has an "output"
# list instead of "choices". Each item with
# type="message" contains a "content" list of
# OutputText objects (type="output_text").
output_items: Final = response_obj.get("output")
output_messages: Final = self._transform_responses_api_output_to_otel(output_items)
output_items: Final = response_view.get("output")
output_messages: Final = self._transform_responses_api_output_to_otel(iter_items(output_items))
if output_messages:
self.safe_set_attribute(
span=span,
@ -2766,7 +2781,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
# ChatCompletionMessageToolCall format expected by
# _tool_calls_kv_pair.
tool_calls = []
for out_item in output_items:
for out_item in iter_items(output_items):
item_d = self._to_dict(out_item)
if item_d and item_d.get("type") == "function_call":
tool_calls.append(
@ -2787,7 +2802,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
)
# Extract finish reason from ResponsesAPIResponse.status
status: Final = response_obj.get("status")
status: Final = response_view.get("status")
if status:
self.safe_set_attribute(
span=span,
@ -2907,7 +2922,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
return obj.model_dump()
return None
def _transform_responses_api_output_to_otel(self, output: list) -> list[dict]:
def _transform_responses_api_output_to_otel(self, output: Iterable[object]) -> list[dict]:
"""
Transform Responses API output items into OTEL GenAI 1.38 format.

3
litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py
View file

@ -1,4 +1,5 @@
from abc import ABC
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
if TYPE_CHECKING:
@ -7,7 +8,7 @@ if TYPE_CHECKING:
class BaseLLMObsOTELAttributes(ABC):
@staticmethod
def set_messages(span: "Span", kwargs: dict[str, Any]):
def set_messages(span: "Span", kwargs: Mapping[str, object]):
pass
@staticmethod

20
litellm/integrations/otel/plumbing/metrics.py
View file

@ -11,6 +11,7 @@ identical metrics. The attribute cardinality filter is reused from v1 by import
from collections.abc import Mapping
from dataclasses import dataclass
from datetime import datetime
from types import MappingProxyType
from typing import Any, Final, Literal, Protocol, TypeAlias
from opentelemetry.metrics import Histogram, Meter
@ -34,6 +35,7 @@ from litellm.integrations.otel.model.semconv import (
resolve_provider,
)
from litellm.integrations.otel.model.utils import to_seconds
from litellm.litellm_core_utils.core_helpers import attribute_or_none, require_attributes
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
@ -177,7 +179,14 @@ class _MetricKwargs(TypedDict, total=False):
end_time: ReadOnly[datetime | float | str | None]
def resolve_error_type(kwargs: Mapping[str, Any]) -> str:
class _SupportsGet(Protocol):
def get(self, key: str, /) -> object: ...
_NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
def resolve_error_type(kwargs: Mapping[str, object]) -> str:
"""The ``error.type`` value for a failed request.
Bounded by construction: the mapped provider exception's class name (the same
@ -188,8 +197,13 @@ def resolve_error_type(kwargs: Mapping[str, Any]) -> str:
span and its exception event, where high cardinality is free.
"""
std_log: Final = kwargs.get("standard_logging_object")
info: Final = getattr(std_log, "error_information", None) or (std_log or {}).get("error_information") or {}
error_class: Final = info.get("error_class") or info.get("error_code")
info: Final = (
attribute_or_none(std_log, "error_information")
or require_attributes(std_log or _NO_FIELDS, _SupportsGet, "get").get("error_information")
or _NO_FIELDS
)
info_fields: Final = require_attributes(info, _SupportsGet, "get")
error_class: Final = info_fields.get("error_class") or info_fields.get("error_code")
if error_class:
return str(error_class)
exception: Final = kwargs.get("exception")

9
litellm/integrations/prompt_layer.py
View file

@ -4,9 +4,12 @@ import os
import traceback
from typing import Final
from pydantic import BaseModel
from pydantic import BaseModel, TypeAdapter
import litellm
from litellm.litellm_core_utils.core_helpers import decoded_json_object
_DECODED_JSON: Final = TypeAdapter(object)
class PromptLayerLogger:
@ -61,8 +64,8 @@ class PromptLayerLogger:
},
)
response_json: Final = request_response.json()
if not request_response.json().get("success", False):
response_json: Final = decoded_json_object(_DECODED_JSON.validate_python(request_response.json()))
if not response_json.get("success", False):
raise Exception("Promptlayer did not successfully log the response!")
print_verbose(f"Prompt Layer Logging: success - final response object: {request_response.text}")

45
litellm/integrations/weave/weave_otel.py
View file

@ -4,7 +4,8 @@ import base64
import json
import os
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Protocol
from opentelemetry.trace import Status, StatusCode
from typing_extensions import override
@ -19,6 +20,7 @@ from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes impor
BaseLLMObsOTELAttributes,
safe_set_attribute,
)
from litellm.litellm_core_utils.core_helpers import has_attributes, require_attributes
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.types.integrations.weave_otel import WeaveOtelConfig, WeaveSpanAttributes
from litellm.types.utils import StandardCallbackDynamicParams
@ -34,6 +36,21 @@ WEAVE_BASE_URL: Final = "https://trace.wandb.ai"
WEAVE_OTEL_ENDPOINT: Final = "/otel/v1/traces"
_NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
class _SupportsGet(Protocol):
def get(self, key: str, /) -> object: ...
class _SupportsReplace(Protocol):
def replace(self, old: str, new: str, /) -> object: ...
class _ModelDumpable(Protocol):
def model_dump(self) -> object: ...
class WeaveLLMObsOTELAttributes(BaseLLMObsOTELAttributes):
"""
Weave-specific LLM observability OTEL attributes.
@ -44,15 +61,15 @@ class WeaveLLMObsOTELAttributes(BaseLLMObsOTELAttributes):
@staticmethod
@override
def set_messages(span: Span, kwargs: dict[str, Any]):
def set_messages(span: Span, kwargs: Mapping[str, object]):
"""Set input messages as span attributes using OpenInference conventions."""
messages: Final = kwargs.get("messages") or []
optional_params: Final = kwargs.get("optional_params") or {}
optional_params: Final = kwargs.get("optional_params") or _NO_FIELDS
prompt: Final = {"messages": messages}
functions: Final = optional_params.get("functions")
tools: Final = optional_params.get("tools")
functions: Final = require_attributes(optional_params, _SupportsGet, "get").get("functions")
tools: Final = require_attributes(optional_params, _SupportsGet, "get").get("tools")
if functions is not None:
prompt["functions"] = functions
if tools is not None:
@ -60,7 +77,7 @@ class WeaveLLMObsOTELAttributes(BaseLLMObsOTELAttributes):
safe_set_attribute(span, OpenInferenceSpanAttributes.INPUT_VALUE, json.dumps(prompt))
def _set_weave_specific_attributes(span: Span, kwargs: Mapping[str, Any], response_obj: Any):
def _set_weave_specific_attributes(span: Span, kwargs: Mapping[str, object], response_obj: object):
"""
Sets Weave-specific metadata attributes onto the OTEL span.
@ -69,25 +86,27 @@ def _set_weave_specific_attributes(span: Span, kwargs: Mapping[str, Any], respon
"""
# Extract all needed data upfront
litellm_params: Final = kwargs.get("litellm_params") or {}
litellm_params: Final = kwargs.get("litellm_params") or _NO_FIELDS
# optional_params = kwargs.get("optional_params") or {}
metadata: Final = kwargs.get("metadata") or {}
metadata: Final = kwargs.get("metadata") or _NO_FIELDS
model: Final = kwargs.get("model") or ""
custom_llm_provider: Final = litellm_params.get("custom_llm_provider") or ""
custom_llm_provider: Final = (
require_attributes(litellm_params, _SupportsGet, "get").get("custom_llm_provider") or ""
)
# Weave supports a custom display name and will default to the model name if not provided.
display_name = metadata.get("display_name")
display_name = require_attributes(metadata, _SupportsGet, "get").get("display_name")
if not display_name and model:
if custom_llm_provider:
display_name = f"{custom_llm_provider}/{model}"
else:
display_name = model
if display_name:
display_name = display_name.replace("/", "__")
display_name = require_attributes(display_name, _SupportsReplace, "replace").replace("/", "__")
safe_set_attribute(span, WeaveSpanAttributes.DISPLAY_NAME.value, display_name)
# Weave threads are OpenInference sessions.
if (session_id := metadata.get("session_id")) is not None:
if (session_id := require_attributes(metadata, _SupportsGet, "get").get("session_id")) is not None:
if isinstance(session_id, (list, dict)):
session_id = safe_dumps(session_id)
safe_set_attribute(span, WeaveSpanAttributes.THREAD_ID.value, session_id)
@ -97,7 +116,7 @@ def _set_weave_specific_attributes(span: Span, kwargs: Mapping[str, Any], respon
# but we override them here to better match Weave's expectations
if response_obj:
output_dict = None
if hasattr(response_obj, "model_dump"):
if has_attributes(response_obj, _ModelDumpable, "model_dump"):
output_dict = response_obj.model_dump()
elif hasattr(response_obj, "get"):
output_dict = response_obj

22
litellm/integrations/websearch_interception/tools.py
View file

@ -7,9 +7,19 @@ to this format for consistent interception and execution.
"""
from collections.abc import Mapping
from typing import Any, Final
from typing import Final, Protocol
from litellm.constants import LITELLM_WEB_SEARCH_TOOL_NAME
from litellm.litellm_core_utils.core_helpers import require_attributes
class _SupportsGetWithDefault(Protocol):
def get(self, key: str, default: str, /) -> object: ...
class _SupportsStartswith(Protocol):
def startswith(self, prefix: str, /) -> object: ...
_WEB_SEARCH_TOOL_DESCRIPTION: Final = (
"Search the web for information. Use this when you need current "
@ -164,7 +174,7 @@ def is_chat_completion_request_web_search_tool(tool: Mapping[str, object]) -> bo
return tool.get("type") == "web_search" or is_web_search_tool(dict(tool))
def is_web_search_tool_chat_completion(tool: dict[str, Any]) -> bool:
def is_web_search_tool_chat_completion(tool: Mapping[str, object]) -> bool:
"""
Check if a tool is a web search tool for Chat Completions API (strict check).
@ -197,7 +207,7 @@ def is_web_search_tool_chat_completion(tool: dict[str, Any]) -> bool:
# Check for OpenAI format: {"type": "function", "function": {"name": "litellm_web_search"}}
if tool_type == "function" and "function" in tool:
function_def: Final = tool.get("function", {})
function_name: Final = function_def.get("name", "")
function_name: Final = require_attributes(function_def, _SupportsGetWithDefault, "get").get("name", "")
if function_name == LITELLM_WEB_SEARCH_TOOL_NAME:
return True
@ -229,7 +239,7 @@ def is_anthropic_native_web_search_tool(tool: Mapping[str, object]) -> bool:
return tool_type.startswith("web_search_") and tool_type != "function"
def is_web_search_tool(tool: dict[str, Any]) -> bool:
def is_web_search_tool(tool: Mapping[str, object]) -> bool:
"""
Check if a tool is a web search tool (native or LiteLLM standard).
@ -281,7 +291,7 @@ def is_web_search_tool(tool: dict[str, Any]) -> bool:
# Check for OpenAI format: {"type": "function", "function": {"name": "..."}}
if tool_type == "function" and "function" in tool:
function_def: Final = tool.get("function", {})
function_name: Final = function_def.get("name", "")
function_name: Final = require_attributes(function_def, _SupportsGetWithDefault, "get").get("name", "")
if function_name == LITELLM_WEB_SEARCH_TOOL_NAME:
return True
@ -290,7 +300,7 @@ def is_web_search_tool(tool: dict[str, Any]) -> bool:
return True
# Check for native Anthropic web_search_* types
if tool_type.startswith("web_search_"):
if require_attributes(tool_type, _SupportsStartswith, "startswith").startswith("web_search_"):
return True
# Check for Claude Code's web_search with a type field

12
litellm/integrations/weights_biases.py
View file

@ -1,11 +1,13 @@
from typing import Final
from litellm.litellm_core_utils.core_helpers import iter_items, subscript
imported_openAIResponse = True
try:
import io
import logging
from collections.abc import Mapping
from typing import Any, Literal, Protocol, TypeVar
from typing import Literal, Protocol, TypeVar
from wandb.sdk.data_types import trace_tree
@ -25,7 +27,7 @@ try:
class OpenAIRequestResponseResolver:
def __call__(
self,
request: dict[str, Any],
request: Mapping[str, object],
response: OpenAIResponse,
time_elapsed: float,
) -> trace_tree.WBTraceTree | None:
@ -110,14 +112,14 @@ try:
def _resolve_chat_completion(
self,
request: dict[str, Any],
request: Mapping[str, object],
response: OpenAIResponse,
time_elapsed: float,
) -> trace_tree.WBTraceTree:
"""Resolves the request and response objects for `openai.Completion`."""
prompt: Final = io.StringIO()
for message in request["messages"]:
prompt.write(f"\n\n**{message['role']}**: {message['content']}\n")
for message in iter_items(request["messages"]):
prompt.write(f"\n\n**{subscript(message, 'role')}**: {subscript(message, 'content')}\n")
request_str: Final = prompt.getvalue()
choices: Final = [

12
litellm/interactions/litellm_responses_transformation/transformation.py
View file

@ -8,11 +8,11 @@ This module handles transforming between:
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final, cast
from typing import Final, Protocol, cast
from pydantic import BaseModel
from litellm.litellm_core_utils.core_helpers import billed_usage_cost
from litellm.litellm_core_utils.core_helpers import attribute_or_none, billed_usage_cost, has_attributes
from litellm.types.interactions import (
InteractionInput,
InteractionsAPIOptionalRequestParams,
@ -27,6 +27,10 @@ from litellm.types.llms.openai import (
_STEP_TYPE_ROLES: Final = MappingProxyType({"user_input": "user", "model_output": "assistant"})
class _IsoFormattable(Protocol):
def isoformat(self) -> object: ...
class LiteLLMResponsesInteractionsConfig:
"""Configuration class for transforming between Interactions API and Responses API."""
@ -232,12 +236,12 @@ class LiteLLMResponsesInteractionsConfig:
)
# Convert created_at to ISO string
created_at: Final = getattr(responses_response, "created_at", None)
created_at: Final = attribute_or_none(responses_response, "created_at")
if isinstance(created_at, int):
from datetime import datetime
created = datetime.fromtimestamp(created_at).isoformat()
elif created_at is not None and hasattr(created_at, "isoformat"):
elif created_at is not None and has_attributes(created_at, _IsoFormattable, "isoformat"):
created = created_at.isoformat()
else:
created = None

19
litellm/interactions/streaming_iterator.py
View file

@ -7,8 +7,9 @@ from the Google Interactions API, similar to the responses API streaming iterato
import asyncio
import json
from collections.abc import Mapping
from datetime import datetime
from typing import Any, Final
from typing import Final, Protocol
import httpx
from pydantic import ConfigDict, TypeAdapter
@ -16,7 +17,7 @@ from pydantic import ConfigDict, TypeAdapter
from litellm._logging import verbose_logger
from litellm.constants import STREAM_SSE_DONE_STRING
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.core_helpers import process_response_headers, require_attributes
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base
from litellm.litellm_core_utils.thread_pool_executor import executor
@ -32,6 +33,10 @@ _INTERACTIONS_STREAMING_CHUNK_ADAPTER: Final = TypeAdapter(
)
class _SupportsGet(Protocol):
def get(self, key: str, default: object, /) -> object: ...
class BaseInteractionsAPIStreamingIterator:
"""
Base class for streaming iterators that process responses from the Interactions API.
@ -45,7 +50,7 @@ class BaseInteractionsAPIStreamingIterator:
model: str | None,
interactions_api_config: BaseInteractionsAPIConfig,
logging_obj: LiteLLMLoggingObj,
litellm_metadata: dict[str, Any] | None = None,
litellm_metadata: Mapping[str, object] | None = None,
custom_llm_provider: str | None = None,
):
self.response = response
@ -65,9 +70,9 @@ class BaseInteractionsAPIStreamingIterator:
model=model or "",
optional_params=self.logging_obj.model_call_details.get("litellm_params", {}),
)
_model_info: Final[dict] = litellm_metadata.get("model_info", {}) if litellm_metadata else {}
_model_info: Final[object] = litellm_metadata.get("model_info", {}) if litellm_metadata else {}
self._hidden_params = {
"model_id": _model_info.get("id", None),
"model_id": require_attributes(_model_info, _SupportsGet, "get").get("id", None),
"api_base": _api_base,
}
self._hidden_params["additional_headers"] = process_response_headers(self.response.headers or {})
@ -135,7 +140,7 @@ class InteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator):
model: str | None,
interactions_api_config: BaseInteractionsAPIConfig,
logging_obj: LiteLLMLoggingObj,
litellm_metadata: dict[str, Any] | None = None,
litellm_metadata: Mapping[str, object] | None = None,
custom_llm_provider: str | None = None,
):
super().__init__(
@ -202,7 +207,7 @@ class SyncInteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator)
model: str | None,
interactions_api_config: BaseInteractionsAPIConfig,
logging_obj: LiteLLMLoggingObj,
litellm_metadata: dict[str, Any] | None = None,
litellm_metadata: Mapping[str, object] | None = None,
custom_llm_provider: str | None = None,
):
super().__init__(

2
litellm/litellm_core_utils/audio_utils/utils.py
View file

@ -313,7 +313,7 @@ def calculate_request_duration(file: FileTypes) -> float | None:
file_content = content
elif hasattr(content, "read") and not isinstance(content, (str, os.PathLike)):
# File-like object in tuple
current_pos: Final = getattr(content, "tell", lambda: None)()
current_pos: Final = content.tell() if hasattr(content, "tell") else None
# Seek to start to ensure we read the entire content
if hasattr(content, "seek"):
content.seek(0)

25
litellm/litellm_core_utils/core_helpers.py
View file

@ -4,6 +4,7 @@ import copy
import logging
import math
import re
import sys
from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable
@ -133,7 +134,7 @@ class _SupportsIter(Protocol):
@runtime_checkable
class _SupportsGetItem(Protocol):
def __getitem__(self, index: int, /) -> object: ...
def __getitem__(self, key: str | int, /) -> object: ...
def attribute_or_none(source: object, name: str) -> object:
@ -166,6 +167,28 @@ def iter_items(value: object) -> Iterable[object]:
raise TypeError(f"'{type(value).__name__}' object is not iterable")
@runtime_checkable
class _SupportsContains(Protocol):
def __contains__(self, key: object, /) -> bool: ...
_NOT_A_CONTAINER: Final = "is not a container or iterable" if sys.version_info >= (3, 14) else "is not iterable"
def json_contains(value: object, key: object) -> bool:
if isinstance(value, _SupportsContains):
return key in value
if isinstance(value, (_SupportsIter, _SupportsGetItem)):
return key in tuple(iter_items(value))
raise TypeError(f"argument of type '{type(value).__name__}' {_NOT_A_CONTAINER}")
def subscript(value: object, key: str | int) -> object:
if isinstance(value, _SupportsGetItem):
return value[key]
raise TypeError(f"'{type(value).__name__}' object is not subscriptable")
def attribute_items(source: object, name: str) -> Iterable[object]:
value: Final = attribute_or_none(source, name)
return iter_items(value) if value else ()

17
litellm/litellm_core_utils/dd_tracing.py
View file

@ -5,8 +5,9 @@ If the ddtrace package is not installed, the tracer will be a no-op.
"""
from contextlib import contextmanager
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Any, Final, Protocol
from litellm.litellm_core_utils.core_helpers import has_attributes
from litellm.secret_managers.main import get_secret_bool
if TYPE_CHECKING:
@ -15,6 +16,14 @@ else:
DD_TRACER = Any
class _TagStrSpan(Protocol):
def set_tag_str(self, key: str, value: str, /) -> object: ...
class _TagSpan(Protocol):
def set_tag(self, key: str, value: str, /) -> object: ...
class NullSpan:
"""A no-op span implementation."""
@ -81,7 +90,7 @@ else:
tracer = NullTracer()
def get_active_span() -> Any | None:
def get_active_span() -> object:
"""
Return the active Datadog span, checking current span first and then root span.
"""
@ -115,10 +124,10 @@ def set_active_span_tag(tag_key: str, tag_value: str) -> bool:
return False
try:
if hasattr(span, "set_tag_str"):
if has_attributes(span, _TagStrSpan, "set_tag_str"):
span.set_tag_str(tag_key, str(tag_value))
return True
if hasattr(span, "set_tag"):
if has_attributes(span, _TagSpan, "set_tag"):
span.set_tag(tag_key, str(tag_value))
return True
except Exception:

93
litellm/litellm_core_utils/litellm_logging.py
View file

@ -19,7 +19,7 @@ from datetime import datetime as dt_object
from enum import Enum
from functools import lru_cache
from types import MappingProxyType, TracebackType
from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, TypeVar, cast
from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, Protocol, TypeVar, cast
from httpx import Response, Timeout
from openai import AsyncStream as OpenAIAsyncStream
@ -90,6 +90,7 @@ from litellm.litellm_core_utils.classifier_logging import (
is_classifier_call,
)
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
finite_cost,
get_provider_response_headers_from_hidden_params,
is_expected_client_error,
@ -99,6 +100,7 @@ from litellm.litellm_core_utils.core_helpers import (
proxy_stamped_used_client_oauth_token,
reconstruct_model_name,
reports_native_messages_cost,
require_attributes,
set_response_cost_in_hidden_params,
single_log_line,
)
@ -666,6 +668,31 @@ class UsageObservation(Enum):
ESTIMATED = "estimated"
class _A2AUsageSlot(Protocol):
usage: Mapping[str, object] | None
class _A2AResponseCopier(Protocol):
model_copy: Callable[..., _A2AUsageSlot]
class _MetadataLookup(Protocol):
get: Callable[[str], object]
class _ParamsLookup(Protocol):
def get(self, key: str, default: object, /) -> object: ...
def _callback_control_params(litellm_params: object) -> dict[str, object]:
mapping: Final = as_str_mapping(litellm_params)
if isinstance(mapping, dict):
return mapping
if mapping is not None:
return dict(mapping)
return litellm_params.model_dump() if isinstance(litellm_params, BaseModel) else {}
class Logging(LiteLLMLoggingBaseClass):
global \
supabaseClient, \
@ -929,7 +956,9 @@ class Logging(LiteLLMLoggingBaseClass):
new_call.start_time = start_time
new_call.litellm_params = litellm_params
new_call.model_call_details = {
**{key: value for key, value in self.model_call_details.items() if not key.startswith("_accounting_")},
**{
key: value for key, value in self._model_call_details_view.items() if not key.startswith("_accounting_")
},
"litellm_call_id": litellm_call_id,
"start_time": start_time,
"first_api_call_start_time": None,
@ -2467,7 +2496,7 @@ class Logging(LiteLLMLoggingBaseClass):
return self.response_cost_calculator(result=result, cache_hit=cache_hit)
@staticmethod
def is_sync_litellm_request(litellm_params: Mapping[str, object]) -> bool:
def is_sync_litellm_request(litellm_params: _ParamsLookup) -> bool:
"""True for sync SDK entrypoints (``completion``), false for async (``acompletion``, etc.)."""
return (
litellm_params.get(CallTypes.acompletion.value, False) is not True
@ -2918,11 +2947,12 @@ class Logging(LiteLLMLoggingBaseClass):
"""Deprecated alias of mark_logging_complete, kept for callers of the old name"""
self.mark_logging_complete(event_type=event_type)
def should_run_callback(self, callback: litellm.CALLBACK_TYPES, litellm_params: dict, event_hook: str) -> bool:
def should_run_callback(self, callback: litellm.CALLBACK_TYPES, litellm_params: object, event_hook: str) -> bool:
if litellm.global_disable_no_log_param:
return True
if litellm_params.get("no-log", False) is True:
request_params: Final = require_attributes(litellm_params, _ParamsLookup, "get")
if request_params.get("no-log", False) is True:
# proxy cost tracking cal backs should run
if not (isinstance(callback, CustomLogger) and "_PROXY_" in callback.__class__.__name__):
@ -2932,7 +2962,7 @@ class Logging(LiteLLMLoggingBaseClass):
# Check for dynamically disabled callbacks via headers
if EnterpriseCallbackControls is not None and EnterpriseCallbackControls.is_callback_disabled_dynamically(
callback=callback,
litellm_params=litellm_params,
litellm_params=_callback_control_params(litellm_params),
standard_callback_dynamic_params=self.standard_callback_dynamic_params,
):
verbose_logger.debug(
@ -3643,8 +3673,8 @@ class Logging(LiteLLMLoggingBaseClass):
build_logging_payload=True,
publish_logging_payload=not for_async_request,
)
litellm_params = self.model_call_details.get("litellm_params", {})
is_sync_request: Final = self.is_sync_litellm_request(litellm_params)
litellm_params = self._model_call_details_view.get("litellm_params", {})
is_sync_request: Final = self.is_sync_litellm_request(require_attributes(litellm_params, _ParamsLookup, "get"))
try:
## BUILD COMPLETE STREAMED RESPONSE
complete_streaming_response: (
@ -3800,18 +3830,23 @@ class Logging(LiteLLMLoggingBaseClass):
model = call_details_view["model"]
messages = call_details_view["messages"]
optional_params = kwargs.get("optional_params", {})
litellm_params = kwargs.get("litellm_params", {})
optional_params = call_details_view.get("optional_params", {})
litellm_params = call_details_view.get("litellm_params", {})
supabaseClient.log_event(
model=model,
messages=messages,
end_user=optional_params.get("user", "default"),
end_user=require_attributes(optional_params, _ParamsLookup, "get").get("user", "default"),
response_obj=result,
start_time=start_time,
end_time=end_time,
litellm_call_id=(
current_call_id
if (current_call_id := litellm_params.get("litellm_call_id")) is not None
if (
current_call_id := require_attributes(litellm_params, _ParamsLookup, "get").get(
"litellm_call_id", None
)
)
is not None
else str(uuid.uuid4())
),
print_verbose=print_verbose,
@ -4369,7 +4404,7 @@ class Logging(LiteLLMLoggingBaseClass):
for callback in delivery_callbacks:
# check if callback can run for this request
litellm_params = self.model_call_details.get("litellm_params", {})
litellm_params = self._model_call_details_view.get("litellm_params", {})
should_run = self.should_run_callback(
callback=callback,
litellm_params=litellm_params,
@ -4658,8 +4693,8 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.debug("Logging Details LiteLLM-Failure Call: %s", litellm.failure_callback)
if not self.should_run_logging(event_type="sync_failure"): # prevent double logging
return
litellm_params: Final = self.model_call_details.get("litellm_params", {})
is_sync_request: Final = self.is_sync_litellm_request(litellm_params)
litellm_params: Final = self._model_call_details_view.get("litellm_params", {})
is_sync_request: Final = self.is_sync_litellm_request(require_attributes(litellm_params, _ParamsLookup, "get"))
try:
start_time, end_time = self._failure_handler_helper_fn(
@ -4879,7 +4914,7 @@ class Logging(LiteLLMLoggingBaseClass):
self.mark_logging_complete(event_type="async_failure")
for callback in callbacks:
try:
litellm_params = self.model_call_details.get("litellm_params", {})
litellm_params = self._model_call_details_view.get("litellm_params", {})
should_run = self.should_run_callback(
callback=callback,
litellm_params=litellm_params,
@ -5441,7 +5476,7 @@ class Logging(LiteLLMLoggingBaseClass):
httpx_response: Final[Response | None] = self.model_call_details.get("httpx_response", None)
if httpx_response is None:
raise ValueError("Google GenAI Generate Content: httpx_response is None")
dict_result: Final = httpx_response.json()
dict_result: Final[GenerateContentResponseBody] = httpx_response.json()
result = litellm.VertexGeminiConfig().transform_google_generate_content_to_openai_model_response(
completion_response=dict_result,
model_response=litellm.ModelResponse(),
@ -5454,7 +5489,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
return result
def _handle_a2a_response_logging(self, result: Any) -> object:
def _handle_a2a_response_logging(self, result: object) -> object:
"""
Handles logging for A2A (Agent-to-Agent) responses.
@ -5473,7 +5508,7 @@ class Logging(LiteLLMLoggingBaseClass):
return result
# Deep copy result and add usage
result_copy: Final = result.model_copy(deep=True)
result_copy: Final = require_attributes(result, _A2AResponseCopier, "model_copy").model_copy(deep=True)
result_copy.usage = usage.model_dump() if hasattr(usage, "model_dump") else dict(usage)
return result_copy
@ -7475,7 +7510,7 @@ class StandardLoggingPayloadSetup:
@staticmethod
def get_standard_logging_payload_trace_id(
logging_obj: Logging,
litellm_params: Mapping[str, Any],
litellm_params: Mapping[str, object],
) -> str:
"""
Returns the `litellm_trace_id` for this request
@ -7491,9 +7526,13 @@ class StandardLoggingPayloadSetup:
"""
dynamic_litellm_session_id: Final = litellm_params.get("litellm_session_id")
dynamic_litellm_trace_id: Final = litellm_params.get("litellm_trace_id")
metadata: Final[Mapping[str, object] | None] = litellm_params.get("metadata")
metadata_session_id: Final = metadata.get("session_id") if metadata else None
metadata_trace_id: Final = metadata.get("trace_id") if metadata else None
metadata: Final = litellm_params.get("metadata")
metadata_session_id: Final = (
require_attributes(metadata, _MetadataLookup, "get").get("session_id") if metadata else None
)
metadata_trace_id: Final = (
require_attributes(metadata, _MetadataLookup, "get").get("trace_id") if metadata else None
)
ordered_candidates: Final[tuple[object, object, object, object]] = (
(dynamic_litellm_trace_id, dynamic_litellm_session_id, metadata_trace_id, metadata_session_id)
@ -7508,7 +7547,7 @@ class StandardLoggingPayloadSetup:
@staticmethod
def get_standard_logging_payload_session_id(
logging_obj: Logging,
litellm_params: Mapping[str, Any],
litellm_params: Mapping[str, object],
) -> str:
"""
Returns the end-user/conversation `litellm_session_id` for this request, independent of trace_id.
@ -7523,8 +7562,10 @@ class StandardLoggingPayloadSetup:
dynamic_litellm_session_id: Final[object] = litellm_params.get("litellm_session_id")
if dynamic_litellm_session_id:
return str(dynamic_litellm_session_id)
metadata: Final[Mapping[str, object] | None] = litellm_params.get("metadata")
metadata_session_id: Final = metadata.get("session_id") if metadata else None
metadata: Final = litellm_params.get("metadata")
metadata_session_id: Final = (
require_attributes(metadata, _MetadataLookup, "get").get("session_id") if metadata else None
)
if metadata_session_id:
return str(metadata_session_id)
return logging_obj.litellm_session_id

35
litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py
View file

@ -1,7 +1,8 @@
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Any, Final
from typing import Any, Final, Protocol
from litellm.litellm_core_utils.core_helpers import iter_items, require_attributes
from litellm.types.utils import (
CompletionTokensDetailsWrapper,
PromptTokensDetailsWrapper,
@ -55,19 +56,29 @@ _INTERACTIONS_MODALITY_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
)
def _modality_field(entry: Mapping[str, object]) -> str | None:
return _INTERACTIONS_MODALITY_FIELDS.get(str(entry.get("modality", "")).lower())
class _SupportsGet(Protocol):
def get(self, key: str, default: object = None, /) -> object: ...
def _modality_field(entry: object) -> str | None:
return _INTERACTIONS_MODALITY_FIELDS.get(
str(require_attributes(entry, _SupportsGet, "get").get("modality", "")).lower()
)
def _token_count(value: object) -> int:
return value if isinstance(value, int) else 0
def _modality_token_sums(entries: Sequence[Mapping[str, object]]) -> Mapping[str, int]:
def _modality_token_sums(entries: Sequence[object]) -> Mapping[str, int]:
fields: Final = frozenset(field for entry in entries if (field := _modality_field(entry)) is not None)
return MappingProxyType(
{
field: sum(_token_count(entry.get("tokens")) for entry in entries if _modality_field(entry) == field)
field: sum(
_token_count(require_attributes(entry, _SupportsGet, "get").get("tokens"))
for entry in entries
if _modality_field(entry) == field
)
for field in fields
}
)
@ -119,12 +130,16 @@ class InteractionsUsageObjectTransformation:
return "total_input_tokens" in usage_object or "total_output_tokens" in usage_object
@staticmethod
def transform_interactions_usage_object(usage_object: Mapping[str, Any]) -> Usage:
input_entries: Final = tuple(usage_object.get("input_tokens_by_modality") or ()) + tuple(
usage_object.get("tool_use_tokens_by_modality") or ()
def transform_interactions_usage_object(usage_object: Mapping[str, object]) -> Usage:
input_entries: Final = tuple(iter_items(usage_object.get("input_tokens_by_modality") or ())) + tuple(
iter_items(usage_object.get("tool_use_tokens_by_modality") or ())
)
cached_sums: Final = _modality_token_sums(
tuple(iter_items(usage_object.get("cached_tokens_by_modality") or ()))
)
output_sums: Final = _modality_token_sums(
tuple(iter_items(usage_object.get("output_tokens_by_modality") or ()))
)
cached_sums: Final = _modality_token_sums(tuple(usage_object.get("cached_tokens_by_modality") or ()))
output_sums: Final = _modality_token_sums(tuple(usage_object.get("output_tokens_by_modality") or ()))
total_cached_tokens: Final = _token_count(usage_object.get("total_cached_tokens"))
input_sums: Final = _subtract_cached_from_input(

12
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -6,10 +6,10 @@ from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field, replace
from datetime import datetime, timezone, tzinfo
from types import MappingProxyType
from typing import Final, Literal, Protocol, TypedDict, cast
from typing import Final, Literal, Protocol, SupportsFloat, SupportsIndex, TypedDict, cast
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
from typing_extensions import ReadOnly
from typing_extensions import Buffer, ReadOnly
import litellm
from litellm._internal_context import current_billing_time
@ -48,7 +48,6 @@ _IMAGE_RESPONSE_CALL_TYPES: Final = frozenset(
}
)
# Pre-resolved DataResidency enum values for fast membership checks
_VALID_DATA_RESIDENCIES: Final = frozenset(r.value for r in DataResidency)
_DEPLOYMENT_PRICING_KEYS: Final[frozenset[str]] = frozenset(CustomPricingLiteLLMParams.model_fields)
@ -1428,6 +1427,13 @@ def get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str |
multiplier: Final = model_info.get(f"regional_processing_uplift_multiplier_{residency}")
if multiplier is None:
return 1.0
if not isinstance(multiplier, (str, Buffer, SupportsFloat, SupportsIndex)):
verbose_logger.error(
"Invalid regional_processing_uplift_multiplier_%s for model (got %s); defaulting to 1.0",
residency,
type(multiplier).__name__,
)
return 1.0
try:
return float(multiplier)
except (TypeError, ValueError):

4
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -6,7 +6,7 @@ import json
from collections.abc import Hashable, Iterable, Mapping, Sequence
from dataclasses import dataclass
from itertools import chain, groupby
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
from typing import TYPE_CHECKING, Final, Literal, TypeAlias, cast
from pydantic import TypeAdapter
@ -1143,7 +1143,7 @@ def strip_prefill_prefix_field(message: AllMessageValues) -> AllMessageValues:
return _message_without_fields(message, frozenset({"prefix"}))
def filter_value_from_dict(dictionary: dict, key: str, depth: int = 0) -> Any:
def filter_value_from_dict(dictionary: dict, key: str, depth: int = 0) -> Mapping[object, object]:
"""
Filters a value from a dictionary

33
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -10,7 +10,7 @@ from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Se
from dataclasses import dataclass, replace
from enum import Enum
from types import MappingProxyType
from typing import Annotated, Any, Final, TypeAlias, TypedDict, TypeVar, cast, overload
from typing import Annotated, Any, Final, Protocol, TypeAlias, TypedDict, TypeVar, cast, overload
from jinja2.sandbox import ImmutableSandboxedEnvironment
from pydantic import BaseModel, BeforeValidator, ConfigDict, JsonValue, TypeAdapter, ValidationError, model_validator
@ -22,7 +22,7 @@ import litellm.types.llms
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.constants import REDACTED_BY_LITELLM
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, require_attributes
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
anthropic_system_messages,
find_anthropic_server_tool_result,
@ -4150,7 +4150,7 @@ class BedrockImageProcessor:
_validate_format = validate_format
@staticmethod
def _get_document_format(mime_type: str, supported_doc_formats: list[str]) -> str:
def _get_document_format(mime_type: str, supported_doc_formats: Sequence[str]) -> str:
"""
Get the document format from the mime type
@ -6571,6 +6571,20 @@ def has_tool_with_name(tools: object, tool_name: str) -> bool:
return False
class _SupportsModelDump(Protocol):
def model_dump(self) -> dict[str, object]: ...
class _StructuredMessagesSource(Protocol):
def get_structured_messages(self, data: dict[str, object], /) -> Sequence[object] | None: ...
def _instantiate_structured_message_sources(
source_classes: Iterable[Callable[[], _StructuredMessagesSource]],
) -> Sequence[_StructuredMessagesSource]:
return [source_cls() for source_cls in source_classes]
def resolve_structured_messages(
messages: list[dict[str, object]] | None,
request_kwargs: dict[str, Any],
@ -6608,16 +6622,17 @@ def resolve_structured_messages(
break
# 2. Fallback: try each mapped handler until one produces messages
handlers_to_try: Final[list[Any]] = []
if call_type is not None and call_type in mappings:
handlers_to_try.append(mappings[call_type]())
else:
handlers_to_try.extend(handler_cls() for handler_cls in mappings.values())
handlers_to_try: Final = _instantiate_structured_message_sources(
(mappings[call_type],) if call_type is not None and call_type in mappings else mappings.values()
)
for handler in handlers_to_try:
structured = handler.get_structured_messages(request_kwargs)
if structured:
return [msg if isinstance(msg, dict) else msg.model_dump() for msg in structured]
return [
msg if isinstance(msg, dict) else require_attributes(msg, _SupportsModelDump, "model_dump").model_dump()
for msg in structured
]
return None

15
litellm/litellm_core_utils/redact_messages.py
View file

@ -12,7 +12,7 @@ import copy
import inspect
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import replace
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Any, Final, Protocol
from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
@ -21,7 +21,9 @@ from litellm.constants import REDACTED_BY_LITELLM
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.classifier_logging import without_classifier_audit
from litellm.litellm_core_utils.core_helpers import (
attribute_or_none,
get_metadata_variable_name_from_kwargs,
has_attributes,
)
from litellm.litellm_core_utils.served_output_texts import SERVED_OUTPUT_TEXTS_KEY
from litellm.llms.base_llm.guardrail_translation.utils import stream_item_field, stream_item_items
@ -42,6 +44,11 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObject = Any
class _HasArguments(Protocol):
arguments: str
MESSAGE_REDACTION_ENABLE_HEADERS: Final = ("litellm-enable-message-redaction", "x-litellm-enable-message-redaction")
@ -163,7 +170,7 @@ def _redact_bedrock_guardrail_trace(response: object) -> None:
}
response.update(traces)
return
for field, value in ((name, getattr(response, name, None)) for name in BEDROCK_GUARDRAIL_TRACE_FIELDS):
for field, value in ((name, attribute_or_none(response, name)) for name in BEDROCK_GUARDRAIL_TRACE_FIELDS):
if value is not None:
setattr(response, field, _redacted_bedrock_trace(value))
@ -191,8 +198,8 @@ def _redact_tool_calls(tool_calls) -> None:
if not tool_calls:
return
for tool_call in tool_calls:
function = getattr(tool_call, "function", None)
if function is not None and hasattr(function, "arguments"):
function = attribute_or_none(tool_call, "function")
if function is not None and has_attributes(function, _HasArguments, "arguments"):
function.arguments = REDACTED_BY_LITELLM

38
litellm/litellm_core_utils/streaming_handler.py
View file

@ -7,7 +7,7 @@ import logging
import threading
import time
import traceback
from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence, Sized
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypeVar, cast
@ -17,7 +17,7 @@ import httpx
from openai.types.chat import ChatCompletionChunk
from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice
from openai.types.chat.chat_completion_chunk import ChoiceDelta
from pydantic import BaseModel, ValidationError
from pydantic import BaseModel, TypeAdapter, ValidationError
from typing_extensions import NotRequired, TypedDict
import litellm
@ -27,7 +27,9 @@ from litellm.constants import STREAM_CHUNK_CHOICE_FIELDS, STREAM_FINISH_CHOICE_F
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.core_helpers import (
UPSTREAM_USAGE_COST,
attribute_or_none,
chunk_usage,
decoded_json_object,
provider_reported_usage_cost,
stamp_litellm_bill_on_usage,
)
@ -129,6 +131,13 @@ def _apply_provider_fields(response: ModelResponseStream, fields: Mapping[str, o
setattr(response, key, value)
def _entry_len(entries: Mapping[str, object], key: str) -> int:
value: Final = entries[key]
if isinstance(value, Sized):
return len(value)
raise TypeError(f"object of type '{type(value).__name__}' has no len()")
def _prepend_upstream_choice_extras(
choice: StreamingChoices, upstream_choice: ChunkChoice, extra_keys: frozenset[str]
) -> None:
@ -346,6 +355,9 @@ class _ParsedChunkHiddenParams(LiteLLMBaseModel):
upstream_provider: str | None = None
_DECODED_JSON: Final = TypeAdapter(object)
def _provider_response_model(chunk: object) -> str | None:
model: Final[object] = chunk.get("model") if isinstance(chunk, Mapping) else getattr(chunk, "model", None)
return model if isinstance(model, str) and model else None
@ -432,9 +444,8 @@ class CustomStreamWrapper:
self._closed = False
self._stream_created_time: float = time.time()
litellm_params: Final[GenericLiteLLMParams] = GenericLiteLLMParams.model_validate(
dict(**self.logging_obj.model_call_details.get("litellm_params", {}))
)
raw_litellm_params: Final = dict[str, object](**self.logging_obj.model_call_details.get("litellm_params", {}))
litellm_params: Final[GenericLiteLLMParams] = GenericLiteLLMParams.model_validate(raw_litellm_params)
self.merge_reasoning_content_in_choices: bool = litellm_params.merge_reasoning_content_in_choices or False
self.sent_first_thinking_block = False
self.sent_last_thinking_block = False
@ -465,7 +476,7 @@ class CustomStreamWrapper:
_api_base: Final = get_api_base(
model=model or "",
optional_params=self.logging_obj.model_call_details.get("litellm_params", {}),
optional_params=raw_litellm_params,
)
self._hidden_params: dict[str, object] = {
@ -953,7 +964,7 @@ class CustomStreamWrapper:
if "text_output" in chunk:
response = CustomStreamWrapper.strip_sse_data_from_chunk(chunk) or ""
response = response.strip()
parsed_response = json.loads(response)
parsed_response = decoded_json_object(_DECODED_JSON.validate_python(json.loads(response)))
else:
return {
"text": "",
@ -1087,7 +1098,7 @@ class CustomStreamWrapper:
def is_chunk_non_empty(
self,
completion_obj: dict[str, Any],
completion_obj: Mapping[str, object],
model_response: ModelResponseStream,
response_obj: Mapping[str, object],
) -> bool:
@ -1097,7 +1108,7 @@ class CustomStreamWrapper:
or (
"tool_calls" in completion_obj
and completion_obj["tool_calls"] is not None
and len(completion_obj["tool_calls"]) > 0
and _entry_len(completion_obj, "tool_calls") > 0
)
or ("function_call" in completion_obj and completion_obj["function_call"] is not None)
or (
@ -1850,7 +1861,9 @@ class CustomStreamWrapper:
elif delta.tool_calls is not None:
if isinstance(delta.tool_calls, list):
for t in delta.tool_calls:
if hasattr(t, "functions") and hasattr(getattr(t, "functions"), "arguments"):
if hasattr(t, "functions") and hasattr(
attribute_or_none(t, "functions"), "arguments"
):
if t.function.arguments is None:
t.function.arguments = ""
_json_delta: Final[_DeltaDumpDict] = delta.model_dump()
@ -1888,7 +1901,7 @@ class CustomStreamWrapper:
return
## CHECK FOR TOOL USE
if "tool_calls" in completion_obj and len(completion_obj["tool_calls"]) > 0:
if "tool_calls" in completion_obj and _entry_len(completion_obj, "tool_calls") > 0:
if self.is_function_call is True: # user passed in 'functions' param
completion_obj["function_call"] = completion_obj["tool_calls"][0]["function"]
completion_obj["tool_calls"] = None
@ -2817,8 +2830,9 @@ class CustomStreamWrapper:
def _record_and_strip_usage_chunk(self, chunk: ModelResponseStream) -> ModelResponseStream | None:
self.chunks.append(chunk.model_copy())
dumped_chunk: Final = MappingProxyType[str, object](chunk.model_dump())
usage_free_chunk: Final = self.model_response_creator(
chunk={key: value for key, value in chunk.model_dump().items() if key != "usage"},
chunk={key: value for key, value in dumped_chunk.items() if key != "usage"},
hidden_params=cast( # cast-ok: preserve mapping operations on dynamic storage
Mapping[str, object], getattr(chunk, HIDDEN_PARAMS_ATTR)
),

77
litellm/llms/a2a/chat/guardrail_translation/handler.py
View file

@ -11,13 +11,15 @@ A2A Protocol Format:
"""
import json
from collections.abc import Sequence
from typing import TYPE_CHECKING, Any, Final, Optional
from collections.abc import Mapping, Sequence
from functools import reduce
from typing import TYPE_CHECKING, Any, Final, Optional, Protocol, runtime_checkable
from pydantic import ConfigDict, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import has_attributes, require_attributes, subscript
from litellm.llms.base_llm.guardrail_translation.base_translation import (
BaseTranslation,
StreamingScanKey,
@ -30,6 +32,38 @@ if TYPE_CHECKING:
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
_UNSUPPORTED_RESPONSE: Final = object()
class _ModelDumpable(Protocol):
def model_dump(self) -> object: ...
class _ResultLookup(Protocol):
def get(self, key: str, default: object, /) -> object: ...
@runtime_checkable
class _SupportsSetItem(Protocol):
def __setitem__(self, key: str, value: object, /) -> None: ...
def _item_assignment_target(value: object) -> _SupportsSetItem:
if isinstance(value, _SupportsSetItem):
return value
raise TypeError(f"'{type(value).__name__}' object does not support item assignment")
def _response_payload(response: object) -> object:
if has_attributes(response, _ModelDumpable, "model_dump"):
return response.model_dump()
if isinstance(response, dict):
return response
return _UNSUPPORTED_RESPONSE
def _path_step(current: object, key: str) -> object:
return subscript(current, int(key) if key.isdigit() else key)
class _A2ATextPart(TypedDict, total=False):
@ -123,7 +157,7 @@ class A2AGuardrailHandler(BaseTranslation):
async def process_output_response(
self,
response: Any,
response: object,
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
@ -147,17 +181,12 @@ class A2AGuardrailHandler(BaseTranslation):
Returns:
Modified response with guardrails applied to text content
"""
# Handle both dict and Pydantic model responses
if hasattr(response, "model_dump"):
response_dict = response.model_dump()
is_pydantic = True
elif isinstance(response, dict):
response_dict = response
is_pydantic = False
else:
payload: Final = _response_payload(response)
if payload is _UNSUPPORTED_RESPONSE:
verbose_proxy_logger.warning("A2A: Unknown response type %s, skipping guardrail", type(response))
return response
response_dict: Final = require_attributes(payload, _ResultLookup, "get")
result: Final = response_dict.get("result", {})
if not result or not isinstance(result, dict):
verbose_proxy_logger.debug("A2A: No result in response, skipping guardrail")
@ -218,15 +247,8 @@ class A2AGuardrailHandler(BaseTranslation):
verbose_proxy_logger.debug("A2A: Processed output response")
# Update the original response
if is_pydantic:
# For Pydantic models, we need to update the underlying dict
# and the model will reflect the changes
response_dict["result"] = result
return response
else:
response["result"] = result
return response
_item_assignment_target(response_dict)["result"] = result
return response
async def process_output_streaming_response(
self,
@ -448,20 +470,11 @@ class A2AGuardrailHandler(BaseTranslation):
def _apply_text_to_path(
self,
result: dict[str | int, Any],
result: Mapping[str, object],
path: tuple[str, ...],
part_idx: int,
text: str,
) -> None:
"""Apply guardrailed text back to the specified path in the result."""
# Navigate to the parts list
current = result
for key in path:
if key.isdigit():
# Array index
current = current[int(key)]
else:
current = current[key]
# Update the text in the part
current[part_idx]["text"] = text
parts: Final = reduce(_path_step, path, result)
_item_assignment_target(subscript(parts, part_idx))["text"] = text

2
litellm/llms/anthropic/chat/handler.py
View file

@ -1081,7 +1081,7 @@ class ModelResponseIterator:
if "caller" in content_block_start["content_block"]:
caller_data: Final = content_block_start["content_block"]["caller"]
if caller_data and tool_use is not None:
tool_use["caller"] = cast(dict[str, Any], caller_data)
tool_use["caller"] = caller_data
elif content_block_start["content_block"]["type"] == "redacted_thinking":
(
thinking_blocks,

4
litellm/llms/anthropic/common_utils.py
View file

@ -2107,7 +2107,7 @@ def _strip_empty_content_blocks_from_message(message: object) -> object:
def strip_empty_content_blocks_from_anthropic_messages(
messages: Sequence[object],
) -> list[Any]:
) -> Sequence[object]:
"""
Return a new message list with empty or whitespace-only ``{"type": "text"}``
content blocks and empty *unsigned* ``{"type": "thinking"}`` blocks removed.
@ -2220,7 +2220,7 @@ def _sanitize_tool_use_id_content_block(block: object) -> object:
return block
def sanitize_tool_use_ids_in_anthropic_messages(messages: Sequence[object]) -> list[Any]:
def sanitize_tool_use_ids_in_anthropic_messages(messages: Sequence[object]) -> Sequence[object]:
"""
Return a new message list with ``tool_use`` / ``server_tool_use`` ``id`` and
``tool_result`` ``tool_use_id`` values rewritten to satisfy Anthropic's

14
litellm/llms/anthropic/messages/native/agentic_streaming_iterator.py
View file

@ -17,11 +17,14 @@ execute it.
import asyncio
import contextlib
import json
from collections.abc import AsyncIterator
from collections.abc import AsyncIterable, AsyncIterator
from typing import TYPE_CHECKING, Any, Final, cast
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES
from litellm.litellm_core_utils.core_helpers import decoded_json_object, has_attributes
from litellm.types.integrations.custom_logger import AgenticLoopLimitError, AgenticLoopSafetyError
if TYPE_CHECKING:
@ -42,6 +45,7 @@ AGENTIC_LOOP_LIMIT_ERROR_SSE_BYTES: Final = (
b'"The agentic loop reached its limit before this turn finished. '
b'Raise max_agentic_loops to allow more loops."}}\n\n'
)
_DECODED_JSON: Final = TypeAdapter(object)
def is_server_fulfilled_tool_leak_error(chunk: object) -> bool:
@ -76,10 +80,10 @@ def _parse_sse_events(raw: bytes) -> list[tuple]:
continue
data_str = stripped[len("data:") :].strip()
try:
data = json.loads(data_str)
data = _DECODED_JSON.validate_python(json.loads(data_str))
except (json.JSONDecodeError, ValueError):
continue
event_type = current_event_type or data.get("type", "")
event_type = current_event_type or decoded_json_object(data).get("type", "")
current_event_type = None
events.append((event_type, data))
return events
@ -371,7 +375,7 @@ class AgenticAnthropicStreamingIterator:
verbose_logger.debug("AgenticStreamingIterator: Could not rebuild response from SSE bytes")
return
result: Final = await self._http_handler._call_agentic_completion_hooks(
result: Final[object] = await self._http_handler._call_agentic_completion_hooks(
response=rebuilt,
model=self._model,
messages=self._messages,
@ -386,7 +390,7 @@ class AgenticAnthropicStreamingIterator:
if result is None:
return
if hasattr(result, "__aiter__"):
if has_attributes(result, AsyncIterable[object], "__aiter__"):
self._follow_up_iterator = result.__aiter__()
elif isinstance(result, dict):
from litellm.llms.anthropic.messages.native.fake_stream_iterator import (

15
litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py
View file

@ -2,8 +2,8 @@
## Translates OpenAI call to Anthropic `/v1/messages` format
import asyncio
from collections import deque
from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final
from collections.abc import AsyncGenerator, AsyncIterable, Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
from typing import TYPE_CHECKING, Final
import anyio
from pydantic import ConfigDict, field_validator
@ -12,6 +12,7 @@ from litellm import verbose_logger
from litellm._logging import redact_internal_details_from_client_message
from litellm._uuid import uuid
from litellm.exceptions import APIError, MidStreamFallbackError
from litellm.litellm_core_utils.core_helpers import has_attributes, iter_items
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
encrypted_reasoning_signature,
)
@ -126,7 +127,7 @@ class AnthropicResponsesStreamWrapper:
def __init__(
self,
responses_stream: Any,
responses_stream: object,
model: str,
litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
safeguards_evaluator: "SafeguardsEvaluator | None" = None,
@ -414,7 +415,9 @@ class AnthropicResponsesStreamWrapper:
response_obj: Final = getattr(event, "response", None) or (
event.get("response") if isinstance(event, dict) else None
)
output: Final = (getattr(response_obj, "output", None) or ()) if response_obj is not None else ()
output: Final[Iterable[object]] = (
(getattr(response_obj, "output", None) or ()) if response_obj is not None else ()
)
refusal_text: Final = responses_output_refusal_text(output) or (self._refusal_text or None)
status: Final = getattr(response_obj, "status", None) if response_obj is not None else None
has_tool_call: Final = any(
@ -508,14 +511,14 @@ class AnthropicResponsesStreamWrapper:
return self._chunk_queue.popleft()
try:
if hasattr(self.responses_stream, "__aiter__"):
if has_attributes(self.responses_stream, AsyncIterable[object], "__aiter__"):
async for event in self.responses_stream:
self._process_event(event)
if self._chunk_queue:
return self._chunk_queue.popleft()
else:
if self._sync_responses_iterator is None:
self._sync_responses_iterator = iter(self.responses_stream)
self._sync_responses_iterator = iter(iter_items(self.responses_stream))
sync_iterator: Final = self._sync_responses_iterator
missing: Final = object()
while (event := await asyncio.to_thread(next, sync_iterator, missing)) is not missing:

17
litellm/llms/anthropic/messages/responses_adapters/transformation.py
View file

@ -10,6 +10,8 @@ from collections.abc import Iterable, Mapping, Sequence
from itertools import groupby
from typing import Any, Final, cast
from pydantic import TypeAdapter
from litellm.litellm_core_utils.prompt_templates.common_utils import (
TOOL_RESULT_IMAGE_BOUNDARY,
TOOL_RESULT_IMAGE_PLACEHOLDER,
@ -58,6 +60,7 @@ from litellm.types.llms.openai import (
REASONING_SUMMARY_PART_SEPARATOR: Final = "\n\n"
RESPONSES_INCLUDE_ENCRYPTED_REASONING: Final = "reasoning.encrypted_content"
_DECODED_JSON: Final = TypeAdapter(object)
class LiteLLMAnthropicToResponsesAPIAdapter:
@ -805,15 +808,17 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
content.append(reasoning_block)
elif item_type == "function_call":
try:
input_data = json.loads(item.get("arguments", "{}"))
input_data = _DECODED_JSON.validate_python(json.loads(item.get("arguments", "{}")))
except (json.JSONDecodeError, TypeError):
input_data = {}
content.append(
AnthropicResponseContentBlockToolUse(
type="tool_use",
id=item.get("call_id") or item.get("id", ""),
name=item.get("name", ""),
input=input_data,
AnthropicResponseContentBlockToolUse.model_validate(
{
"type": "tool_use",
"id": item.get("call_id") or item.get("id", ""),
"name": item.get("name", ""),
"input": input_data,
}
).model_dump(exclude_none=True)
)
stop_reason = "tool_use"

36
litellm/llms/anthropic/messages/stream_assembly.py
View file

@ -4,10 +4,13 @@ from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping, map_finish_reason
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object, map_finish_reason
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import is_str_keyed_dict
from litellm.llms.anthropic.chat.handler import (
ModelResponseIterator as AnthropicModelResponseIterator,
)
@ -21,6 +24,8 @@ from litellm.types.utils import (
TextCompletionResponse,
)
_DECODED_JSON: Final = TypeAdapter(object)
def split_sse_chunk_into_events(chunk: str | bytes) -> list[str]:
"""
@ -98,6 +103,14 @@ _FAST_PATH_DISALLOWED_DELTA_TYPES: Final = frozenset(
)
def _joined_text(parts: Sequence[object]) -> str:
strings: Final = [part for part in parts if isinstance(part, str)]
if len(strings) == len(parts):
return "".join(strings)
first_bad: Final = next(position for position, part in enumerate(parts) if not isinstance(part, str))
raise TypeError(f"sequence item {first_bad}: expected str instance, {type(parts[first_bad]).__name__} found")
def collapse_pure_text_chunks(
all_chunks: Sequence[str | bytes],
) -> list[str] | None:
@ -120,10 +133,10 @@ def collapse_pure_text_chunks(
if ev:
normalized.append(ev)
text_block_indexes: Final[set] = set()
text_block_indexes: Final[set[object]] = set()
out: Final[list[str]] = []
pending_text: list[str] = []
pending_index: int | None = None
pending_text: list[object] = []
pending_index: object = None
saw_any_text_delta = False
def flush() -> None:
@ -132,7 +145,7 @@ def collapse_pure_text_chunks(
merged: Final = {
"type": "content_block_delta",
"index": pending_index if pending_index is not None else 0,
"delta": {"type": "text_delta", "text": "".join(pending_text)},
"delta": {"type": "text_delta", "text": _joined_text(pending_text)},
}
out.append("data: " + json.dumps(merged))
pending_text = []
@ -158,20 +171,20 @@ def collapse_pure_text_chunks(
if not json_str.startswith("{"):
continue
try:
data = json.loads(json_str)
data = decoded_json_object(_DECODED_JSON.validate_python(json.loads(json_str)))
except (json.JSONDecodeError, ValueError):
return None
etype = data.get("type")
if etype == "content_block_start":
block = data.get("content_block") or {}
block = decoded_json_object(data.get("content_block") or {})
if block.get("type") != "text":
return None
text_block_indexes.add(data.get("index"))
flush()
out.append(ev)
elif etype == "content_block_delta":
delta = data.get("delta") or {}
delta = decoded_json_object(data.get("delta") or {})
dtype = delta.get("type")
if dtype in _FAST_PATH_DISALLOWED_DELTA_TYPES:
return None
@ -291,9 +304,10 @@ def extract_sse_data(event_str: str) -> dict | None:
if payload is None or not payload.startswith("{"):
return None
try:
return json.loads(payload)
decoded: Final = _DECODED_JSON.validate_python(json.loads(payload))
except (ValueError, TypeError):
return None
return decoded if is_str_keyed_dict(decoded) else None
def build_usage_only_response_from_chunks(
@ -492,7 +506,7 @@ def _carries_no_data(fields: Sequence[tuple[str, str]]) -> bool:
def _json_mapping(payload: str) -> Mapping[str, object] | None:
try:
value: Final[object] = json.loads(payload)
value: Final = _DECODED_JSON.validate_python(json.loads(payload))
except ValueError:
return None
return as_str_mapping(value)
@ -555,7 +569,7 @@ def is_anthropic_sse_stream(all_chunks: Sequence[object]) -> bool:
def _is_json_no_reader_can_take(payload: str) -> bool:
try:
value: Final[object] = json.loads(payload)
value: Final = _DECODED_JSON.validate_python(json.loads(payload))
except ValueError:
return False
if value is None:

29
litellm/llms/apiserpent/search/transformation.py
View file

@ -12,7 +12,9 @@ from typing import Final, Literal, cast
from urllib.parse import urlencode
import httpx
from pydantic import TypeAdapter
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object, iter_items
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.apiserpent.search.defaults import (
@ -33,6 +35,7 @@ from litellm.secret_managers.main import get_secret_str
DEEP_SEARCH_PARAM: Final = "deep"
APISERPENT_BASE: Final = "https://apiserpent.com"
APISERPENT_PARAMS_KEY: Final = "_apiserpent_params"
_DECODED_JSON: Final = TypeAdapter(object)
class APISerpentSearchConfig(BaseSearchConfig):
@ -150,22 +153,24 @@ class APISerpentSearchConfig(BaseSearchConfig):
Full format nests results under ``results.organic[]``; simple format
returns a flat ``results[]`` array. Both expose title/url/snippet.
"""
response_json: Final = raw_response.json()
response_json: Final = decoded_json_object(_DECODED_JSON.validate_python(raw_response.json()))
raw_results: Final = response_json.get("results") or {}
organic: Final = raw_results.get("organic", []) if isinstance(raw_results, dict) else raw_results
nested_results: Final = as_str_mapping(raw_results)
organic: Final = nested_results.get("organic", []) if nested_results is not None else raw_results
results: Final[list[SearchResult]] = []
for result in organic:
results.append(
SearchResult(
title=result.get("title", ""),
url=result.get("url", ""),
snippet=result.get("snippet", ""),
date=result.get("date"),
last_updated=None,
)
results: Final = [
SearchResult.model_validate(
{
"title": result.get("title", ""),
"url": result.get("url", ""),
"snippet": result.get("snippet", ""),
"date": result.get("date"),
"last_updated": None,
}
)
for result in map(decoded_json_object, iter_items(organic))
]
return SearchResponse(
results=results,

12
litellm/llms/azure/azure.py
View file

@ -2,7 +2,7 @@ import asyncio
import json
import time
from collections.abc import Callable, Coroutine
from typing import Final
from typing import Final, Protocol
import httpx
from openai import (
@ -17,6 +17,7 @@ from pydantic import BaseModel
import litellm
from litellm.constants import DEFAULT_MAX_RETRIES
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.logging_utils import speech_request_body, track_llm_api_timing
from litellm.llms.custom_httpx.http_handler import (
@ -60,6 +61,11 @@ from .image_generation import (
from .image_generation.http_utils import azure_deployment_image_generation_json_body
class _HasResponseText(Protocol):
@property
def text(self) -> str: ...
class AzureOpenAIAssistantsAPIConfig:
"""
Reference: https://learn.microsoft.com/en-us/azure/ai-services/openai/assistants-reference-messages?tabs=python#create-message
@ -853,11 +859,11 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
except Exception as e:
status_code: Final = getattr(e, "status_code", 500)
error_headers = getattr(e, "headers", None)
error_response: Final = getattr(e, "response", None)
error_response: Final[object] = getattr(e, "response", None)
error_text = str(e)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
error_text = error_response.text
error_text = require_attributes(error_response, _HasResponseText, "text").text
raise AzureOpenAIError(
status_code=status_code,
message=error_text,

5
litellm/llms/azure/responses/transformation.py
View file

@ -251,9 +251,8 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
def compact_url(self, api_base: str) -> str:
params: Final = httpx.URL(api_base).params
v1_params: Final = (
params if BaseAzureLLM.is_azure_v1_api_version(params.get("api-version")) else params.remove("api-version")
)
api_version: Final = next(iter(params.get_list("api-version")), None)
v1_params: Final = params if BaseAzureLLM.is_azure_v1_api_version(api_version) else params.remove("api-version")
return super().compact_url(str(_v1_responses_url(api_base).copy_with(params=v1_params)))
#########################################################

6
litellm/llms/azure_ai/agents/handler.py
View file

@ -308,7 +308,7 @@ class AzureAIAgentsHandler:
logging_obj: LiteLLMLoggingObj,
optional_params: dict,
litellm_params: dict,
timeout: float,
timeout: float | str | httpx.Timeout | None,
client: HTTPHandler | None = None,
headers: dict | None = None,
) -> ModelResponse:
@ -432,7 +432,7 @@ class AzureAIAgentsHandler:
logging_obj: LiteLLMLoggingObj,
optional_params: dict,
litellm_params: dict,
timeout: float,
timeout: float | str | httpx.Timeout | None,
client: AsyncHTTPHandler | None = None,
headers: dict | None = None,
) -> ModelResponse:
@ -559,7 +559,7 @@ class AzureAIAgentsHandler:
logging_obj: LiteLLMLoggingObj,
optional_params: dict,
litellm_params: dict,
timeout: float,
timeout: float | str | httpx.Timeout | None,
headers: dict | None = None,
) -> AsyncIterator[ModelResponseStream]:
"""Execute async streaming completion using Azure Agent Service with native SSE."""

2
litellm/llms/azure_ai/agents/transformation.py
View file

@ -323,7 +323,7 @@ class AzureAIAgentsConfig(BaseConfig):
logging_obj: LiteLLMLoggingObj,
optional_params: dict,
litellm_params: dict,
timeout: float | Any,
timeout: float | str | httpx.Timeout | None,
acompletion: bool,
stream: bool | None = False,
headers: dict | None = None,

8
litellm/llms/azure_ai/anthropic/count_tokens/handler.py
View file

@ -8,7 +8,7 @@ from collections.abc import Mapping
from typing import Final
import httpx
from pydantic import JsonValue
from pydantic import JsonValue, TypeAdapter
import litellm
from litellm._logging import verbose_logger
@ -22,6 +22,8 @@ from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
)
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
_DECODED_JSON: Final = TypeAdapter(object)
class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
"""
@ -42,7 +44,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
system: object = None,
optional_params: Mapping[str, JsonValue] | None = None,
context_management: JsonValue = None,
) -> dict[str, object]:
) -> object:
"""
Handle a CountTokens request using httpx with Azure authentication.
@ -126,7 +128,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
message=error_text,
)
azure_response: Final = response.json()
azure_response: Final = _DECODED_JSON.validate_python(response.json())
verbose_logger.debug("Azure AI Anthropic response: %s", azure_response)

6
litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py
View file

@ -11,6 +11,7 @@ from pydantic import JsonValue
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import decoded_json_object
from litellm.llms.anthropic.count_tokens.transformation import (
AnthropicCountFields,
ChatCountTranslationError,
@ -139,13 +140,14 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
)
if result is not None:
response_fields: Final = decoded_json_object(result)
return TokenCountResponse.model_validate(
{
"total_tokens": result.get("input_tokens", 0),
"total_tokens": response_fields.get("input_tokens", 0),
"request_model": request_model,
"model_used": model_to_use,
"tokenizer_type": "azure_ai_anthropic_api",
"original_response": result,
"original_response": response_fields,
}
)
except AnthropicError as e:

10
litellm/llms/azure_ai/anthropic/messages_transformation.py
View file

@ -3,7 +3,7 @@ Azure Anthropic messages transformation config - extends AnthropicMessagesConfig
"""
from collections.abc import Sequence
from typing import Any, Final
from typing import Final
from litellm.llms.anthropic.messages.native.transformation import (
AnthropicMessagesConfig,
@ -109,14 +109,10 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig):
Processes both `system` and `messages` content blocks.
"""
def _sanitize(cache_control: Any) -> None:
if isinstance(cache_control, dict):
cache_control.pop("scope", None)
def _process_content_list(content: list) -> None:
for item in content:
if isinstance(item, dict) and "cache_control" in item:
_sanitize(item["cache_control"])
if isinstance(item, dict) and "cache_control" in item and isinstance(item["cache_control"], dict):
item["cache_control"].pop("scope", None)
if "system" in anthropic_messages_request:
system: Final = anthropic_messages_request["system"]

13
litellm/llms/azure_ai/image_edit/flux2_transformation.py
View file

@ -2,11 +2,12 @@ import base64
from collections.abc import Mapping, Sequence
from io import BufferedReader
from types import MappingProxyType
from typing import Any, Final
from typing import Final, Protocol
from httpx._types import RequestFiles
import litellm
from litellm.litellm_core_utils.core_helpers import has_attributes
from litellm.llms.azure_ai.common_utils import (
AzureFoundryModelInfo,
get_azure_ai_auth_headers,
@ -21,6 +22,10 @@ from litellm.types.llms.openai import FileTypes
from litellm.types.router import GenericLiteLLMParams
class _BinaryReadable(Protocol):
def read(self) -> bytes: ...
class AzureFoundryFlux2ImageEditConfig(OpenAIImageEditConfig):
"""
Azure AI Foundry FLUX 2 image edit config
@ -100,7 +105,7 @@ class AzureFoundryFlux2ImageEditConfig(OpenAIImageEditConfig):
if image is None:
raise ValueError("FLUX 2 image edit requires an image.")
images: Final = tuple(image) if isinstance(image, list) else (image,)
images: Final[Sequence[object]] = [*image] if isinstance(image, list) else [image]
if not images:
raise ValueError("FLUX 2 image edit requires at least one image.")
max_reference_images: Final = 10 if "flex" in model.lower() else 8
@ -121,14 +126,14 @@ class AzureFoundryFlux2ImageEditConfig(OpenAIImageEditConfig):
}
return request_body, []
def _convert_image_to_base64(self, image: Any) -> str:
def _convert_image_to_base64(self, image: object) -> str:
"""Convert image file to base64 string"""
if isinstance(image, BufferedReader):
image_bytes = image.read()
image.seek(0) # Reset file pointer for potential reuse
elif isinstance(image, bytes):
image_bytes = image
elif hasattr(image, "read"):
elif has_attributes(image, _BinaryReadable, "read"):
image_bytes = image.read()
else:
raise ValueError(f"Unsupported image type: {type(image)}")

16
litellm/llms/base_llm/guardrail_translation/base_translation.py
View file

@ -1,7 +1,9 @@
from abc import ABC, abstractmethod
from collections.abc import Sequence
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional
from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional, Protocol
from litellm.litellm_core_utils.core_helpers import has_attributes
if TYPE_CHECKING:
from fastapi import HTTPException
@ -15,6 +17,10 @@ if TYPE_CHECKING:
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
class _ModelDumpable(Protocol):
def model_dump(self) -> object: ...
@dataclass(slots=True)
class StreamTransformSink:
"""Out-parameter used by ``process_output_streaming_response`` to hand the
@ -81,7 +87,7 @@ class BaseTranslation(ABC):
@staticmethod
def transform_user_api_key_dict_to_metadata(
user_api_key_dict: Any | None,
user_api_key_dict: object,
) -> dict[str, object]:
"""
Transform user_api_key_dict to a metadata dict with prefixed keys.
@ -99,7 +105,11 @@ class BaseTranslation(ABC):
return {}
# Convert to dict if it's a Pydantic object
user_dict = user_api_key_dict.model_dump() if hasattr(user_api_key_dict, "model_dump") else user_api_key_dict
user_dict: Final = (
user_api_key_dict.model_dump()
if has_attributes(user_api_key_dict, _ModelDumpable, "model_dump")
else user_api_key_dict
)
if not isinstance(user_dict, dict):
return {}

14
litellm/llms/base_llm/managed_resources/isolation.py
View file

@ -11,8 +11,9 @@ are denied so an empty user_id can never select an unscoped query.
"""
from collections.abc import Mapping, Sequence
from typing import Any, Final
from typing import Final, Protocol
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.types.proxy.auth.user_api_key_auth import (
UserAPIKeyAuth,
)
@ -47,15 +48,20 @@ def resolve_resource_owner_id(
return None
def build_list_page(items: list[Any], has_more: bool = False) -> dict[str, object]:
class _Identified(Protocol):
@property
def id(self) -> object: ...
def build_list_page(items: Sequence[object], has_more: bool = False) -> dict[str, object]:
"""Build the OpenAI-style paginated list response shape used by managed
file/batch/vector-store listings. ``first_id`` and ``last_id`` are
sourced from each item's ``.id`` attribute."""
return {
"object": "list",
"data": items,
"first_id": items[0].id if items else None,
"last_id": items[-1].id if items else None,
"first_id": require_attributes(items[0], _Identified, "id").id if items else None,
"last_id": require_attributes(items[-1], _Identified, "id").id if items else None,
"has_more": has_more,
}

15
litellm/llms/bedrock/chat/invoke_handler.py
View file

@ -11,7 +11,7 @@ from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.caching.caching import InMemoryCache
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.core_helpers import decoded_json_object, map_finish_reason
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_mapping
from litellm.litellm_core_utils.llm_response_utils.get_headers import provider_request_id_headers
@ -50,6 +50,7 @@ from litellm.types.utils import GenericStreamingChunk as GChunk
from ..common_utils import (
BedrockError,
BedrockEventStreamResponseDict,
bedrock_event_chunk_text,
bedrock_event_stream_header,
bedrock_event_stream_response,
bedrock_stream_event_error_status,
@ -59,6 +60,7 @@ from ..common_utils import (
get_bedrock_response_stream_shape,
get_bedrock_stream_event_statuses,
get_bedrock_tool_name,
parse_bedrock_event_stream,
)
bedrock_tool_name_mappings: Final[InMemoryCache] = InMemoryCache(max_size_in_memory=50, default_ttl=600)
@ -922,7 +924,7 @@ class AWSEventStreamDecoder:
response_stream_shape: "Shape",
request_id_headers: httpx.Headers | None = None,
) -> str | None:
parsed_response: Final = self.parser.parse(response_dict, response_stream_shape)
parsed_response: Final = parse_bedrock_event_stream(self.parser, response_dict, response_stream_shape)
if response_dict["status_code"] != 200:
raise build_bedrock_stream_error(response_dict, response_stream_shape, headers=request_id_headers)
@ -932,11 +934,12 @@ class AWSEventStreamDecoder:
raise build_bedrock_stream_event_error(
event_type, event_error_status, response_dict["body"], headers=request_id_headers
)
if "chunk" in parsed_response:
chunk = parsed_response.get("chunk")
if not chunk:
parsed_fields: Final = decoded_json_object(parsed_response)
if "chunk" in parsed_fields:
event_chunk: Final = parsed_fields.get("chunk")
if not event_chunk:
return None
return chunk.get("bytes").decode()
return bedrock_event_chunk_text(event_chunk)
else:
chunk = response_dict.get("body")
if not chunk:

32
litellm/llms/bedrock/common_utils.py
View file

@ -13,7 +13,7 @@ import warnings
from collections.abc import Callable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias
from typing_extensions import (
ReadOnly,
@ -24,6 +24,7 @@ from typing_extensions import (
if TYPE_CHECKING:
from botocore.eventstream import EventStreamMessage
from botocore.model import ServiceModel, Shape
from botocore.parsers import EventStreamJSONParser
from litellm.types.llms.bedrock import BedrockCreateBatchRequest
@ -33,7 +34,7 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError
import litellm
from litellm import verbose_logger
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.litellm_core_utils.core_helpers import lax_bool
from litellm.litellm_core_utils.core_helpers import decoded_json_object, lax_bool, require_attributes
from litellm.litellm_core_utils.optional_imports import ensure_optional_import
from litellm.litellm_core_utils.prompt_templates.json_schema_utils import (
schema_property_count,
@ -135,7 +136,7 @@ _BEDROCK_AWS_AUTH_PARAMETER_KEYS: Final[tuple[str, ...]] = (*AWS_AUTH_PARAM_KEYS
def merge_bedrock_aws_request_params(
litellm_params: Mapping[str, object],
optional_params: Mapping[str, object],
) -> dict[str, Any]:
) -> dict[str, object]:
"""Merge deployment and request parameters without allowing auth escalation.
Deployment configuration is authoritative for AWS authentication. When a
@ -2516,15 +2517,16 @@ class BedrockEventStreamDecoderBase:
),
)
response_dict: Final = event.to_response_dict()
parsed_response: Final = self.parser.parse(response_dict, response_stream_shape)
parsed_response: Final = parse_bedrock_event_stream(self.parser, response_dict, response_stream_shape)
if response_dict["status_code"] != 200:
raise build_bedrock_stream_error(response_dict, response_stream_shape)
if "chunk" in parsed_response:
chunk = parsed_response.get("chunk")
if not chunk:
parsed_fields: Final = decoded_json_object(parsed_response)
if "chunk" in parsed_fields:
event_chunk: Final = parsed_fields.get("chunk")
if not event_chunk:
return None
return chunk.get("bytes").decode()
return bedrock_event_chunk_text(event_chunk)
else:
chunk = response_dict.get("body")
if not chunk:
@ -2541,6 +2543,20 @@ def _decoded_json_value(raw: str) -> object:
return _JSON_VALUE.validate_python(json.loads(raw))
class _DecodablePayload(Protocol):
def decode(self) -> str: ...
def parse_bedrock_event_stream(
parser: EventStreamJSONParser, response_dict: Mapping[str, object], response_stream_shape: Shape
) -> object:
return _JSON_VALUE.validate_python(parser.parse(response_dict, response_stream_shape))
def bedrock_event_chunk_text(chunk: object) -> str:
return require_attributes(decoded_json_object(chunk).get("bytes"), _DecodablePayload, "decode").decode()
def get_anthropic_beta_from_headers(headers: dict) -> list[str]:
"""
Extract anthropic-beta header values and convert them to a list.

2
litellm/llms/bedrock/count_tokens/bedrock_token_counter.py
View file

@ -45,7 +45,7 @@ class _CountTokensReply(LiteLLMPydanticObjectBase):
@dataclass(frozen=True, slots=True)
class CountedTokens:
input_tokens: int
original_response: Mapping[str, JsonValue]
original_response: Mapping[str, object]
tokenizer_type: str

10
litellm/llms/bedrock/count_tokens/handler.py
View file

@ -7,14 +7,18 @@ Simplified handler leveraging existing LiteLLM Bedrock infrastructure.
from typing import Any, Final
import httpx
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import decoded_json_object
from litellm.llms.bedrock.base_aws_llm import run_aws_signing
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensConfig
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client
_DECODED_JSON: Final = TypeAdapter(object)
class BedrockCountTokensHandler(BedrockCountTokensConfig):
"""
@ -29,7 +33,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
litellm_params: dict[str, Any],
resolved_model: str,
client: AsyncHTTPHandler | None = None,
) -> dict[str, Any]:
) -> dict[str, object]:
"""
Handle a CountTokens request using existing LiteLLM patterns.
@ -111,12 +115,12 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
response=response,
)
bedrock_response: Final = response.json()
bedrock_response: Final = _DECODED_JSON.validate_python(response.json())
verbose_logger.debug("Bedrock response: %s", bedrock_response)
# Transform response back to expected format
final_response: Final = self.transform_bedrock_response_to_anthropic(bedrock_response)
final_response: Final = self.transform_bedrock_response_to_anthropic(decoded_json_object(bedrock_response))
verbose_logger.debug("Final response: %s", final_response)

4
litellm/llms/bedrock/count_tokens/transformation.py
View file

@ -247,9 +247,7 @@ class BedrockCountTokensConfig(BaseAWSLLM):
)
return join_api_base(base_url, f"model/{encoded_model_id}/count-tokens")
def transform_bedrock_response_to_anthropic(
self, bedrock_response: Mapping[str, JsonValue]
) -> dict[str, JsonValue]:
def transform_bedrock_response_to_anthropic(self, bedrock_response: Mapping[str, object]) -> dict[str, object]:
"""
Transform Bedrock CountTokens response to Anthropic format.

25
litellm/llms/bedrock/files/transformation.py
View file

@ -11,7 +11,7 @@ from datetime import datetime
from email.utils import parsedate_to_datetime
from itertools import chain
from types import MappingProxyType
from typing import Any, Final, Literal, TypeAlias, TypedDict
from typing import Any, Final, Literal, Protocol, TypeAlias, TypedDict
from urllib.parse import quote, unquote, urlencode
import httpx
@ -19,6 +19,7 @@ from httpx import Headers, Response
from openai.types.file_deleted import FileDeleted
from pydantic import ConfigDict, Field, TypeAdapter
from typing_extensions import ReadOnly
from typing_extensions import TypedDict as ExtraItemsTypedDict
from litellm._logging import verbose_logger
from litellm._uuid import uuid
@ -38,6 +39,7 @@ from litellm.litellm_core_utils.cloud_storage_security import (
split_configured_cloud_bucket_name,
validate_managed_cloud_file_id,
)
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.litellm_core_utils.prompt_templates.common_utils import (
text_completion_prompt_to_messages,
)
@ -151,6 +153,10 @@ def _retrieved_s3_file_size(raw_response: Response) -> int:
)
class _SupportsRstrip(Protocol):
def rstrip(self, chars: str, /) -> str: ...
class _S3DeleteContext(LiteLLMBaseModel):
file_id: str = Field(min_length=1)
@ -168,6 +174,10 @@ _LITELLM_PARAMS_THE_MAPPER_TAKES: Final = frozenset({"allowed_openai_params"})
_MAPPED_PARAMS_THE_REQUEST_HANDLER_STRIPS: Final = frozenset({"json_mode"})
class _ParamsTheMapperOwns(ExtraItemsTypedDict, total=False, extra_items=object):
pass
_UPLOAD_PARAMS: Final = TypeAdapter(dict[str, object])
@ -710,9 +720,11 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
encoded_object_name: Final = encode_s3_object_key_for_url(object_name)
# S3 endpoint URL format
s3_endpoint_url: Final = (
s3_endpoint_url: Final = require_attributes(
request_params.get("s3_endpoint_url")
or f"https://s3.{aws_region_name}.{get_aws_dns_suffix(aws_region_name)}"
or f"https://s3.{aws_region_name}.{get_aws_dns_suffix(aws_region_name)}",
_SupportsRstrip,
"rstrip",
).rstrip("/")
return f"{s3_endpoint_url}/{bucket_name}/{encoded_object_name}"
@ -1062,7 +1074,8 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
from litellm.types.utils import LlmProviders
messages: Final = map_developer_role_to_system_role(openai_request_body.get("messages", []))
optional_params: Final = {k: v for k, v in openai_request_body.items() if k not in ["model", "messages"]}
request_params: Final[Mapping[str, object]] = openai_request_body
optional_params: Final = {k: v for k, v in request_params.items() if k not in ["model", "messages"]}
# --- Anthropic: use existing AmazonAnthropicClaudeConfig ---
if provider == LlmProviders.ANTHROPIC:
@ -1085,8 +1098,8 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
custom_llm_provider="bedrock",
messages=messages,
model_info=deployment_model_info(litellm_params),
**MappingProxyType(
{
**_ParamsTheMapperOwns(
**{
k: v
for k, v in optional_params.items()
if not is_litellm_owned_kwarg(k) or k in _LITELLM_PARAMS_THE_MAPPER_TAKES

17
litellm/llms/bedrock/image_edit/amazon_nova_canvas_image_edit_transformation.py
View file

@ -18,8 +18,10 @@ from typing import TYPE_CHECKING, Any, Final, cast
import httpx
from httpx._types import RequestFiles
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import decoded_json_object, iter_items
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.types.images.main import ImageEditOptionalRequestParams
@ -38,6 +40,8 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObj = Any
_DECODED_JSON: Final = TypeAdapter(object)
def _nova_canvas_task_body(
*,
@ -401,7 +405,7 @@ class BedrockAmazonNovaCanvasImageEditConfig(BaseImageEditConfig):
json_mode: bool | None = None,
) -> ImageResponse:
try:
response_data: Final = raw_response.json()
response_data: Final = _DECODED_JSON.validate_python(raw_response.json())
except Exception as e:
raise self.get_error_class(
error_message=f"Error parsing Nova Canvas image edit response: {e}",
@ -416,18 +420,19 @@ class BedrockAmazonNovaCanvasImageEditConfig(BaseImageEditConfig):
headers=raw_response.headers,
)
images: Final[list[str]] = response_data.get("images") or []
response_object: Final = decoded_json_object(response_data)
images: Final[object] = response_object.get("images") or []
if "errors" in response_data and not images:
if "errors" in response_object and not images:
raise self.get_error_class(
error_message=f"Nova Canvas image edit error: {response_data['errors']}",
error_message=f"Nova Canvas image edit error: {response_object['errors']}",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
# Nova Canvas InvokeModel success body uses "images" and optional "error" (AWS docs);
# it does not use Stability-style "finish_reasons".
error_msg = response_data.get("message") or response_data.get("error")
error_msg = response_object.get("message") or response_object.get("error")
if error_msg and not images:
if not isinstance(error_msg, str):
error_msg = str(error_msg)
@ -439,7 +444,7 @@ class BedrockAmazonNovaCanvasImageEditConfig(BaseImageEditConfig):
model_response: Final = ImageResponse()
model_response.data = []
for image_b64 in images:
for image_b64 in iter_items(images):
if image_b64:
model_response.data.append(
ImageObject(

34
litellm/llms/bedrock/image_edit/stability_transformation.py
View file

@ -22,11 +22,13 @@ API Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/model-parame
"""
import base64
from typing import TYPE_CHECKING, Any, Final, cast
from typing import TYPE_CHECKING, Any, Final, Protocol, cast, runtime_checkable
import httpx
from httpx._types import RequestFiles
from pydantic import TypeAdapter
from litellm.litellm_core_utils.core_helpers import decoded_json_object, iter_items, json_contains
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.types.images.main import ImageEditOptionalRequestParams
@ -44,6 +46,19 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObj = Any
_DECODED_JSON: Final = TypeAdapter(object)
@runtime_checkable
class _SupportsGetItem(Protocol):
def __getitem__(self, key: str | int, /) -> object: ...
def _subscript(value: object, key: str | int) -> object:
if isinstance(value, _SupportsGetItem):
return value[key]
raise TypeError(f"'{type(value).__name__}' object is not subscriptable")
class BedrockStabilityImageEditConfig(BaseImageEditConfig):
"""
@ -299,7 +314,7 @@ class BedrockStabilityImageEditConfig(BaseImageEditConfig):
OpenAI expects: {"data": [{"b64_json": "base64..."}], "created": timestamp}
"""
try:
response_data: Final = raw_response.json()
response_data: Final = _DECODED_JSON.validate_python(raw_response.json())
except Exception as e:
raise self.get_error_class(
error_message=f"Error parsing Bedrock Stability response: {e}",
@ -308,18 +323,19 @@ class BedrockStabilityImageEditConfig(BaseImageEditConfig):
)
# Check for errors in response
if "errors" in response_data:
if json_contains(response_data, "errors"):
raise self.get_error_class(
error_message=f"Bedrock Stability error: {response_data['errors']}",
error_message=f"Bedrock Stability error: {_subscript(response_data, 'errors')}",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
# Check finish_reasons
finish_reasons: Final = response_data.get("finish_reasons", [])
if finish_reasons and finish_reasons[0]:
response_fields: Final = decoded_json_object(response_data)
finish_reasons: Final = response_fields.get("finish_reasons", [])
if finish_reasons and _subscript(finish_reasons, 0):
raise self.get_error_class(
error_message=f"Bedrock Stability error: {finish_reasons[0]}",
error_message=f"Bedrock Stability error: {_subscript(finish_reasons, 0)}",
status_code=400,
headers=raw_response.headers,
)
@ -329,9 +345,9 @@ class BedrockStabilityImageEditConfig(BaseImageEditConfig):
model_response.data = []
# Extract images from response
images: Final = response_data.get("images", [])
images: Final = response_fields.get("images", [])
if images:
for image_b64 in images:
for image_b64 in iter_items(images):
if image_b64:
model_response.data.append(
ImageObject(

5
litellm/llms/bedrock/passthrough/guardrail_translation/handler.py
View file

@ -14,7 +14,7 @@ from pydantic import InstanceOf, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object
from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler
from litellm.llms.base_llm.guardrail_translation.base_translation import (
BaseTranslation,
@ -849,6 +849,7 @@ class InvokeStreamFrame:
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
_DECODED_JSON: Final = TypeAdapter(object)
_Frame: Final = TypeVar("_Frame", ConverseStreamFrame, InvokeStreamFrame)
@ -1298,7 +1299,7 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
texts: Sequence[tuple[_StreamGroupKey, str]] = ()
if event_type == "contentBlockDelta":
try:
payload_dict: dict[str, object] = json.loads(payload_bytes)
payload_dict = decoded_json_object(_DECODED_JSON.validate_python(json.loads(payload_bytes)))
texts = _delta_frame_texts(payload_dict)
except Exception as e:
verbose_proxy_logger.debug(

54
litellm/llms/bedrock/vector_stores/transformation.py
View file

@ -1,11 +1,12 @@
from collections.abc import Mapping
from copy import deepcopy
from typing import TYPE_CHECKING, Any, Final, cast
from typing import TYPE_CHECKING, Any, Final, Protocol, cast, runtime_checkable
from urllib.parse import urlparse
import httpx
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import decoded_json_object
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
@ -34,6 +35,32 @@ else:
LiteLLMLoggingObj = Any
@runtime_checkable
class _SupportsSetDefault(Protocol):
def setdefault(self, key: str, default: object, /) -> object: ...
@runtime_checkable
class _SupportsSetItem(Protocol):
def __setitem__(self, key: str, value: object, /) -> None: ...
def _setdefault_target(value: object) -> _SupportsSetDefault:
if isinstance(value, _SupportsSetDefault):
return value
raise AttributeError(f"{type(value).__name__!r} object has no attribute 'setdefault'", name="setdefault", obj=value)
def _item_assignment_target(value: object) -> _SupportsSetItem:
if isinstance(value, _SupportsSetItem):
return value
raise TypeError(f"'{type(value).__name__}' object does not support item assignment")
def _vector_search_configuration(retrieval_config: object) -> Mapping[str, object]:
return decoded_json_object(decoded_json_object(retrieval_config).get("vectorSearchConfiguration", {}))
class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
"""Vector store configuration for AWS Bedrock Knowledge Bases."""
@ -206,7 +233,7 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
api_base: str,
litellm_logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
extra_body: dict[str, Any] | None = None,
extra_body: dict[str, object] | None = None,
) -> tuple[str, dict]:
if isinstance(query, list):
query = " ".join(query)
@ -218,30 +245,33 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
"retrievalQuery": BedrockKBRetrievalQuery(text=query),
}
retrieval_config: dict[str, Any] = {}
if isinstance(extra_body, dict):
retrieval_config = deepcopy(
extra_body.get("retrievalConfiguration") or extra_body.get("retrieval_configuration") or {}
)
retrieval_config: Final[object] = (
deepcopy(extra_body.get("retrievalConfiguration") or extra_body.get("retrieval_configuration") or {})
if isinstance(extra_body, dict)
else {}
)
max_results: Final = vector_store_search_optional_params.get("max_num_results")
if max_results is not None:
existing_number_of_results = retrieval_config.get("vectorSearchConfiguration", {}).get("numberOfResults")
existing_number_of_results: Final = _vector_search_configuration(retrieval_config).get("numberOfResults")
if existing_number_of_results is not None and existing_number_of_results != max_results:
verbose_logger.debug(
"Overriding extra_body retrievalConfiguration.vectorSearchConfiguration.numberOfResults (%s) with max_num_results=%s",
existing_number_of_results,
max_results,
)
retrieval_config.setdefault("vectorSearchConfiguration", {})["numberOfResults"] = max_results
_item_assignment_target(_setdefault_target(retrieval_config).setdefault("vectorSearchConfiguration", {}))[
"numberOfResults"
] = max_results
filters: Final = vector_store_search_optional_params.get("filters")
if filters is not None:
existing_filter: Final = retrieval_config.get("vectorSearchConfiguration", {}).get("filter")
existing_filter: Final = _vector_search_configuration(retrieval_config).get("filter")
if existing_filter is not None and existing_filter != filters:
verbose_logger.debug(
"Overriding extra_body retrievalConfiguration.vectorSearchConfiguration.filter with filters from vector_store_search_optional_params"
)
retrieval_config.setdefault("vectorSearchConfiguration", {})["filter"] = filters
_item_assignment_target(_setdefault_target(retrieval_config).setdefault("vectorSearchConfiguration", {}))[
"filter"
] = filters
if retrieval_config:
request_body["retrievalConfiguration"] = cast(BedrockKBRetrievalConfiguration, retrieval_config)
user_context: Final = self._user_context(extra_body=extra_body, litellm_params=litellm_params)

34
litellm/llms/black_forest_labs/image_edit/transformation.py
View file

@ -10,7 +10,7 @@ API Reference: https://docs.bfl.ai/
import base64
import time
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Any, Final, Protocol
import httpx
from httpx._types import RequestFiles
@ -19,6 +19,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from litellm.litellm_core_utils.core_helpers import as_object_list, has_attributes
from litellm.litellm_core_utils.url_utils import async_safe_get, join_api_base, safe_get
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.secret_managers.main import get_secret_str
@ -52,6 +53,19 @@ class _BflPolledResult(TypedDict):
_BFL_POLLED_RESULT_ADAPTER: Final = TypeAdapter(_BflPolledResult)
_DECODED_JSON: Final = TypeAdapter(object)
_UNTYPED_VALUE: Final = TypeAdapter(object)
class _BinaryReadable(Protocol):
def read(self) -> bytes: ...
class _Tellable(Protocol):
def tell(self) -> int: ...
class _Seekable(Protocol):
def seek(self, offset: int, /) -> object: ...
_BFL_REQUEST_PARAMS: Final = (
@ -197,7 +211,7 @@ class BlackForestLabsImageEditConfig(BaseImageEditConfig):
def _read_image_bytes(
self,
image: Any,
image: object,
depth: int = 0,
max_depth: int = DEFAULT_MAX_RECURSE_DEPTH,
) -> bytes:
@ -208,9 +222,8 @@ class BlackForestLabsImageEditConfig(BaseImageEditConfig):
)
if isinstance(image, bytes):
return image
elif isinstance(image, list):
# If it's a list, take the first image
return self._read_image_bytes(image[0], depth=depth + 1, max_depth=max_depth)
elif (image_list := as_object_list(image)) is not None:
return self._read_image_bytes(image_list[0], depth=depth + 1, max_depth=max_depth)
elif isinstance(image, str):
if image.startswith(("http://", "https://")):
response: Final = safe_get(litellm.module_level_client, image, timeout=60.0)
@ -221,13 +234,12 @@ class BlackForestLabsImageEditConfig(BaseImageEditConfig):
"Unsupported image input: plain string values that are not URLs are not accepted. "
"Provide image bytes or a file-like object."
)
elif hasattr(image, "read"):
# File-like object
pos: Final = getattr(image, "tell", lambda: 0)()
if hasattr(image, "seek"):
elif has_attributes(image, _BinaryReadable, "read"):
pos: Final = image.tell() if has_attributes(image, _Tellable, "tell") else 0
if has_attributes(image, _Seekable, "seek"):
image.seek(0)
data: Final = image.read()
if hasattr(image, "seek"):
if has_attributes(image, _Seekable, "seek"):
image.seek(pos)
return data
else:
@ -265,7 +277,7 @@ class BlackForestLabsImageEditConfig(BaseImageEditConfig):
# Handle mask if provided (for inpainting)
if "mask" in image_edit_optional_request_params:
mask: Final = image_edit_optional_request_params["mask"]
mask: Final = _UNTYPED_VALUE.validate_python(image_edit_optional_request_params["mask"])
mask_bytes: Final = self._read_image_bytes(mask)
request_body["mask"] = base64.b64encode(mask_bytes).decode("utf-8")

8
litellm/llms/bytez/chat/transformation.py
View file

@ -6,7 +6,9 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final
import httpx
from pydantic import TypeAdapter
from litellm.litellm_core_utils.core_helpers import decoded_json_object
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
from litellm.litellm_core_utils.url_utils import encode_url_path_segments
@ -36,6 +38,8 @@ else:
# 5 minute timeout (models may need to load)
STREAMING_TIMEOUT: Final = 60 * 5
_DECODED_JSON: Final = TypeAdapter(object)
class BytezChatConfig(BaseConfig):
"""
@ -194,11 +198,11 @@ class BytezChatConfig(BaseConfig):
) -> ModelResponse:
json: Final = raw_response.json()
error: Final = json.get("error")
error: Final = decoded_json_object(_DECODED_JSON.validate_python(json)).get("error")
if error is not None:
raise BytezError(
message=str(json["error"]),
message=str(error),
status_code=raw_response.status_code,
)

10
litellm/llms/cohere/common_utils.py
View file

@ -1,6 +1,8 @@
import json
from typing import Final, Literal
from pydantic import TypeAdapter
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.cohere import CohereV2StreamedToolCall
@ -12,6 +14,8 @@ from litellm.types.utils import (
ProviderSpecificModelInfo,
)
_DECODED_JSON: Final = TypeAdapter(object)
class CohereError(BaseLLMException):
def __init__(self, status_code, message):
@ -190,8 +194,10 @@ class ModelResponseIterator:
if index != -1:
str_line = str_line[index:]
data_json: Final = json.loads(str_line)
return self.chunk_parser(chunk=data_json)
data_json: Final = _DECODED_JSON.validate_python(json.loads(str_line))
if isinstance(data_json, dict):
return self.chunk_parser(chunk=data_json)
raise AttributeError(f"{type(data_json).__name__!r} object has no attribute 'get'", name="get", obj=data_json)
# Async iterator
def __aiter__(self):

38
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -153,7 +153,7 @@ from litellm.types.containers.main import (
)
from litellm.types.decisions import DecisionsIRRequest, DecisionsIRResponse
from litellm.types.extraction import ExtractionResponse
from litellm.types.files import StreamingMediaUploadConfig, TwoStepFileUploadConfig
from litellm.types.files import StreamingMediaUploadConfig, TwoStepFileUploadConfig, TwoStepFileUploadRequest
from litellm.types.integrations.custom_logger import (
NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
AgenticLoopLimitError,
@ -188,6 +188,7 @@ from litellm.types.responses.main import DeleteResponseResult
from litellm.types.responses.streaming_websocket import ResponsesWebSocketRequestDefaults
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import (
AgenticLoopParams,
CallTypes,
EmbeddingResponse,
FileTypes,
@ -239,6 +240,10 @@ def _client_request_headers(
return _merge_headers_case_insensitive(optional_headers or {}, extra_headers)
def _requested_stream(response_api_optional_request_params: Mapping[str, object]) -> object:
return response_api_optional_request_params.get("stream", False)
def _rust_responses_websocket_enabled(
custom_llm_provider: str | None,
) -> bool:
@ -386,12 +391,19 @@ class _SignedBodyKwargs(TypedDict, total=False):
json: ReadOnly[dict[str, object]]
class _AgenticLoopParamsView(TypedDict):
agentic_loop_params: ReadOnly[AgenticLoopParams]
_GOAWAY_RETRY_MESSAGE: Final = "Vector store search connection was closed by an HTTP/2 GOAWAY, retrying once"
def _closed_by_http2_goaway(error: httpx.RemoteProtocolError) -> bool:
cause: Final = error.__cause__
return cause is not None and any(isinstance(arg, ConnectionTerminated) for arg in cause.args)
if cause is None:
return False
cause_args: Final[tuple[object, ...]] = cause.args
return any(isinstance(arg, ConnectionTerminated) for arg in cause_args)
async def _asend_with_goaway_retry(send: Callable[[], Awaitable[httpx.Response]]) -> httpx.Response:
@ -433,8 +445,12 @@ def _signed_body_kwargs(*, signed_body: bytes | None, data: dict[str, object]) -
return {"json": data}
def _response_content_type(headers: Mapping[str, str]) -> str:
return headers.get("content-type", "")
def _answered_json(response: httpx.Response) -> bool:
return "application/json" in response.headers.get("content-type", "").lower()
return "application/json" in _response_content_type(response.headers).lower()
def _stream_query_params(stream: bool, starting_after: int | None) -> Mapping[str, object]:
@ -3064,7 +3080,7 @@ class BaseLLMHTTPHandler:
}
# Check if streaming is requested
stream = response_api_optional_request_params.get("stream", False)
stream = _requested_stream(response_api_optional_request_params)
caller_requested_stream: Final = bool(stream or (extra_body or {}).get("stream"))
api_base: Final = responses_api_provider_config.get_complete_url(
@ -4136,7 +4152,7 @@ class BaseLLMHTTPHandler:
):
# Handle pre-signed requests (e.g., from Bedrock S3 uploads)
# Type narrowing: this is a plain dict, not TwoStepFileUploadConfig
presigned_request: Final = cast(dict[str, Any], transformed_request)
presigned_request: Final = cast(TwoStepFileUploadRequest, transformed_request)
upload_response = getattr(sync_httpx_client, presigned_request["method"].lower())(
url=presigned_request["url"],
headers=presigned_request["headers"],
@ -4350,7 +4366,7 @@ class BaseLLMHTTPHandler:
):
# Handle pre-signed requests (e.g., from Bedrock S3 uploads)
# Type narrowing: this is a plain dict, not TwoStepFileUploadConfig
presigned_request: Final = cast(dict[str, Any], transformed_request)
presigned_request: Final = cast(TwoStepFileUploadRequest, transformed_request)
upload_response = await getattr(async_httpx_client, presigned_request["method"].lower())(
url=presigned_request["url"],
headers=presigned_request["headers"],
@ -6129,8 +6145,10 @@ class BaseLLMHTTPHandler:
full_model_name = model
if logging_obj is not None:
agentic_params: Final[Mapping[str, object]] = logging_obj.model_call_details.get("agentic_loop_params", {})
full_model_name = cast(str, agentic_params.get("model", model))
agentic_view: Final[_AgenticLoopParamsView] = {
"agentic_loop_params": logging_obj.model_call_details.get("agentic_loop_params", {})
}
full_model_name = agentic_view["agentic_loop_params"].get("model", model)
optional_params: Final = dict(anthropic_messages_optional_request_params)
optional_params.update(patch.optional_params)
@ -11116,7 +11134,7 @@ class BaseLLMHTTPHandler:
extra_body: dict[str, object] | None = None,
timeout: float | httpx.Timeout | None = None,
client: HTTPHandler | AsyncHTTPHandler | None = None,
):
) -> Mapping[str, object]:
if client is None or not isinstance(client, AsyncHTTPHandler):
async_httpx_client = get_async_httpx_client(
llm_provider=litellm.LlmProviders(custom_llm_provider),
@ -11407,7 +11425,7 @@ class BaseLLMHTTPHandler:
extra_body: dict[str, object] | None = None,
timeout: float | httpx.Timeout | None = None,
client: HTTPHandler | AsyncHTTPHandler | None = None,
):
) -> Mapping[str, object]:
if client is None or not isinstance(client, AsyncHTTPHandler):
async_httpx_client = get_async_httpx_client(
llm_provider=litellm.LlmProviders(custom_llm_provider),

22
litellm/llms/dashscope/embed/transformation.py
View file

@ -11,12 +11,13 @@ Endpoint
Docs - https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api
"""
from typing import Final
from typing import Final, Protocol, runtime_checkable
import httpx
from pydantic import ConfigDict, TypeAdapter, with_config
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.litellm_core_utils.core_helpers import json_contains
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@ -29,6 +30,19 @@ from ..common_utils import DashScopeError, resolve_dashscope_openai_api_base
DEFAULT_API_BASE: Final = "https://dashscope.aliyuncs.com/compatible-mode/v1"
_DECODED_JSON: Final = TypeAdapter(object)
@runtime_checkable
class _SupportsGetItem(Protocol):
def __getitem__(self, key: str, /) -> object: ...
def _subscript(value: object, key: str) -> object:
if isinstance(value, _SupportsGetItem):
return value[key]
raise TypeError(f"'{type(value).__name__}' object is not subscriptable")
@with_config(ConfigDict(extra="allow", strict=True))
class _DashScopeUsage(TypedDict, total=False):
@ -149,7 +163,7 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
litellm_params: dict,
) -> EmbeddingResponse:
try:
response_json: Final = raw_response.json()
response_json: Final = _DECODED_JSON.validate_python(raw_response.json())
except Exception as e:
raise DashScopeError(
status_code=raw_response.status_code,
@ -163,8 +177,8 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
original_response=response_json,
)
if "error" in response_json:
error: Final[object] = response_json["error"]
if json_contains(response_json, "error"):
error: Final = _subscript(response_json, "error")
message: Final = error.get("message", str(error)) if isinstance(error, dict) else str(error)
raise DashScopeError(
status_code=raw_response.status_code,

33
litellm/llms/dataforseo/search/transformation.py
View file

@ -4,13 +4,14 @@ Calls DataForSEO SERP API to search the web.
DataForSEO API Reference: https://docs.dataforseo.com/v3/serp/google/organic/live/advanced/?bash
"""
from collections.abc import Iterable, Mapping
from typing import Final, Literal
from collections.abc import Iterable, Mapping, Sized
from typing import Final, Literal, Protocol, runtime_checkable
import httpx
from pydantic import ConfigDict, TypeAdapter
from litellm.constants import DEFAULT_DATAFORSEO_LOCATION_CODE
from litellm.litellm_core_utils.core_helpers import decoded_json_object, json_contains
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.search.transformation import (
BaseSearchConfig,
@ -21,6 +22,24 @@ from litellm.secret_managers.main import get_secret_str
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
_JSON_VALUES: Final = TypeAdapter(Iterable[object], config=ConfigDict(hide_input_in_errors=True))
_DECODED_JSON: Final = TypeAdapter(object)
@runtime_checkable
class _SupportsGetItem(Protocol):
def __getitem__(self, key: str | int, /) -> object: ...
def _subscript(value: object, key: str | int) -> object:
if isinstance(value, _SupportsGetItem):
return value[key]
raise TypeError(f"'{type(value).__name__}' object is not subscriptable")
def _json_len(value: object) -> int:
if isinstance(value, Sized):
return len(value)
raise TypeError(f"object of type '{type(value).__name__}' has no len()")
class DataForSEOSearchConfig(BaseSearchConfig):
@ -193,20 +212,20 @@ class DataForSEOSearchConfig(BaseSearchConfig):
Returns:
SearchResponse with standardized format
"""
response_json: Final = raw_response.json()
response_json: Final = _DECODED_JSON.validate_python(raw_response.json())
# Transform results to SearchResult objects
results: Final = []
# DataForSEO wraps results in tasks array
if "tasks" in response_json and len(response_json["tasks"]) > 0:
task: Final = response_json["tasks"][0]
if json_contains(response_json, "tasks") and _json_len(_subscript(response_json, "tasks")) > 0:
task: Final = decoded_json_object(_subscript(_subscript(response_json, "tasks"), 0))
# Check if task was successful
if task.get("status_code") == 20000 and "result" in task:
# Result is an array, take first element
if len(task["result"]) > 0:
result: Final = _JSON_OBJECT.validate_python(task["result"][0])
if _json_len(task["result"]) > 0:
result: Final = _JSON_OBJECT.validate_python(_subscript(task["result"], 0))
# Items contain the actual search results
for raw_item in _JSON_VALUES.validate_python(result.get("items", [])):

34
litellm/llms/gemini/common_utils.py
View file

@ -4,10 +4,11 @@ import json
import math
from collections.abc import Iterable, Mapping, Sequence
from types import MappingProxyType
from typing import Any, Final, cast
from typing import Final, cast
import httpx
from pydantic import JsonValue, TypeAdapter
from typing_extensions import TypedDict as ExtraItemsTypedDict
import litellm
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
@ -510,6 +511,13 @@ def get_api_key_from_env() -> str | None:
return get_secret_str("GOOGLE_API_KEY") or get_secret_str("GEMINI_API_KEY")
_COUNT_TOKENS_REQUEST_FIELDS: Final = frozenset({"model", "contents", "system_instruction", "tools"})
class _ForwardedLitellmParams(ExtraItemsTypedDict, total=False, extra_items=object):
pass
class GoogleAIStudioTokenCounter(BaseTokenCounter):
"""Token counter implementation for Google AI Studio provider."""
@ -526,7 +534,7 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
model_to_use: str,
messages: list[dict[str, object]] | None,
contents: list[dict[str, object]] | None,
deployment: dict[str, Any] | None = None,
deployment: dict[str, object] | None = None,
request_model: str = "",
tools: list[dict[str, object]] | None = None,
system: object | None = None,
@ -537,6 +545,7 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
) -> TokenCountResponse | None:
import copy
from litellm.litellm_core_utils.core_helpers import decoded_json_object
from litellm.llms.anthropic.messages.context_management.inference_view import (
InferenceRequestViewError,
count_context_edits,
@ -553,11 +562,12 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
vertex_gemini_count_context,
)
litellm_params: Final = decoded_json_object((deployment or {}).get("litellm_params", {}))
request_view: Final = await inference_request_view(
messages or [],
system,
messages_format,
count_context_edits(context_management, tools, (deployment or {}).get("litellm_params", {}), model_to_use),
count_context_edits(context_management, tools, litellm_params, model_to_use),
)
if isinstance(request_view, InferenceRequestViewError):
return count_context_error_response(request_view, request_model, model_to_use)
@ -570,8 +580,7 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
if blocks_error is not None:
return count_context_error_response(blocks_error, request_model, model_to_use)
deployment = deployment or {}
count_tokens_params_request: Final = copy.deepcopy(deployment.get("litellm_params", {}))
count_tokens_params_request: Final = copy.deepcopy(litellm_params)
resolved_contents: Final = (
contents
if contents is not None
@ -581,16 +590,15 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
)
if isinstance(resolved_contents, VertexGeminiCountContextError):
return count_context_error_response(resolved_contents, request_model, model_to_use)
count_tokens_params: Final = {
"model": model_to_use,
"contents": resolved_contents,
"system_instruction": count_context.system_instruction,
"tools": count_context.tools,
}
count_tokens_params_request.update(count_tokens_params)
try:
result: Final = await GoogleAIStudioTokenCounter().acount_tokens(
**count_tokens_params_request,
**_ForwardedLitellmParams(
**{k: v for k, v in count_tokens_params_request.items() if k not in _COUNT_TOKENS_REQUEST_FIELDS}
),
model=model_to_use,
contents=resolved_contents,
system_instruction=count_context.system_instruction,
tools=count_context.tools,
)
except litellm.APIError as error:
return provider_error_response(error, request_model, model_to_use)

55
litellm/llms/gemini/count_tokens/handler.py
View file

@ -1,11 +1,12 @@
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final
from collections.abc import Iterable, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, Protocol, runtime_checkable
import httpx
from pydantic import ConfigDict, RootModel, with_config
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm.litellm_core_utils.core_helpers import has_attributes, iter_items, json_contains
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client
from litellm.types.utils import LlmProviders
@ -26,8 +27,44 @@ class _GeminiCountTokensResponseRoot(RootModel[_GeminiCountTokensResponse]):
model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
@runtime_checkable
class _SupportsGetItem(Protocol):
def __getitem__(self, key: str, /) -> object: ...
@runtime_checkable
class _SupportsSetItem(Protocol):
def __setitem__(self, key: str, value: object, /) -> None: ...
class _KeywordArguments(Protocol):
def keys(self) -> Iterable[str]: ...
def __getitem__(self, key: str, /) -> object: ...
def _subscript(value: object, key: str) -> object:
if isinstance(value, _SupportsGetItem):
return value[key]
raise TypeError(f"'{type(value).__name__}' object is not subscriptable")
def _item_assignment_target(value: object) -> _SupportsSetItem:
if isinstance(value, _SupportsSetItem):
return value
raise TypeError(f"'{type(value).__name__}' object does not support item assignment")
def _function_response_kwargs(value: object) -> _KeywordArguments:
if has_attributes(value, _KeywordArguments, "keys"):
return value
raise TypeError(
f"google.genai.types.FunctionResponse() argument after ** must be a mapping, not {type(value).__name__}"
)
class GoogleAIStudioTokenCounter:
def _clean_contents_for_gemini_api(self, contents: Any) -> object:
def _clean_contents_for_gemini_api(self, contents: Sequence[Mapping[str, object]] | None) -> object:
"""
Clean up contents to remove unsupported fields for the Gemini API.
@ -52,12 +89,14 @@ class GoogleAIStudioTokenCounter:
for content in cleaned_contents:
parts = content["parts"]
for part in parts:
if "functionResponse" in part:
function_response_data = part["functionResponse"]
function_response_part = FunctionResponse(**function_response_data)
for part in iter_items(parts):
if json_contains(part, "functionResponse"):
function_response_data = _subscript(part, "functionResponse")
function_response_part = FunctionResponse(**_function_response_kwargs(function_response_data))
function_response_part.id = None
part["functionResponse"] = function_response_part.model_dump(exclude_none=True)
_item_assignment_target(part)["functionResponse"] = function_response_part.model_dump(
exclude_none=True
)
return cleaned_contents

19
litellm/llms/gemini/interactions/transformation.py
View file

@ -9,13 +9,14 @@ Per OpenAPI spec (https://ai.google.dev/static/api/interactions.openapi.json):
Requests use Api-Revision 2026-05-20 (`steps` schema).
"""
from collections.abc import Iterable, Mapping
from typing import TYPE_CHECKING, Any, Final, Protocol, TypeAlias
import httpx
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, process_response_headers
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, has_attributes, process_response_headers
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
from litellm.llms.base_llm.interactions.transformation import BaseInteractionsAPIConfig
from litellm.llms.gemini.common_utils import GeminiError, GeminiModelInfo
@ -83,6 +84,18 @@ class _CancelPayloadSource(Protocol):
def json(self) -> _CancelPayload: ...
class _ImageConfigItems(Protocol):
def keys(self) -> Iterable[str]: ...
def __getitem__(self, key: str, /) -> object: ...
def _image_config_items(image_config: object) -> _ImageConfigItems:
if has_attributes(image_config, _ImageConfigItems, "keys"):
return image_config
raise TypeError(f"'{type(image_config).__name__}' object is not a mapping")
def _interaction_body(response: _InteractionPayloadSource) -> _InteractionPayload:
"""Decode the body of an Interactions API interaction response."""
return response.json()
@ -241,7 +254,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
request_body["response_format"] = response_format
# image_config moves out of generation_config into response_format.
generation_config: dict[str, Any] | None = optional_params.get("generation_config")
generation_config: Mapping[str, object] | None = optional_params.get("generation_config")
if generation_config is not None:
image_config = None
if isinstance(generation_config, dict):
@ -255,7 +268,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
if image_config is not None:
# Move image_config to response_format with type=image.
image_rf: Final[_JsonObject] = {"type": "image", **image_config}
image_rf: Final[_JsonObject] = {"type": "image", **_image_config_items(image_config)}
existing_rf: Final = request_body.get("response_format")
if existing_rf is None:
request_body["response_format"] = image_rf

47
litellm/llms/gigachat/chat/streaming.py
View file

@ -4,9 +4,12 @@ GigaChat Streaming Response Handler
import json
import uuid
from collections.abc import Mapping, Sequence
from typing import Any, Final
from collections.abc import Awaitable, Mapping, Sequence
from typing import Final, Protocol
from pydantic import TypeAdapter
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.llms.gigachat.utils import convert_usage
from litellm.types.llms.openai import (
ChatCompletionToolCallChunk,
@ -14,13 +17,27 @@ from litellm.types.llms.openai import (
)
from litellm.types.utils import ChatCompletionUsageBlock, GenericStreamingChunk
_DECODED_JSON: Final = TypeAdapter(object)
class _SyncChunkSource(Protocol):
def __next__(self) -> object: ...
class _AsyncChunkSource(Protocol):
def __anext__(self) -> Awaitable[object]: ...
class _SupportsGet(Protocol):
def get(self, key: str, /) -> object: ...
class GigaChatModelResponseIterator:
"""Iterator for GigaChat streaming responses."""
def __init__(
self,
streaming_response: Any,
streaming_response: object,
sync_stream: bool,
json_mode: bool | None = False,
):
@ -28,7 +45,7 @@ class GigaChatModelResponseIterator:
self.response_iterator = self.streaming_response
self.json_mode = json_mode
def chunk_parser(self, chunk: Mapping[str, object]) -> GenericStreamingChunk:
def chunk_parser(self, chunk: _SupportsGet) -> GenericStreamingChunk:
"""Parse a single streaming chunk from GigaChat."""
choices: Sequence = chunk.get("choices") or ()
if not choices:
@ -106,14 +123,14 @@ class GigaChatModelResponseIterator:
def __next__(self) -> GenericStreamingChunk:
try:
chunk = self.response_iterator.__next__()
chunk: Final = require_attributes(self.response_iterator, _SyncChunkSource, "__next__").__next__()
if isinstance(chunk, str):
# Parse SSE format: data: {...}
chunk = chunk.removeprefix("data: ")
if chunk.strip() == "[DONE]":
payload: Final = chunk.removeprefix("data: ")
if payload.strip() == "[DONE]":
raise StopIteration
try:
chunk = json.loads(chunk)
decoded: Final = _DECODED_JSON.validate_python(json.loads(payload))
except json.JSONDecodeError:
return GenericStreamingChunk(
text="",
@ -123,7 +140,8 @@ class GigaChatModelResponseIterator:
usage=None,
index=0,
)
return self.chunk_parser(chunk)
return self.chunk_parser(require_attributes(decoded, _SupportsGet, "get"))
return self.chunk_parser(require_attributes(chunk, _SupportsGet, "get"))
except StopIteration:
raise
@ -132,14 +150,14 @@ class GigaChatModelResponseIterator:
async def __anext__(self) -> GenericStreamingChunk:
try:
chunk = await self.response_iterator.__anext__()
chunk: Final = await require_attributes(self.response_iterator, _AsyncChunkSource, "__anext__").__anext__()
if isinstance(chunk, str):
# Parse SSE format
chunk = chunk.removeprefix("data: ")
if chunk.strip() == "[DONE]":
payload: Final = chunk.removeprefix("data: ")
if payload.strip() == "[DONE]":
raise StopAsyncIteration
try:
chunk = json.loads(chunk)
decoded: Final = _DECODED_JSON.validate_python(json.loads(payload))
except json.JSONDecodeError:
return GenericStreamingChunk(
text="",
@ -149,6 +167,7 @@ class GigaChatModelResponseIterator:
usage=None,
index=0,
)
return self.chunk_parser(chunk)
return self.chunk_parser(require_attributes(decoded, _SupportsGet, "get"))
return self.chunk_parser(require_attributes(chunk, _SupportsGet, "get"))
except StopAsyncIteration:
raise

51
litellm/llms/huggingface/embedding/handler.py
View file

@ -1,11 +1,13 @@
import json
import os
from typing import Final, Literal, Protocol, get_args
from collections.abc import Iterator
from typing import Final, Literal, Protocol, get_args, runtime_checkable
import httpx
from pydantic import TypeAdapter
import litellm
from litellm.litellm_core_utils.core_helpers import as_object_list, iter_items, json_contains
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
@ -38,6 +40,28 @@ class _SupportsTokenEncode(Protocol):
def encode(self, text: str) -> list[int]: ...
@runtime_checkable
class _SupportsGetItem(Protocol):
def __getitem__(self, key: str | int, /) -> object: ...
def _subscript(value: object, key: str | int) -> object:
if isinstance(value, _SupportsGetItem):
return value[key]
raise TypeError(f"'{type(value).__name__}' object is not subscriptable")
def _unpack_pair(value: object) -> tuple[object, object]:
if not hasattr(value, "__iter__") and not hasattr(value, "__getitem__"):
raise TypeError(f"cannot unpack non-iterable {type(value).__name__} object")
first, second = iter_items(value)
return first, second
def _pairs(value: object) -> Iterator[tuple[object, object]]:
return (_unpack_pair(item) for item in iter_items(value))
def get_hf_task_embedding_for_model(
model: str, task_type: str | None, api_base: str, timeout: float | httpx.Timeout | None = None
) -> object:
@ -183,15 +207,15 @@ class HuggingFaceEmbedding(BaseLLM):
def _process_embedding_response(
self,
embeddings: dict,
embeddings: object,
model_response: EmbeddingResponse,
model: str,
input: list,
encoding: _SupportsTokenEncode,
) -> EmbeddingResponse:
output_data: Final = []
if "similarities" in embeddings:
for idx, embedding in embeddings["similarities"]:
if json_contains(embeddings, "similarities"):
for idx, embedding in _pairs(_subscript(embeddings, "similarities")):
output_data.append(
{
"object": "embedding",
@ -200,8 +224,9 @@ class HuggingFaceEmbedding(BaseLLM):
}
)
else:
for idx, embedding in enumerate(embeddings):
if isinstance(embedding, float) or isinstance(embedding, list) and isinstance(embedding[0], float):
for idx, embedding in enumerate(iter_items(embeddings)):
embedding_list = as_object_list(embedding)
if isinstance(embedding, float) or embedding_list is not None and isinstance(embedding_list[0], float):
output_data.append(
{
"object": "embedding",
@ -214,7 +239,7 @@ class HuggingFaceEmbedding(BaseLLM):
{
"object": "embedding",
"index": idx,
"embedding": embedding[0][0], # flatten list returned from hf
"embedding": _subscript(_subscript(embedding, 0), 0), # flatten list returned from hf
}
)
model_response.object = "list"
@ -288,10 +313,10 @@ class HuggingFaceEmbedding(BaseLLM):
original_response=response,
)
embeddings: Final = response.json()
embeddings: Final = _DECODED_JSON.validate_python(response.json())
if "error" in embeddings:
raise HuggingFaceError(status_code=500, message=embeddings["error"])
if json_contains(embeddings, "error"):
raise HuggingFaceError(status_code=500, message=_subscript(embeddings, "error"))
## PROCESS RESPONSE ##
return self._process_embedding_response(
@ -394,10 +419,10 @@ class HuggingFaceEmbedding(BaseLLM):
original_response=response,
)
embeddings: Final = response.json()
embeddings: Final = _DECODED_JSON.validate_python(response.json())
if "error" in embeddings:
raise HuggingFaceError(status_code=500, message=embeddings["error"])
if json_contains(embeddings, "error"):
raise HuggingFaceError(status_code=500, message=_subscript(embeddings, "error"))
## PROCESS RESPONSE ##
return self._process_embedding_response(

5
litellm/llms/huggingface/embedding/transformation.py
View file

@ -365,7 +365,10 @@ class HuggingFaceEmbeddingConfig(BaseConfig):
)
content: str = ""
for chunk in streamed_response:
content += chunk["choices"][0]["delta"]["content"]
delta_content = chunk.choices[0].delta.content
if delta_content is None:
raise TypeError('can only concatenate str (not "NoneType") to str')
content += delta_content
completion_response: Final[list[dict[str, str]]] = [{"generated_text": content}]
## LOGGING
logging_obj.post_call(

19
litellm/llms/litellm_proxy/skills/code_execution.py
View file

@ -15,11 +15,12 @@ import base64
import json
from collections.abc import Mapping, Sequence
from enum import Enum
from typing import Any, Final, Protocol, TypedDict
from typing import Final, Protocol, TypedDict
from typing_extensions import ReadOnly
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import require_attributes
class _ToolParameterSchema(TypedDict, total=False):
@ -237,17 +238,21 @@ class CodeExecutionHandler:
execution_results: Final[list[_ExecutionResult]] = []
executor: Final = SkillsSandboxExecutor(timeout=self.sandbox_timeout)
response: Any = None # Initialize to avoid possibly unbound error
response: _CompletionResponse | None = None # Initialize to avoid possibly unbound error
for iteration in range(self.max_iterations):
verbose_logger.debug("CodeExecutionHandler: Iteration %s/%s", iteration + 1, self.max_iterations)
# Make LLM call
response = await litellm.acompletion(
model=model,
messages=current_messages,
tools=tools,
**kwargs,
response = require_attributes(
await litellm.acompletion(
model=model,
messages=current_messages,
tools=tools,
**kwargs,
),
_CompletionResponse,
"choices",
)
choice: _ResponseChoice = response.choices[0]

6
litellm/llms/litellm_proxy/skills/transformation.py
View file

@ -10,8 +10,6 @@ Pattern follows litellm/llms/litellm_proxy/responses/transformation.py
from collections.abc import Coroutine, Sequence
from typing import TYPE_CHECKING, Final, Optional
from pydantic import JsonValue
from litellm.types.llms.anthropic_skills import (
DeleteSkillResponse,
ListSkillsResponse,
@ -47,7 +45,7 @@ class LiteLLMSkillsTransformationHandler:
file_content: bytes | None = None,
file_name: str | None = None,
file_type: str | None = None,
metadata: dict[str, JsonValue] | None = None,
metadata: dict[str, object] | None = None,
user_id: str | None = None,
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
_is_async: bool = False,
@ -130,7 +128,7 @@ class LiteLLMSkillsTransformationHandler:
file_content: bytes | None = None,
file_name: str | None = None,
file_type: str | None = None,
metadata: dict[str, JsonValue] | None = None,
metadata: dict[str, object] | None = None,
user_id: str | None = None,
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
) -> Skill:

9
litellm/llms/milvus/vector_stores/transformation.py
View file

@ -252,10 +252,11 @@ class MilvusVectorStoreConfig(BaseQueryEmbeddingVectorStoreConfig):
# Build attributes with all available metadata
# Exclude system fields and already-processed fields
attributes = {}
for key, value in result.items():
if key not in ["id", "content", "distance", text_field]:
attributes[key] = value
attributes = {
key: value
for key, value in decoded_json_object(_DECODED_JSON.validate_python(result)).items()
if key not in ["id", "content", "distance", text_field]
}
result_obj = VectorStoreSearchResult(
score=score,

12
litellm/llms/mistral/chat/transformation.py
View file

@ -21,6 +21,7 @@ from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import is_str_keyed_dict
from litellm.litellm_core_utils.prompt_templates.common_utils import (
convert_content_list_to_str,
strip_litellm_internal_message_fields,
@ -82,6 +83,7 @@ _MISTRAL_REQUEST_ROUTING: Final = TypeAdapter(MistralRequestRouting)
_MISTRAL_CACHE_KEY_PARAMS: Final = TypeAdapter(MistralCacheKeyParams)
_MISTRAL_REASONING_EFFORT_PARAMS: Final = TypeAdapter(MistralReasoningEffortParams)
_MISTRAL_FILE_PART: Final = TypeAdapter(MistralFilePart)
_DECODED_JSON: Final = TypeAdapter(object)
_REASONING_PROMPT_REASON: Final = "litellm sends Magistral a reasoning system prompt instead of the requested level."
_NON_LEVEL_EFFORT_REASON: Final = (
"the Mistral API takes it only as a level such as 'high', so Mistral applies its default effort."
@ -1005,9 +1007,13 @@ class MistralConfig(OpenAIGPTConfig):
logging_obj.model_call_details["response_headers"] = raw_response.headers
# Handle Mistral-specific response transformations
response_data = raw_response.json()
response_data = self._handle_empty_content_response(response_data)
response_data = self._handle_content_list_to_str_conversion(response_data)
decoded_response: Final = _DECODED_JSON.validate_python(raw_response.json())
if not is_str_keyed_dict(decoded_response):
missing_get: Final = f"{type(decoded_response).__name__!r} object has no attribute 'get'"
raise AttributeError(missing_get, name="get", obj=decoded_response)
response_data: Final = self._handle_content_list_to_str_conversion(
self._handle_empty_content_response(decoded_response)
)
final_response_obj: Final = cast(
ModelResponse,

2
litellm/llms/oci/chat/cohere.py
View file

@ -286,7 +286,7 @@ def handle_cohere_response(
def handle_cohere_stream_chunk(
dict_chunk: Mapping[str, JsonValue],
dict_chunk: Mapping[str, object],
prior_tool_calls_emitted: bool = False,
prior_text_emitted: bool = False,
) -> ModelResponseStream:

3
litellm/llms/oci/chat/generic.py
View file

@ -8,6 +8,7 @@ parsing, and streaming chunk parsing for models served with
import datetime
import hashlib
from collections.abc import Mapping
from typing import Final
import httpx
@ -363,7 +364,7 @@ def handle_generic_response(
return model_response
def handle_generic_stream_chunk(dict_chunk: dict) -> ModelResponseStream:
def handle_generic_stream_chunk(dict_chunk: Mapping) -> ModelResponseStream:
"""Parse a single GENERIC SSE chunk into a LiteLLM ModelResponseStream."""
# OCI streams tool calls progressively — early chunks may omit required fields.
if dict_chunk.get("message") and dict_chunk["message"].get("toolCalls"):

4
litellm/llms/oci/chat/transformation.py
View file

@ -19,6 +19,7 @@ from pydantic import TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_logger
from litellm.constants import DEFAULT_OCI_CHAT_MAX_TOKENS
from litellm.litellm_core_utils.core_helpers import decoded_json_object
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
from litellm.litellm_core_utils.prompt_templates.common_utils import (
merge_consecutive_system_messages,
@ -891,13 +892,14 @@ class OCIStreamWrapper(CustomStreamWrapper):
if payload == "[DONE]":
return None
try:
dict_chunk: Final = json.loads(payload)
decoded_chunk: Final = _DECODED_JSON.validate_python(json.loads(payload))
except json.JSONDecodeError as e:
raise OCIError(
status_code=500,
message=f"Chunk cannot be parsed as JSON: {e}",
)
dict_chunk: Final = decoded_json_object(decoded_chunk)
if dict_chunk.get("apiFormat") == "COHERE":
result: Final = handle_cohere_stream_chunk(
dict_chunk,

24
litellm/llms/ollama/common_utils.py
View file

@ -1,6 +1,7 @@
import base64
import io
from typing import Final, overload
from collections.abc import Iterable
from typing import Final, Protocol, overload
import httpx
from openai.types.chat import ChatCompletionChunk
@ -8,6 +9,7 @@ from pydantic import BaseModel, TypeAdapter
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@ -19,6 +21,14 @@ _THINK_OFF_REASON: Final = "Ollama takes think only as on or off, and litellm ma
_DECODED_JSON: Final = TypeAdapter(object)
class _JsonLookup(Protocol):
def get(self, key: str, default: object, /) -> object: ...
class _JsonItems(Protocol):
def items(self) -> Iterable[tuple[str, object]]: ...
class OllamaResponseIdNormalizer:
def __init__(self) -> None:
self.response_id: str | None = None
@ -206,15 +216,15 @@ class OllamaModelInfo(BaseLLMModelInfo):
return any(name.lower() in model_cost_keys for name in potential_model_names)
@staticmethod
def _supports_function_calling(ollama_model_info: dict) -> bool:
_template: Final[str] = str(ollama_model_info.get("template", "") or "")
def _supports_function_calling(ollama_model_info: object) -> bool:
_template: Final[str] = str(require_attributes(ollama_model_info, _JsonLookup, "get").get("template", "") or "")
return "tools" in _template.lower()
@staticmethod
def _get_max_tokens(ollama_model_info: dict) -> int | None:
_model_info: Final[dict] = ollama_model_info.get("model_info", {})
def _get_max_tokens(ollama_model_info: object) -> object:
_model_info: Final = require_attributes(ollama_model_info, _JsonLookup, "get").get("model_info", {})
for key, value in _model_info.items():
for key, value in require_attributes(_model_info, _JsonItems, "items").items():
if "context_length" in key:
return value
return None
@ -253,7 +263,7 @@ class OllamaModelInfo(BaseLLMModelInfo):
"max_output_tokens": None,
}
model_info: Final = response.json()
model_info: Final = _DECODED_JSON.validate_python(response.json())
max_tokens: Final = self._get_max_tokens(model_info)
return {

21
litellm/llms/ollama/completion/handler.py
View file

@ -6,12 +6,13 @@ Ollama /chat/completion calls handled in llm_http_handler.py
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Any, Final, Protocol, TypedDict
from typing import Final, Protocol, TypedDict
import httpx
from typing_extensions import NotRequired, ReadOnly
import litellm
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.types.mcp import has_header
from litellm.types.utils import EmbeddingResponse
@ -23,6 +24,14 @@ class TokenEncoder(Protocol):
def encode(self, text: str, /) -> Sequence[int]: ...
class _DebugLogger(Protocol):
def debug(self, message: str, /) -> object: ...
class _WarningLogger(Protocol):
def warning(self, message: str, /) -> object: ...
class OllamaEmbeddingResponse(TypedDict):
"""Body of an Ollama ``/api/embed`` response."""
@ -51,7 +60,7 @@ def _process_ollama_embedding_response(
prompts: list[str],
model: str,
model_response: EmbeddingResponse,
logging_obj: Any,
logging_obj: object,
encoding: TokenEncoder | None,
) -> EmbeddingResponse:
output_data: Final = []
@ -66,11 +75,15 @@ def _process_ollama_embedding_response(
if encoding is not None:
input_tokens = len(encoding.encode("".join(prompts)))
if logging_obj:
logging_obj.debug("Ollama response missing prompt_eval_count; estimated with encoding.")
require_attributes(logging_obj, _DebugLogger, "debug").debug(
"Ollama response missing prompt_eval_count; estimated with encoding."
)
else:
input_tokens = 0
if logging_obj:
logging_obj.warning("Missing prompt_eval_count and no encoding provided; defaulted to 0.")
require_attributes(logging_obj, _WarningLogger, "warning").warning(
"Missing prompt_eval_count and no encoding provided; defaulted to 0."
)
model_response.object = "list"
model_response.data = output_data

Some files were not shown because too many files have changed in this diff Show more