diff --git a/litellm/_lazy_imports.py b/litellm/_lazy_imports.py index 46126c444a6..1a7dc343745 100644 --- a/litellm/_lazy_imports.py +++ b/litellm/_lazy_imports.py @@ -19,7 +19,7 @@ import importlib import sys from collections.abc import Callable, Mapping from types import ModuleType -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Any, Final from typing_extensions import ReadOnly, TypedDict @@ -456,7 +456,7 @@ def _lazy_import_http_handlers(name: str) -> object: params: Final = {"timeout": async_timeout, "client_alias": "module level aclient"} # Create the client instance - provider_id: Final = cast(Any, "litellm_module_level_client") + provider_id: Final = "litellm_module_level_client" async_client: Final = get_async_httpx_client( llm_provider=provider_id, params=params, diff --git a/litellm/integrations/otel/plumbing/metrics.py b/litellm/integrations/otel/plumbing/metrics.py index 7ec3711ee65..7bee27bc30d 100644 --- a/litellm/integrations/otel/plumbing/metrics.py +++ b/litellm/integrations/otel/plumbing/metrics.py @@ -173,6 +173,7 @@ class _MetricKwargs(TypedDict, total=False): response_cost: ReadOnly[float | None] completion_start_time: ReadOnly[datetime | float | str | None] api_call_start_time: ReadOnly[datetime | float | str | None] + end_time: ReadOnly[datetime | float | str | None] def resolve_error_type(kwargs: Mapping[str, Any]) -> str: @@ -414,13 +415,11 @@ class GenAIMetricRecorder: if generation_time > 0: self._metrics.time_per_output_token.record(generation_time / completion_tokens, attributes=common_attrs) - def _record_response_duration(self, kwargs: Mapping[str, Any], end_time: datetime, common_attrs: dict) -> None: + def _record_response_duration(self, kwargs: _MetricKwargs, end_time: datetime, common_attrs: dict) -> None: api_call_start_time: Final = kwargs.get("api_call_start_time") if api_call_start_time is None: return _end_time = kwargs.get("end_time") or end_time - if _end_time is None: - _end_time = datetime.now() api_call_start: Final = to_seconds(api_call_start_time) end_ts: Final = to_seconds(_end_time) if api_call_start is None or end_ts is None: diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 41f13817d75..714752a2ff4 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3437,7 +3437,7 @@ class Logging(LiteLLMLoggingBaseClass): start_time=start_time, end_time=end_time, print_verbose=print_verbose, - level=LogfireLevel.INFO.value, + level=LogfireLevel.INFO, ) if callback == "lunary" and lunaryLogger is not None: @@ -4394,7 +4394,7 @@ class Logging(LiteLLMLoggingBaseClass): response_obj=result, start_time=start_time, end_time=end_time, - level=LogfireLevel.ERROR.value, + level=LogfireLevel.ERROR, print_verbose=print_verbose, ) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 3d865bd1819..1d75a2c10fe 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -3182,10 +3182,8 @@ def anthropic_messages_pt( def extract_between_tags(tag: str, string: str, strip: bool = False) -> list[str]: - ext_list = re.findall(f"<{tag}>(.+?)", string, re.DOTALL) - if strip: - ext_list = [e.strip() for e in ext_list] - return ext_list + matches: Final = re.finditer(f"<{tag}>(.+?)", string, re.DOTALL) + return [match.group(1).strip() if strip else match.group(1) for match in matches] def contains_tag(tag: str, string: str) -> bool: @@ -6028,7 +6026,7 @@ def _parse_tool_call_arguments(raw: object, tool_name: str | None, context: str) return ({},) normalized_raw: Final = "{}" if raw == REDACTED_BY_LITELLM else raw try: - parsed: Final = parse_tool_call_arguments(normalized_raw, tool_name=tool_name, context=context) + parsed: Final[object] = parse_tool_call_arguments(normalized_raw, tool_name=tool_name, context=context) except ValueError as e: salvaged: Final = salvage_concatenated_tool_arguments(normalized_raw) if salvaged: diff --git a/litellm/litellm_core_utils/safe_json_dumps.py b/litellm/litellm_core_utils/safe_json_dumps.py index 63242a580e7..24c401281ef 100644 --- a/litellm/litellm_core_utils/safe_json_dumps.py +++ b/litellm/litellm_core_utils/safe_json_dumps.py @@ -34,7 +34,7 @@ def safe_json_structure( def _transform(key: str | None, value: str) -> str: return value if value_transform is None else value_transform(key, value) - def _serialize(obj: object, seen: set[int], depth: int, key: str | None = None) -> Any: + def _serialize(obj: object, seen: set[int], depth: int, key: str | None = None) -> object: # Check for maximum depth. if depth > max_depth: return "MaxDepthExceeded" @@ -48,32 +48,32 @@ def safe_json_structure( if id(obj) in seen: return "CircularReference Detected" seen.add(id(obj)) - result: dict | list | tuple | set | str if isinstance(obj, dict): - result = {} - for k, v in obj.items(): - if isinstance(k, (str)): - clean_k = k.replace("\x00", "") if "\x00" in k else k - result[clean_k] = _serialize(v, seen, depth + 1, clean_k) + mapping_result: Final = { + (clean_k := k.replace("\x00", "") if "\x00" in k else k): _serialize(v, seen, depth + 1, clean_k) + for k, v in obj.items() + if isinstance(k, str) + } seen.remove(id(obj)) - return result + return mapping_result elif isinstance(obj, list): - result = [_serialize(item, seen, depth + 1, key) for item in obj] + list_result: Final = [_serialize(item, seen, depth + 1, key) for item in obj] seen.remove(id(obj)) - return result + return list_result elif isinstance(obj, tuple): - result = tuple(_serialize(item, seen, depth + 1, key) for item in obj) + tuple_result: Final = tuple(_serialize(item, seen, depth + 1, key) for item in obj) seen.remove(id(obj)) - return result + return tuple_result elif isinstance(obj, set): - result = sorted([_serialize(item, seen, depth + 1, key) for item in obj]) + sortable_values: Final[list[Any]] = [_serialize(item, seen, depth + 1, key) for item in obj] + set_result: Final = sorted(sortable_values) seen.remove(id(obj)) - return result + return set_result elif isinstance(obj, BaseModel): dumped: Final = obj.model_dump() - result = _serialize(dumped, seen, depth + 1, key) + model_result: Final = _serialize(dumped, seen, depth + 1, key) seen.remove(id(obj)) - return result + return model_result else: # Fall back to string conversion for non-serializable objects. try: diff --git a/litellm/llms/bedrock/chat/agentcore/transformation.py b/litellm/llms/bedrock/chat/agentcore/transformation.py index bb5bb246a93..d8a0bc930e0 100644 --- a/litellm/llms/bedrock/chat/agentcore/transformation.py +++ b/litellm/llms/bedrock/chat/agentcore/transformation.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Final, Optional, Union from urllib.parse import quote import httpx +from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm._uuid import uuid @@ -52,6 +53,9 @@ else: AsyncHTTPHandler = Any +_JSON_VALUE: Final = TypeAdapter(object) + + class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): def __init__(self, **kwargs): BaseConfig.__init__(self, **kwargs) @@ -314,7 +318,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): return None try: - data: Final[object] = json.loads(json_str) + data: Final = _JSON_VALUE.validate_python(json.loads(json_str)) # Skip non-dict data (some lines contain JSON strings) return data if isinstance(data, dict) else None except json.JSONDecodeError: @@ -482,7 +486,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): if "application/json" in content_type: # Direct JSON response verbose_logger.debug("Parsing JSON response") - response_json: Final[object] = raw_response.json() + response_json: Final = _JSON_VALUE.validate_python(raw_response.json()) verbose_logger.debug("Response JSON: %s", response_json) return self._parse_json_response(response_json) else: @@ -566,7 +570,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): continue try: - data_obj = json.loads(json_str) + data_obj = _JSON_VALUE.validate_python(json.loads(json_str)) if not isinstance(data_obj, dict): continue @@ -706,7 +710,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): ) try: body: Final = response.read() - response_json: Final[object] = json.loads(body) + response_json: Final = _JSON_VALUE.validate_python(json.loads(body)) except (json.JSONDecodeError, Exception) as e: raise BedrockError( status_code=response.status_code, @@ -788,7 +792,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): continue try: - data_obj = json.loads(json_str) + data_obj = _JSON_VALUE.validate_python(json.loads(json_str)) if not isinstance(data_obj, dict): continue @@ -928,7 +932,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): ) try: body: Final = await response.aread() - response_json: Final[object] = json.loads(body) + response_json: Final = _JSON_VALUE.validate_python(json.loads(body)) except (json.JSONDecodeError, Exception) as e: raise BedrockError( status_code=response.status_code, diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index cd3727401ef..0d8f11e6627 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -12,7 +12,8 @@ from itertools import chain from typing import TYPE_CHECKING, Final, Literal, cast, overload import httpx -from pydantic import TypeAdapter, ValidationError +from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger @@ -129,6 +130,16 @@ from ..common_utils import ( normalize_bedrock_opus_output_config_effort, ) + +@with_config(ConfigDict(extra="allow", strict=True)) +class _JsonContentToolCall(TypedDict): + type: ReadOnly[NotRequired[object]] + name: ReadOnly[NotRequired[str]] + arguments: ReadOnly[NotRequired[str | dict[str, object] | None]] + + +_JSON_CONTENT_TOOL_CALL: Final = TypeAdapter(_JsonContentToolCall) + if TYPE_CHECKING: from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer @@ -2453,7 +2464,7 @@ class AmazonConverseConfig(BaseConfig): if message.content is not None: try: tool_call_names: Final = self.get_tool_call_names(tools) - json_content: Final = json.loads(message.content) + json_content: Final = _JSON_CONTENT_TOOL_CALL.validate_python(json.loads(message.content)) if json_content.get("type") == "function" and json_content.get("name") in tool_call_names: tool_calls: Final = [ChatCompletionMessageToolCall(function=Function(**json_content))] diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py index f7566ac4923..9cd8212f2c7 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py @@ -10,6 +10,8 @@ import time from typing import TYPE_CHECKING, Any, Final import httpx +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger @@ -34,6 +36,15 @@ else: LiteLLMLoggingObj = Any +@with_config(ConfigDict(extra="allow", strict=True)) +class _PegasusResponse(TypedDict): + message: ReadOnly[NotRequired[str]] + finishReason: ReadOnly[NotRequired[str]] + + +_PEGASUS_RESPONSE: Final = TypeAdapter(_PegasusResponse) + + class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig): """ Handles transforming OpenAI-style requests into Bedrock InvokeModel requests for @@ -207,7 +218,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig): ModelResponse with choices[0].message.content and finish_reason """ try: - completion_response: Final = raw_response.json() + completion_response: Final = _PEGASUS_RESPONSE.validate_python(raw_response.json()) except Exception as e: raise BedrockError( message=f"Error parsing response: {raw_response.text}, error: {e}", diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index 51b5a99272e..b04a18866b3 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -1,11 +1,12 @@ import copy import json import time -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, cast, get_args import httpx -from pydantic import TypeAdapter, ValidationError +from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger @@ -48,6 +49,27 @@ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM _GUARDRAIL_CONFIG_VALIDATOR: Final["TypeAdapter[GuardrailConfigBlock]"] = TypeAdapter(GuardrailConfigBlock) + +@with_config(ConfigDict(extra="allow", strict=True)) +class _CohereGeneration(TypedDict): + text: ReadOnly[str | None] + finish_reason: ReadOnly[str] + + +@with_config(ConfigDict(extra="allow", strict=True)) +class _CohereInvokeResponse(TypedDict): + text: NotRequired[ReadOnly[str | None]] + generations: NotRequired[ReadOnly[Sequence[_CohereGeneration]]] + + +@with_config(ConfigDict(extra="allow", strict=True)) +class _LlamaInvokeResponse(TypedDict): + generation: ReadOnly[str | None] + + +_COHERE_INVOKE_RESPONSE: Final = TypeAdapter(_CohereInvokeResponse) +_LLAMA_INVOKE_RESPONSE: Final = TypeAdapter(_LlamaInvokeResponse) + _GUARDRAIL_CONFIG_EXPECTED_FORMAT: Final = ( "{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}" ) @@ -310,12 +332,13 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): outputText: str | None = None try: if provider == "cohere": - if "text" in completion_response: - outputText = completion_response["text"] - elif "generations" in completion_response: - outputText = completion_response["generations"][0]["text"] + cohere_response: Final = _COHERE_INVOKE_RESPONSE.validate_python(completion_response) + if "text" in cohere_response: + outputText = cohere_response["text"] + elif "generations" in cohere_response: + outputText = cohere_response["generations"][0]["text"] model_response.choices[0].finish_reason = map_finish_reason( - completion_response["generations"][0]["finish_reason"] + cohere_response["generations"][0]["finish_reason"] ) elif provider == "anthropic": return litellm.AmazonAnthropicClaudeConfig().transform_response( @@ -361,7 +384,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): elif provider == "ai21": outputText = completion_response.get("completions")[0].get("data").get("text") elif provider == "meta" or provider == "llama" or provider == "deepseek_r1": - outputText = completion_response["generation"] + outputText = _LLAMA_INVOKE_RESPONSE.validate_python(completion_response)["generation"] elif provider == "mistral": outputText = litellm.AmazonMistralConfig.get_outputText(completion_response, model_response) else: # amazon titan diff --git a/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py b/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py index a90b4829634..b7ede794f36 100644 --- a/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py +++ b/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py @@ -10,7 +10,7 @@ from itertools import accumulate, chain from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Optional, Protocol, TypeAlias, TypeVar -from pydantic import InstanceOf, TypeAdapter, ValidationError +from pydantic import ConfigDict, InstanceOf, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger @@ -768,6 +768,7 @@ class InvokeStreamFrame: _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object]) +_STRICT_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(strict=True)) _Frame: Final = TypeVar("_Frame", ConverseStreamFrame, InvokeStreamFrame) @@ -1217,7 +1218,7 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation): texts: Sequence[tuple[_StreamGroupKey, str]] = () if event_type == "contentBlockDelta": try: - payload_dict: dict[str, object] = json.loads(payload_bytes) + payload_dict = _STRICT_JSON_OBJECT.validate_python(json.loads(payload_bytes)) texts = _delta_frame_texts(payload_dict) except Exception as e: verbose_proxy_logger.debug( @@ -1255,7 +1256,9 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation): headers_bytes = frame_raw[12 : 12 + orig_hdrs_len] try: - payload_dict = json.loads(frame_raw[12 + orig_hdrs_len : orig_total - 4]) + payload_dict = _STRICT_JSON_OBJECT.validate_python( + json.loads(frame_raw[12 + orig_hdrs_len : orig_total - 4]) + ) for local_idx, (_, container, key) in enumerate( _collect_stream_delta_text_holders(payload_dict.get("delta")) ): diff --git a/litellm/llms/bedrock/realtime/transformation.py b/litellm/llms/bedrock/realtime/transformation.py index 9cb38822881..952d8858f78 100644 --- a/litellm/llms/bedrock/realtime/transformation.py +++ b/litellm/llms/bedrock/realtime/transformation.py @@ -53,6 +53,7 @@ from litellm.types.realtime import ( RealtimeResponseTypedDict, ) +_JSON_OBJECT_MESSAGE: Final = TypeAdapter(dict[str, JsonValue], config=ConfigDict(strict=True)) _JSON_VALUE: Final = TypeAdapter(object) _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object]) @@ -856,7 +857,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): List of Bedrock format messages (JSON strings) """ try: - json_message: Final[dict[str, JsonValue]] = json.loads(message) + json_message: Final = _JSON_OBJECT_MESSAGE.validate_python(json.loads(message)) except json.JSONDecodeError: verbose_logger.warning("Invalid JSON message: %s", message[:200]) return [] diff --git a/litellm/llms/black_forest_labs/image_edit/transformation.py b/litellm/llms/black_forest_labs/image_edit/transformation.py index 7cfb0daf334..f38ce9ecd6f 100644 --- a/litellm/llms/black_forest_labs/image_edit/transformation.py +++ b/litellm/llms/black_forest_labs/image_edit/transformation.py @@ -51,6 +51,7 @@ class _BflPolledResult(TypedDict): _BFL_POLLED_RESULT_ADAPTER: Final = TypeAdapter(_BflPolledResult) +_JSON_VALUE: Final = TypeAdapter(object) _BFL_REQUEST_PARAMS: Final = ( @@ -316,7 +317,7 @@ class BlackForestLabsImageEditConfig(BaseImageEditConfig): The response contains: {"status": "Ready", "result": {"sample": "https://..."}} """ try: - raw_response_data: Final[object] = raw_response.json() + raw_response_data: Final = _JSON_VALUE.validate_python(raw_response.json()) except Exception as e: raise BlackForestLabsError( status_code=raw_response.status_code, diff --git a/litellm/llms/claude_code/harness/transformation.py b/litellm/llms/claude_code/harness/transformation.py index f75d86c7e63..763d771d697 100644 --- a/litellm/llms/claude_code/harness/transformation.py +++ b/litellm/llms/claude_code/harness/transformation.py @@ -11,7 +11,7 @@ from __future__ import annotations import itertools import json -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass, field from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final @@ -231,12 +231,16 @@ def _system_events(event: Mapping[str, object], state: ClaudeCodeStreamState) -> return event_list() -def _record_result(event: Mapping[str, Any], state: ClaudeCodeStreamState) -> Sequence[Event]: +def _record_result(event: Mapping[str, object], state: ClaudeCodeStreamState) -> Sequence[Event]: state.result_seen = True state.is_error = bool(event.get("is_error", False)) result = event.get("result") state.result_text = result if isinstance(result, str) else None - state.errors = [str(e) for e in event.get("errors") or ()] # mutable-ok: mirrors the JSON errors array + errors: Final = event.get("errors") or () + if not isinstance(errors, Iterable): + raise TypeError(f"'{type(errors).__name__}' object is not iterable") + error_items: Final[Iterable[object]] = errors + state.errors = [str(e) for e in error_items] # mutable-ok: mirrors the JSON errors array state.structured_output = event.get("structured_output") if event.get("session_id"): state.session_id = str(event["session_id"]) @@ -301,7 +305,7 @@ class ClaudeCodeHarnessConfig(BaseCLIHarnessConfig): options: ClaudeCodeOptions = self.get_options(ctx) model = ctx.model # Background calls (titles, summaries) use the same model group, like OpenCode. - model_env: Final = ( + model_env: Final[Mapping[str, str]] = ( MappingProxyType({"ANTHROPIC_MODEL": model, "ANTHROPIC_SMALL_FAST_MODEL": model}) if model else MappingProxyType({}) diff --git a/litellm/llms/dashscope/embed/transformation.py b/litellm/llms/dashscope/embed/transformation.py index d0c2b4dff0c..35132346261 100644 --- a/litellm/llms/dashscope/embed/transformation.py +++ b/litellm/llms/dashscope/embed/transformation.py @@ -14,6 +14,8 @@ Docs - https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api from typing import Final import httpx +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.url_utils import ensure_api_route @@ -28,6 +30,23 @@ from ..common_utils import DashScopeError DEFAULT_API_BASE: Final = "https://dashscope.aliyuncs.com/compatible-mode/v1" +@with_config(ConfigDict(extra="allow", strict=True)) +class _DashScopeUsage(TypedDict, total=False): + prompt_tokens: NotRequired[ReadOnly[int]] + total_tokens: NotRequired[ReadOnly[int]] + + +@with_config(ConfigDict(extra="allow", strict=True)) +class _DashScopeEmbeddingResponse(TypedDict, total=False): + data: NotRequired[ReadOnly[list[object]]] + model: NotRequired[ReadOnly[str | None]] + usage: NotRequired[ReadOnly[_DashScopeUsage | None]] + id: NotRequired[ReadOnly[object]] + + +_DASHSCOPE_EMBEDDING_RESPONSE: Final = TypeAdapter(_DashScopeEmbeddingResponse) + + class DashScopeEmbeddingConfig(BaseEmbeddingConfig): """ Reference: https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api @@ -152,11 +171,12 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig): message=message, ) + parsed_response: Final = _DASHSCOPE_EMBEDDING_RESPONSE.validate_python(response_json) model_response.object = "list" - model_response.data = response_json.get("data", []) - model_response.model = response_json.get("model", model) + model_response.data = parsed_response.get("data", []) + model_response.model = parsed_response.get("model", model) - usage: Final = response_json.get("usage") or {} + usage: Final = parsed_response.get("usage") or {} prompt_tokens: Final = usage.get("prompt_tokens", 0) total_tokens: Final = usage.get("total_tokens", prompt_tokens) setattr( @@ -169,8 +189,8 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig): ), ) - if "id" in response_json: - setattr(model_response, "id", response_json["id"]) + if "id" in parsed_response: + setattr(model_response, "id", parsed_response["id"]) return model_response diff --git a/litellm/llms/databricks/streaming_utils.py b/litellm/llms/databricks/streaming_utils.py index eca87f9dd5f..cc85012b803 100644 --- a/litellm/llms/databricks/streaming_utils.py +++ b/litellm/llms/databricks/streaming_utils.py @@ -1,6 +1,8 @@ import json from typing import Final +from pydantic import ConfigDict, TypeAdapter + import litellm from litellm import verbose_logger from litellm.types.llms.openai import ( @@ -23,6 +25,9 @@ def _usage_block(usage: Usage) -> ChatCompletionUsageBlock: ) +_JSON_CHUNK: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True)) + + class ModelResponseIterator: def __init__(self, streaming_response, sync_stream: bool): self.streaming_response = streaming_response @@ -99,7 +104,7 @@ class ModelResponseIterator: self.response_iterator = self.streaming_response chunk: Final = self._next_sse_data() try: - json_chunk: Final = json.loads(chunk) + json_chunk: Final = _JSON_CHUNK.validate_python(json.loads(chunk)) return self.chunk_parser(chunk=json_chunk) except ValueError as e: verbose_logger.debug( @@ -126,7 +131,7 @@ class ModelResponseIterator: if chunk == "[DONE]": raise StopAsyncIteration try: - json_chunk: Final = json.loads(chunk) + json_chunk: Final = _JSON_CHUNK.validate_python(json.loads(chunk)) return self.chunk_parser(chunk=json_chunk) except ValueError as e: verbose_logger.debug( diff --git a/litellm/llms/edenai/embedding/transformation.py b/litellm/llms/edenai/embedding/transformation.py index c79a6839434..11e8a42124e 100644 --- a/litellm/llms/edenai/embedding/transformation.py +++ b/litellm/llms/edenai/embedding/transformation.py @@ -8,6 +8,7 @@ Docs: https://www.edenai.co/docs/v3/llms/embeddings from typing import TYPE_CHECKING, Final import httpx +from pydantic import ConfigDict, TypeAdapter from litellm.litellm_core_utils.core_helpers import set_response_cost_in_hidden_params from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -23,6 +24,8 @@ if TYPE_CHECKING: _SUPPORTED_PARAMS: Final = ("dimensions", "encoding_format", "user") +_EMBEDDING_BODY: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True)) + class EdenAIEmbeddingConfig(BaseEmbeddingConfig): def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: inherited contract @@ -80,7 +83,7 @@ class EdenAIEmbeddingConfig(BaseEmbeddingConfig): optional_params: dict[str, object], # mutable-ok: inherited contract litellm_params: dict[str, object], # mutable-ok: inherited contract ) -> EmbeddingResponse: - body: Final = raw_response.json() + body: Final = _EMBEDDING_BODY.validate_python(raw_response.json()) logging_obj.post_call(original_response=body) response: Final[EmbeddingResponse] = convert_to_model_response_object( response_object=body, model_response_object=model_response, response_type="embedding" diff --git a/litellm/llms/gigachat/authenticator.py b/litellm/llms/gigachat/authenticator.py index d43e203d7a1..f37c6e88896 100644 --- a/litellm/llms/gigachat/authenticator.py +++ b/litellm/llms/gigachat/authenticator.py @@ -12,6 +12,8 @@ from types import MappingProxyType from typing import Final import httpx +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm._logging import verbose_logger from litellm.caching.caching import InMemoryCache @@ -37,6 +39,17 @@ _token_cache: Final = InMemoryCache() _NO_LITELLM_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) +@with_config(ConfigDict(extra="allow", strict=True)) +class _GigaChatTokenResponse(TypedDict): + tok: NotRequired[ReadOnly[str | None]] + access_token: NotRequired[ReadOnly[str | None]] + exp: NotRequired[ReadOnly[int | float | str | None]] + expires_at: NotRequired[ReadOnly[int | float | str | None]] + + +_TOKEN_RESPONSE: Final = TypeAdapter(_GigaChatTokenResponse) + + class GigaChatAuthError(BaseLLMException): """GigaChat authentication error.""" @@ -230,7 +243,7 @@ async def _request_token_async( def _parse_token_response(response: httpx.Response) -> tuple[str, int]: """Parse OAuth token response.""" - data: Final = response.json() + data: Final = _TOKEN_RESPONSE.validate_python(response.json()) # GigaChat returns either 'tok'/'exp' or 'access_token'/'expires_at' access_token: Final = data.get("tok") or data.get("access_token") diff --git a/litellm/llms/gigachat/passthrough/transformation.py b/litellm/llms/gigachat/passthrough/transformation.py index 872eee6dbf9..6f8cb7753aa 100644 --- a/litellm/llms/gigachat/passthrough/transformation.py +++ b/litellm/llms/gigachat/passthrough/transformation.py @@ -5,6 +5,7 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Final import httpx +from pydantic import ConfigDict, TypeAdapter from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig from litellm.llms.gigachat.authenticator import get_access_token @@ -20,6 +21,8 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.types.utils import CostResponseTypes +_STREAM_CHUNK: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True)) + class GigaChatPassthroughConfig(BasePassthroughConfig): def is_streaming_request(self, endpoint: str, request_data: Mapping[str, object]) -> bool: @@ -161,7 +164,7 @@ class GigaChatPassthroughConfig(BasePassthroughConfig): continue chunk = chunk.removeprefix("data: ") try: - message = json.loads(chunk) + message = _STREAM_CHUNK.validate_python(json.loads(chunk)) except json.JSONDecodeError: continue diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index 080fc3d24c9..b7e4cc6868f 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -5,7 +5,7 @@ This requires websockets, and is currently only supported on LiteLLM Proxy. """ import ssl -from typing import Any, Final, cast +from typing import TYPE_CHECKING, Final, cast from litellm._logging import redact_string, verbose_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES @@ -23,6 +23,9 @@ from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context from ..common_utils import is_openai_backed_api_base from ..openai import OpenAIChatCompletion +if TYPE_CHECKING: + from fastapi import WebSocket + class OpenAIRealtime(OpenAIChatCompletion): """ @@ -131,7 +134,7 @@ class OpenAIRealtime(OpenAIChatCompletion): async def async_realtime( self, model: str, - websocket: Any, + websocket: "WebSocket", logging_obj: LiteLLMLogging, api_base: str | None = None, api_key: str | None = None, diff --git a/litellm/llms/perplexity/embedding/transformation.py b/litellm/llms/perplexity/embedding/transformation.py index 7b4da933976..147d344795d 100644 --- a/litellm/llms/perplexity/embedding/transformation.py +++ b/litellm/llms/perplexity/embedding/transformation.py @@ -12,12 +12,14 @@ This module decodes them into float arrays for OpenAI-compatible responses. """ import base64 +import builtins import struct -from collections.abc import Mapping -from typing import Final +from collections.abc import Mapping, Sequence +from typing import Final, Literal import httpx -from pydantic import ConfigDict, TypeAdapter +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import ReadOnly, TypedDict from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.url_utils import join_api_base @@ -31,6 +33,17 @@ _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_i _INT: Final = TypeAdapter(int) +@with_config(ConfigDict(extra="allow", strict=True)) +class _PerplexityEmbeddingResponse(TypedDict, total=False): + model: ReadOnly[str | None] + object: ReadOnly[Literal["list"]] + data: ReadOnly[Sequence[builtins.object] | Mapping[str, builtins.object] | Literal[""]] + usage: ReadOnly[builtins.object] + + +_EMBEDDING_RESPONSE_ADAPTER: Final = TypeAdapter(_PerplexityEmbeddingResponse) + + class PerplexityEmbeddingError(BaseLLMException): def __init__( self, @@ -153,14 +166,18 @@ class PerplexityEmbeddingConfig(BaseEmbeddingConfig): litellm_params: dict = {}, ) -> EmbeddingResponse: try: - raw_response_json: Final = raw_response.json() + raw_response_json: Final[object] = raw_response.json() except Exception: raise PerplexityEmbeddingError(message=raw_response.text, status_code=raw_response.status_code) - model_response.model = raw_response_json.get("model", model) - model_response.object = raw_response_json.get("object", "list") + response_data: Final[_PerplexityEmbeddingResponse] = _EMBEDDING_RESPONSE_ADAPTER.validate_python( + raw_response_json + ) - raw_data: Final = raw_response_json.get("data", []) + model_response.model = response_data.get("model", model) + model_response.object = response_data.get("object", "list") + + raw_data: Final = response_data.get("data", []) decoded_data: Final[list[dict[str, object]]] = [] for item in raw_data: decoded_item = dict(_JSON_OBJECT.validate_python(item)) @@ -168,7 +185,7 @@ class PerplexityEmbeddingConfig(BaseEmbeddingConfig): decoded_data.append(decoded_item) model_response.data = decoded_data - usage_data: Final = _JSON_OBJECT.validate_python(raw_response_json.get("usage", {})) + usage_data: Final = _JSON_OBJECT.validate_python(response_data.get("usage", {})) usage: Final = Usage( prompt_tokens=_INT.validate_python( usage_data.get("prompt_tokens", 0) or usage_data.get("total_tokens", 0) or 0 diff --git a/litellm/llms/runwayml/text_to_speech/transformation.py b/litellm/llms/runwayml/text_to_speech/transformation.py index 1f052648f13..f46e587a00c 100644 --- a/litellm/llms/runwayml/text_to_speech/transformation.py +++ b/litellm/llms/runwayml/text_to_speech/transformation.py @@ -6,11 +6,12 @@ Maps OpenAI TTS spec to RunwayML Text-to-Speech API import asyncio import time -from collections.abc import Coroutine, Sequence -from typing import TYPE_CHECKING, Any, Final, TypedDict, Union +from collections.abc import Coroutine +from typing import TYPE_CHECKING, Any, Final, Union import httpx -from typing_extensions import ReadOnly +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger @@ -33,12 +34,17 @@ else: HttpxBinaryResponseContent = Any +@with_config(ConfigDict(extra="allow", strict=True)) class _RunwayTtsTaskResponse(TypedDict, total=False): id: ReadOnly[str] status: ReadOnly[str] - output: ReadOnly[Sequence[object]] - failure: ReadOnly[str] - failureCode: ReadOnly[str] + output: ReadOnly[object] + failure: ReadOnly[object] + failureCode: ReadOnly[object] + + +_RUNWAY_TTS_TASK_RESPONSE_ADAPTER: Final = TypeAdapter(_RunwayTtsTaskResponse) +_RUNWAY_TTS_OUTPUT_ADAPTER: Final = TypeAdapter(list[object], config=ConfigDict(strict=True)) class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): @@ -321,7 +327,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): response = client.get(url=task_url, headers=headers) response.raise_for_status() - response_data: _RunwayTtsTaskResponse = response.json() + response_data = _RUNWAY_TTS_TASK_RESPONSE_ADAPTER.validate_python(response.json()) # Check task status status = self._check_task_status(response_data=response_data) @@ -368,7 +374,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): response = await client.get(url=task_url, headers=headers) response.raise_for_status() - response_data: _RunwayTtsTaskResponse = response.json() + response_data = _RUNWAY_TTS_TASK_RESPONSE_ADAPTER.validate_python(response.json()) # Check task status status = self._check_task_status(response_data=response_data) @@ -459,7 +465,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): from litellm.types.llms.openai import HttpxBinaryResponseContent try: - response_data: Final[_RunwayTtsTaskResponse] = raw_response.json() + response_data: Final = _RUNWAY_TTS_TASK_RESPONSE_ADAPTER.validate_python(raw_response.json()) except Exception as e: raise self.get_error_class( error_message=f"Error parsing RunwayML TTS response: {e}", @@ -489,16 +495,20 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): ) # Get the completed task data - task_data: Final[_RunwayTtsTaskResponse] = polled_response.json() + task_data: Final = _RUNWAY_TTS_TASK_RESPONSE_ADAPTER.validate_python(polled_response.json()) verbose_logger.debug("RunwayML TTS polling complete, downloading audio") # Get audio URL from output output: Final = task_data.get("output", []) - if not output or not isinstance(output, list) or len(output) == 0: + if not output or not isinstance(output, list): raise ValueError("RunwayML TTS response missing audio URL in output") - audio_url: Final = output[0] + output_items: Final = _RUNWAY_TTS_OUTPUT_ADAPTER.validate_python(output) + if len(output_items) == 0: + raise ValueError("RunwayML TTS response missing audio URL in output") + + audio_url: Final = output_items[0] if not isinstance(audio_url, str): raise ValueError(f"RunwayML TTS audio URL is not a string: {audio_url}") @@ -528,7 +538,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): from litellm.types.llms.openai import HttpxBinaryResponseContent try: - response_data: Final[_RunwayTtsTaskResponse] = raw_response.json() + response_data: Final = _RUNWAY_TTS_TASK_RESPONSE_ADAPTER.validate_python(raw_response.json()) except Exception as e: raise self.get_error_class( error_message=f"Error parsing RunwayML TTS response: {e}", @@ -558,16 +568,20 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): ) # Get the completed task data - task_data: Final[_RunwayTtsTaskResponse] = polled_response.json() + task_data: Final = _RUNWAY_TTS_TASK_RESPONSE_ADAPTER.validate_python(polled_response.json()) verbose_logger.debug("RunwayML TTS polling complete (async), downloading audio") # Get audio URL from output output: Final = task_data.get("output", []) - if not output or not isinstance(output, list) or len(output) == 0: + if not output or not isinstance(output, list): raise ValueError("RunwayML TTS response missing audio URL in output") - audio_url: Final = output[0] + output_items: Final = _RUNWAY_TTS_OUTPUT_ADAPTER.validate_python(output) + if len(output_items) == 0: + raise ValueError("RunwayML TTS response missing audio URL in output") + + audio_url: Final = output_items[0] if not isinstance(audio_url, str): raise ValueError(f"RunwayML TTS audio URL is not a string: {audio_url}") diff --git a/litellm/llms/triton/embedding/transformation.py b/litellm/llms/triton/embedding/transformation.py index 8c6e0b832fc..619a110c1ea 100644 --- a/litellm/llms/triton/embedding/transformation.py +++ b/litellm/llms/triton/embedding/transformation.py @@ -1,6 +1,9 @@ +from collections.abc import Sequence from typing import Final import httpx +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.llms.base_llm.chat.transformation import AllMessageValues, BaseLLMException from litellm.llms.base_llm.embedding.transformation import ( @@ -14,6 +17,21 @@ from litellm.utils import token_counter from ..common_utils import TritonError +@with_config(ConfigDict(extra="allow", strict=True)) +class _TritonEmbeddingOutput(TypedDict): + shape: ReadOnly[Sequence[int]] + data: ReadOnly[Sequence[int | float]] + + +@with_config(ConfigDict(extra="allow", strict=True)) +class _TritonEmbeddingResponse(TypedDict): + outputs: ReadOnly[Sequence[_TritonEmbeddingOutput]] + model_name: ReadOnly[NotRequired[str | None]] + + +_TRITON_EMBEDDING_RESPONSE: Final = TypeAdapter(_TritonEmbeddingResponse) + + class TritonEmbeddingConfig(BaseEmbeddingConfig): """ Transformations for triton /embeddings endpoint (This is a trtllm model) @@ -83,9 +101,10 @@ class TritonEmbeddingConfig(BaseEmbeddingConfig): except Exception: raise TritonError(message=raw_response.text, status_code=raw_response.status_code) + parsed_response: Final = _TRITON_EMBEDDING_RESPONSE.validate_python(raw_response_json) _embedding_output: Final = [] - _outputs: Final = raw_response_json["outputs"] + _outputs: Final = parsed_response["outputs"] for output in _outputs: _shape = output["shape"] _data = output["data"] @@ -100,7 +119,7 @@ class TritonEmbeddingConfig(BaseEmbeddingConfig): } ) - model_response.model = raw_response_json.get("model_name", "None") + model_response.model = parsed_response.get("model_name", "None") model_response.data = _embedding_output model_response.usage = self._build_embedding_usage(model=model, request_data=request_data) return model_response @@ -134,7 +153,7 @@ class TritonEmbeddingConfig(BaseEmbeddingConfig): return TritonError(message=error_message, status_code=status_code, headers=headers) @staticmethod - def split_embedding_by_shape(data: list[float], shape: list[int]) -> list[list[float]]: + def split_embedding_by_shape(data: Sequence[int | float], shape: Sequence[int]) -> Sequence[Sequence[int | float]]: if len(shape) != 2: raise ValueError("Shape must be of length 2.") embedding_size: Final = shape[1] diff --git a/litellm/llms/vertex_ai/image_generation/image_generation_handler.py b/litellm/llms/vertex_ai/image_generation/image_generation_handler.py index 7ba7b060991..8cbbe87a058 100644 --- a/litellm/llms/vertex_ai/image_generation/image_generation_handler.py +++ b/litellm/llms/vertex_ai/image_generation/image_generation_handler.py @@ -20,6 +20,7 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj _PREDICTIONS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) +_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) class VertexImageGeneration(VertexLLM): @@ -168,7 +169,7 @@ class VertexImageGeneration(VertexLLM): if response.status_code != 200: raise Exception(f"Error: {response.status_code} {response.text}") - json_response: Final = response.json() + json_response: Final = _JSON_OBJECT.validate_python(response.json()) return self.process_image_generation_response(json_response, model_response, model) async def aimage_generation( @@ -272,7 +273,7 @@ class VertexImageGeneration(VertexLLM): if response.status_code != 200: raise Exception(f"Error: {response.status_code} {response.text}") - json_response: Final = response.json() + json_response: Final = _JSON_OBJECT.validate_python(response.json()) return self.process_image_generation_response(json_response, model_response, model) def is_image_generation_response(self, json_response: dict[str, Any]) -> bool: diff --git a/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py b/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py index 9fa86ec30f5..ec2fd16f93e 100644 --- a/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py +++ b/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py @@ -9,10 +9,11 @@ The actual message transformation reuses OpenAIGPTConfig since Gemma uses OpenAI """ from collections.abc import Callable -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Final, cast import httpx -from pydantic import ValidationError +from pydantic import ConfigDict, TypeAdapter, ValidationError +from typing_extensions import NotRequired, ReadOnly, TypeAliasType, TypedDict from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.custom_httpx.http_handler import ( @@ -32,6 +33,17 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer +class _PredictionsEnvelope(TypedDict): + predictions: NotRequired[ReadOnly[dict[str, object]]] + + +_PredictionsEnvelopeType = TypeAliasType("_PredictionsEnvelopeType", _PredictionsEnvelope) +_PREDICTIONS_ENVELOPE: Final[TypeAdapter[_PredictionsEnvelope]] = TypeAdapter( + _PredictionsEnvelopeType, + config=ConfigDict(strict=True, extra="allow"), +) + + def parse_vertex_gemma_container_error(predictions: object) -> VertexGemmaContainerError | None: try: return VertexGemmaContainerError.model_validate(predictions) @@ -142,7 +154,7 @@ class VertexGemmaConfig(OpenAIGPTConfig): def _unwrap_predictions_response( self, - response_json: dict[str, Any], + response_json: _PredictionsEnvelope, ) -> dict[str, object]: """ Unwrap the Vertex Gemma predictions format to OpenAI format. @@ -354,7 +366,7 @@ class VertexGemmaConfig(OpenAIGPTConfig): message=f"Request failed: {response.text}", ) - response_json: Final = response.json() + response_json: Final = _PREDICTIONS_ENVELOPE.validate_python(response.json()) # Unwrap predictions to get OpenAI-compatible response openai_response: Final = self._unwrap_predictions_response(response_json) @@ -449,7 +461,7 @@ class VertexGemmaConfig(OpenAIGPTConfig): message=f"Request failed: {response.text}", ) - response_json: Final = response.json() + response_json: Final = _PREDICTIONS_ENVELOPE.validate_python(response.json()) # Unwrap predictions to get OpenAI-compatible response openai_response: Final = self._unwrap_predictions_response(response_json) diff --git a/litellm/llms/volcengine/responses/transformation.py b/litellm/llms/volcengine/responses/transformation.py index 74f6182bd89..e5a1b32a831 100644 --- a/litellm/llms/volcengine/responses/transformation.py +++ b/litellm/llms/volcengine/responses/transformation.py @@ -3,6 +3,7 @@ from types import UnionType from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, Union, get_args, get_origin import httpx +from pydantic import ConfigDict, TypeAdapter from pydantic import fields as pyd_fields import litellm @@ -37,6 +38,8 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any +_RESPONSE_BODY: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True)) + class _EventModelClass(Protocol): @property @@ -415,7 +418,7 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig): @staticmethod def _parsed_response_body(raw_response: httpx.Response) -> dict[str, object]: - return raw_response.json() + return _RESPONSE_BODY.validate_python(raw_response.json()) @staticmethod def _annotation_origin(annotation: object) -> object: diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index 3b2c2e09df5..f4e4d93b1e1 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -300,12 +300,13 @@ def decode_state_hash(encrypted_state: str) -> dict: if decrypted_json is None: raise ValueError("Failed to decrypt state parameter") - state_data: Final = json.loads(decrypted_json) + state_data: Final = _DECODED_STATE.validate_python(json.loads(decrypted_json)) return state_data _MANAGED_AUTH_CODE_PREFIX: Final = "llm_scope_" _DECODED_JSON: Final = TypeAdapter(object) +_DECODED_STATE: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True)) class _ManagedAuthorizationCode(LiteLLMBaseModel): diff --git a/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py b/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py index 6d4fefcdb6c..665d734e9ab 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py +++ b/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py @@ -11,6 +11,7 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Final import httpx +from pydantic import TypeAdapter from litellm._internal_context import with_service_target from litellm._logging import verbose_logger @@ -42,6 +43,9 @@ if TYPE_CHECKING: from litellm.types.mcp_server.mcp_server_manager import MCPServer +_JSON_VALUE: Final = TypeAdapter(object) + + class MCPOAuth2TokenCache(InMemoryCache): """ In-memory cache for OAuth2 client_credentials tokens, keyed by the identity of the token @@ -160,7 +164,7 @@ class MCPOAuth2TokenCache(InMemoryCache): f"failed with status {exc.response.status_code}" ) from exc - body: Final = response.json() + body: Final = _JSON_VALUE.validate_python(response.json()) if not isinstance(body, dict): raise ValueError( diff --git a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py index c31560a9f63..0263aaf46a8 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py +++ b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py @@ -18,7 +18,8 @@ from typing import Final, Literal, Protocol, TypeAlias import jwt from fastapi import HTTPException from jwt.types import Options -from typing_extensions import assert_never +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import ReadOnly, TypedDict, assert_never from litellm._logging import verbose_logger from litellm.caching.in_memory_cache import InMemoryCache @@ -38,6 +39,20 @@ _ALLOWED_ID_TOKEN_ALGORITHMS: Final = ( "PS512", ) _JWKS_CACHE_TTL_SECONDS: Final = 3600 + + +@with_config(ConfigDict(extra="allow", strict=True)) +class _JwksDocument(TypedDict, total=False): + keys: ReadOnly[Sequence[Mapping[str, object]]] + + +@with_config(ConfigDict(extra="allow", strict=True)) +class _OidcDiscoveryDocument(TypedDict, total=False): + jwks_uri: ReadOnly[object] + + +_JWKS_DOCUMENT: Final = TypeAdapter(_JwksDocument) +_OIDC_DISCOVERY_DOCUMENT: Final = TypeAdapter(_OidcDiscoveryDocument) _jwks_cache: Final = InMemoryCache(default_ttl=_JWKS_CACHE_TTL_SECONDS) JwksFetcher: TypeAlias = Callable[ @@ -103,8 +118,8 @@ async def _fetch_issuer_jwks(binding: MCPOAuthIdentityBinding) -> Sequence[Mappi client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.Oauth2Check) response: Final = await client.get(jwks_url) response.raise_for_status() - document: Final = response.json() - keys: Final = document.get("keys") if isinstance(document, dict) else None + document: Final = _JWKS_DOCUMENT.validate_python(response.json()) + keys: Final = document.get("keys") if not isinstance(keys, list): raise TypeError(f"JWKS document at {jwks_url} has no 'keys' array") await _jwks_cache.async_set_cache(jwks_url, keys, ttl=_JWKS_CACHE_TTL_SECONDS) @@ -116,8 +131,8 @@ async def _discover_jwks_url(issuer: str) -> str: client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.Oauth2Check) response: Final = await client.get(discovery_url) response.raise_for_status() - metadata: Final = response.json() - jwks_uri: Final = metadata.get("jwks_uri") if isinstance(metadata, dict) else None + metadata: Final = _OIDC_DISCOVERY_DOCUMENT.validate_python(response.json()) + jwks_uri: Final = metadata.get("jwks_uri") if not isinstance(jwks_uri, str) or not jwks_uri: raise ValueError(f"OIDC discovery at {discovery_url} returned no jwks_uri") return jwks_uri diff --git a/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py b/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py index 2f88d8c361f..dd2081b769e 100644 --- a/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py +++ b/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py @@ -6,7 +6,9 @@ Filters MCP tools semantically for /chat/completions and /responses endpoints. import asyncio from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final + +from typing_extensions import TypeVar from litellm._logging import verbose_logger from litellm.exceptions import ContextWindowExceededError @@ -21,6 +23,9 @@ if TYPE_CHECKING: from litellm.router import Router +_ToolT: Final = TypeVar("_ToolT", default=object) + + class SemanticToolFilterContextWindowError(Exception): """Raised when the embedding model exceeds its context window, so semantic filtering cannot run.""" @@ -265,9 +270,9 @@ class SemanticMCPToolFilter: async def filter_tools( self, query: str, - available_tools: list[Any], + available_tools: list[_ToolT], top_k: int | None = None, - ) -> list[Any]: + ) -> list[_ToolT]: """ Filter tools semantically based on query. @@ -390,7 +395,7 @@ class SemanticMCPToolFilter: separator: Final = client_name[-len(canonical) - 1] return separator in ("_", "-") - def _get_tools_by_names(self, tool_names: Sequence[str], available_tools: Sequence[object]) -> list[object]: + def _get_tools_by_names(self, tool_names: Sequence[str], available_tools: Sequence[_ToolT]) -> list[_ToolT]: """ Get tools from available_tools by their names, preserving the semantic router's ordering. @@ -406,13 +411,13 @@ class SemanticMCPToolFilter: # Exact matches win over suffix matches when both are present, and # each incoming tool is returned at most once even if two canonical # names happen to be tail-compatible with the same incoming name. - available_by_name: Final[dict[str, object]] = {} + available_by_name: Final[dict[str, _ToolT]] = {} for tool in available_tools: client_name, _ = self.extract_tool_info(tool) if client_name and client_name not in available_by_name: available_by_name[client_name] = tool - matched: Final[list[object]] = [] + matched: Final[list[_ToolT]] = [] used_ids: Final[set[int]] = set() for canonical in tool_names: tool = available_by_name.get(canonical) diff --git a/litellm/proxy/client/models.py b/litellm/proxy/client/models.py index 1b50f8c24f3..a233a3f5ae3 100644 --- a/litellm/proxy/client/models.py +++ b/litellm/proxy/client/models.py @@ -3,9 +3,12 @@ from collections.abc import Mapping from typing import Any, Final import requests +from pydantic import ConfigDict, TypeAdapter from .exceptions import NotFoundError, UnauthorizedError +_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True)) + class ModelsManagementClient: def __init__(self, base_url: str, api_key: str | None = None, timeout: int = 30): @@ -109,7 +112,7 @@ class ModelsManagementClient: try: response: Final = session.send(request.prepare(), timeout=self._timeout) response.raise_for_status() - return response.json() + return _JSON_OBJECT.validate_python(response.json()) except requests.exceptions.HTTPError as e: if e.response.status_code == 401: raise UnauthorizedError(e) @@ -145,7 +148,7 @@ class ModelsManagementClient: try: response: Final = session.send(request.prepare(), timeout=self._timeout) response.raise_for_status() - return response.json() + return _JSON_OBJECT.validate_python(response.json()) except requests.exceptions.HTTPError as e: if e.response.status_code == 401: raise UnauthorizedError(e) @@ -287,7 +290,7 @@ class ModelsManagementClient: try: response: Final = session.send(request.prepare(), timeout=self._timeout) response.raise_for_status() - return response.json() + return _JSON_OBJECT.validate_python(response.json()) except requests.exceptions.HTTPError as e: if e.response.status_code == 401: raise UnauthorizedError(e) diff --git a/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py b/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py index e05db5003fa..29d145691b3 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py +++ b/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py @@ -9,7 +9,7 @@ import json import os import uuid from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict +from typing import TYPE_CHECKING, Any, Final, Literal try: import ulid @@ -28,6 +28,8 @@ except ImportError: HTTPX_AVAILABLE = False from fastapi import HTTPException +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm import DualCache @@ -51,13 +53,17 @@ from litellm.proxy.guardrails._content_utils import ( from litellm.types.guardrails import GuardrailEventHooks -class LassoResponse(TypedDict): +@with_config(ConfigDict(extra="allow", strict=True)) +class LassoResponse(TypedDict, total=False): """Type definition for Lasso API response.""" - violations_detected: bool - deputies: dict[str, bool] - findings: dict[str, list[dict[str, object]]] - messages: list[dict[str, str]] | None + violations_detected: NotRequired[ReadOnly[object]] + deputies: NotRequired[ReadOnly[dict[str, object]]] + findings: NotRequired[ReadOnly[dict[str, object]]] + messages: NotRequired[ReadOnly[list[dict[str, object]] | None]] + + +_LASSO_RESPONSE: Final = TypeAdapter(LassoResponse) if TYPE_CHECKING: @@ -467,7 +473,9 @@ class LassoGuardrail(CustomGuardrail): # Also update data["input"] for Responses-API payloads so the # unredacted text doesn't leak through that field. if isinstance(data.get("input"), str): - text_parts = [msg["content"] for msg in masked_for_input if isinstance(msg.get("content"), str)] + text_parts: Final = [ + content for msg in masked_for_input if isinstance(content := msg.get("content"), str) + ] if text_parts: data["input"] = "\n".join(text_parts) self._log_masking_applied(message_type, dict(response)) @@ -820,7 +828,7 @@ class LassoGuardrail(CustomGuardrail): timeout=self.timeout if self.timeout is not None else 10.0, ) response.raise_for_status() - return response.json() + return _LASSO_RESPONSE.validate_python(response.json()) def _process_lasso_response(self, response: LassoResponse) -> None: """ diff --git a/litellm/proxy/guardrails/guardrail_hooks/noma/noma_v2.py b/litellm/proxy/guardrails/guardrail_hooks/noma/noma_v2.py index 718401b6e68..0202a3dd7e9 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/noma/noma_v2.py +++ b/litellm/proxy/guardrails/guardrail_hooks/noma/noma_v2.py @@ -13,6 +13,8 @@ from datetime import datetime from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, TypeAlias, cast from urllib.parse import urlparse +from pydantic import ConfigDict, TypeAdapter + from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( CustomGuardrail, @@ -37,6 +39,7 @@ if TYPE_CHECKING: _DEFAULT_API_BASE: Final = "https://api.noma.security/" _AIDR_SCAN_ENDPOINT: Final = "/litellm/guardrail" +_AIDR_RESPONSE: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True)) _INTERVENED_INPUT_FIELDS: Final = ("texts", "images", "tools", "tool_calls") _DEFAULT_API_BASE_HOSTNAME: Final = urlparse(_DEFAULT_API_BASE).hostname @@ -257,7 +260,7 @@ class NomaV2Guardrail(CustomGuardrail): response.text, ) response.raise_for_status() - response_json: Final[dict[str, object]] = response.json() + response_json: Final = _AIDR_RESPONSE.validate_python(response.json()) verbose_proxy_logger.debug( "Noma v2 AIDR response parsed: %s", json.dumps(response_json, default=str), diff --git a/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py b/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py index 7af5eaf1da9..2bbe41ac9b6 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py +++ b/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py @@ -164,7 +164,7 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): from litellm.types.llms.openai import OpenAIModerationResponse - return OpenAIModerationResponse(**response.json()) + return OpenAIModerationResponse.model_validate(response.json()) def _check_moderation_result(self, moderation_response: "OpenAIModerationResponse") -> None: """ diff --git a/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py b/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py index a92e67046bd..8faac11ea32 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py +++ b/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py @@ -10,6 +10,8 @@ import os from typing import Any, Final, Literal from fastapi import HTTPException +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( @@ -31,6 +33,26 @@ GUARDRAIL_NAME: Final = "qualifire" DEFAULT_QUALIFIRE_API_BASE: Final = "https://proxy.qualifire.ai" +@with_config(ConfigDict(extra="allow", strict=True)) +class _QualifireFlaggedResult(TypedDict, total=False): + flagged: NotRequired[ReadOnly[object]] + + +@with_config(ConfigDict(extra="allow", strict=True)) +class _QualifireEvaluationResult(TypedDict, total=False): + results: NotRequired[ReadOnly[list[_QualifireFlaggedResult] | None]] + + +@with_config(ConfigDict(extra="allow", strict=True)) +class _QualifireResponse(TypedDict, total=False): + score: NotRequired[ReadOnly[object]] + status: NotRequired[ReadOnly[object]] + evaluationResults: NotRequired[ReadOnly[list[_QualifireEvaluationResult] | None]] + + +_QUALIFIRE_RESPONSE: Final = TypeAdapter(_QualifireResponse) + + class QualifireGuardrail(CustomGuardrail): @classmethod def get_supported_event_hooks(cls) -> list[GuardrailEventHooks]: @@ -244,7 +266,7 @@ class QualifireGuardrail(CustomGuardrail): return api_tools if api_tools else None - def _check_if_flagged(self, result: dict[str, Any]) -> bool: + def _check_if_flagged(self, result: _QualifireResponse) -> bool: """ Check if the Qualifire evaluation result indicates flagged content. @@ -381,7 +403,7 @@ class QualifireGuardrail(CustomGuardrail): timeout=self.timeout, ) response.raise_for_status() - result: Final = response.json() + result: Final = _QUALIFIRE_RESPONSE.validate_python(response.json()) # Extract response info for logging qualifire_response: Final[dict[str, object]] = { diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index a2bb691b401..567b8452ae2 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -101,7 +101,7 @@ if TYPE_CHECKING: from litellm.types.agents import AgentResponse from litellm.types.caching import RedisPipelineIncrementOperation - Span = _Span | Any + Span = _Span InternalUsageCache = _InternalUsageCache else: Span = Any diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 8dc7501bc3b..cbb0dd25722 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -649,7 +649,7 @@ def _metadata_string_tags(metadata_objects: Iterable[Mapping[str, object]]) -> t def _strip_untrusted_request_header_controls( - headers: Any, + headers: object, *, allow_client_message_redaction_opt_out: bool = False, ) -> None: @@ -1579,7 +1579,7 @@ class LiteLLMProxyRequestSetup: @staticmethod def add_internal_user_from_user_mapping( - general_settings: dict | None, + general_settings: Mapping[str, object] | None, user_api_key_dict: UserAPIKeyAuth, headers: dict, ) -> UserAPIKeyAuth: @@ -1598,7 +1598,7 @@ class LiteLLMProxyRequestSetup: return user_api_key_dict @staticmethod - def get_user_from_headers(headers: dict, general_settings: dict | None = None) -> str | None: + def get_user_from_headers(headers: dict, general_settings: Mapping[str, object] | None = None) -> str | None: """ Get the user from the specified header if `general_settings.user_header_name` is set. """ @@ -1619,7 +1619,9 @@ class LiteLLMProxyRequestSetup: return user @staticmethod - def get_openai_org_id_from_headers(headers: dict, general_settings: dict | None = None) -> str | None: + def get_openai_org_id_from_headers( + headers: dict, general_settings: Mapping[str, object] | None = None + ) -> str | None: """ Get the OpenAI Org ID from the headers. """ @@ -1710,7 +1712,7 @@ class LiteLLMProxyRequestSetup: headers: dict, request_data: Mapping[str, object], user_api_key_dict: UserAPIKeyAuth, - general_settings: dict[str, object] | None = None, + general_settings: Mapping[str, object] | None = None, ) -> LitellmDataForBackendLLMCall: """ - Adds user from headers diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py index 5f31aa788e2..e0d997ad635 100644 --- a/litellm/proxy/management_endpoints/access_group_endpoints.py +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -647,8 +647,22 @@ async def update_access_group( _require_proxy_admin(user_api_key_dict) prisma_client: Final = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) - update_fields: Final = data.model_dump(exclude_unset=True) - update_data: Final[dict] = {"updated_by": user_api_key_dict.user_id} + update_fields: Final = { + field: value + for field, value in { + "access_group_name": data.access_group_name, + "description": data.description, + "access_model_names": data.access_model_names, + "access_mcp_server_ids": data.access_mcp_server_ids, + "mcp_tool_permissions": data.mcp_tool_permissions, + "access_agent_ids": data.access_agent_ids, + "access_skill_names": data.access_skill_names, + "assigned_team_ids": data.assigned_team_ids, + "assigned_key_ids": data.assigned_key_ids, + }.items() + if field in data.model_fields_set + } + update_data: Final[dict[str, object]] = {"updated_by": user_api_key_dict.user_id} for field, value in update_fields.items(): if field in ( "assigned_team_ids", @@ -687,10 +701,10 @@ async def update_access_group( old_team_ids: Final[set[str]] = set(attached[access_group_id]) old_key_ids: Final[set[str]] = set(existing.assigned_key_ids or []) new_team_ids: Final[set[str]] = ( - set(update_fields["assigned_team_ids"] or []) if "assigned_team_ids" in update_fields else old_team_ids + set(data.assigned_team_ids or []) if "assigned_team_ids" in update_fields else old_team_ids ) new_key_ids: Final[set[str]] = ( - set(update_fields["assigned_key_ids"] or []) if "assigned_key_ids" in update_fields else old_key_ids + set(data.assigned_key_ids or []) if "assigned_key_ids" in update_fields else old_key_ids ) teams_to_add = list(new_team_ids - old_team_ids) diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index d40f64f435f..36198c02799 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -216,9 +216,12 @@ def _shadow_eval_attempts(prisma_client: "PrismaClient") -> _ShadowEvalAttemptTa return prisma_client.db.litellm_shadowevalattempt +_RAW_ROWS: Final = TypeAdapter(Sequence[dict[str, object]], config=ConfigDict(strict=True)) + + async def _query_raw(prisma_client: "PrismaClient", query: str, *args: object) -> Sequence[Mapping[str, object]]: async with db_span("auto_router_report_query", sql_relation(query)): - return await prisma_client.db.query_raw(query, *args) + return _RAW_ROWS.validate_python(await prisma_client.db.query_raw(query, *args)) async def _authorize_router_dry_run(user_api_key_dict: UserAPIKeyAuth, team_id: str | None) -> LiteLLM_TeamTable | None: diff --git a/litellm/proxy/management_endpoints/router_settings_endpoints.py b/litellm/proxy/management_endpoints/router_settings_endpoints.py index 64566221d4d..5cec608f7ae 100644 --- a/litellm/proxy/management_endpoints/router_settings_endpoints.py +++ b/litellm/proxy/management_endpoints/router_settings_endpoints.py @@ -10,7 +10,7 @@ GET /router/fields - Get router settings field definitions without values (for U import inspect from collections.abc import Mapping from types import MappingProxyType -from typing import Any, Final, get_args +from typing import Final, get_args from fastapi import APIRouter, Depends from pydantic import Field @@ -32,7 +32,7 @@ router: Final = APIRouter() class RouterSettingsResponse(LiteLLMBaseModel): fields: list[RouterSettingsField] = Field(description="List of all configurable router settings with metadata") - current_values: dict[str, Any] = Field(description="Current values of router settings") + current_values: dict[str, object] = Field(description="Current values of router settings") routing_strategy_descriptions: dict[str, str] = Field(description="Descriptions for each routing strategy option") routing_group_strategies: tuple[str, ...] = Field( description="Strategies supported when constructing a routing group" diff --git a/litellm/proxy/management_helpers/team_roster_sync.py b/litellm/proxy/management_helpers/team_roster_sync.py index dc0ba2e47d4..08be455a55d 100644 --- a/litellm/proxy/management_helpers/team_roster_sync.py +++ b/litellm/proxy/management_helpers/team_roster_sync.py @@ -40,6 +40,7 @@ from litellm.proxy.management_helpers.access_group_team_sync import ( ) from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.repositories.chunked_in import delete_many_in, find_many_in, update_many_in +from litellm.repositories.team_repository import TeamRepository if TYPE_CHECKING: from prisma import Prisma @@ -144,7 +145,7 @@ def _changes_nothing(plan: RosterPlan) -> bool: async def _unchanged_roster(prisma_client: PrismaClient, team_id: str) -> RosterSync | TeamGone: - team_row: Final = await prisma_client.writer_db.litellm_teamtable.find_unique(where={"team_id": team_id}) + team_row: Final = await TeamRepository(prisma_client, use_writer=True).table.find_unique(where={"team_id": team_id}) if team_row is None: return TeamGone(team_id=team_id) team: Final = LiteLLM_TeamTable.model_validate(team_row.model_dump()) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index 23ef89b3c5c..0c9ad69ffc0 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -5,7 +5,8 @@ from datetime import datetime from typing import TYPE_CHECKING, Any, Final, cast import httpx -from pydantic import ConfigDict, TypeAdapter +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_proxy_logger @@ -53,6 +54,14 @@ else: _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +@with_config(ConfigDict(extra="allow", strict=True)) +class _AnthropicBatchCreated(TypedDict): + id: ReadOnly[NotRequired[str]] + + +_ANTHROPIC_BATCH_CREATED: Final = TypeAdapter(_AnthropicBatchCreated) + + class AnthropicPassthroughLoggingHandler: @staticmethod def anthropic_passthrough_handler( @@ -552,6 +561,7 @@ class AnthropicPassthroughLoggingHandler: # Only handle successful batch job creation (POST requests with 201 status) if httpx_response.status_code == 200 and "id" in _json_response: + batch_created: Final = _ANTHROPIC_BATCH_CREATED.validate_python(_json_response) # Transform Anthropic response to LiteLLM batch format anthropic_batches_config: Final = AnthropicBatchesConfig() litellm_batch_response: Final = anthropic_batches_config.transform_retrieve_batch_response( @@ -565,7 +575,7 @@ class AnthropicPassthroughLoggingHandler: litellm_batch_response.status = "validating" # Extract batch ID from the response - batch_id: Final = _json_response.get("id", "") + batch_id: Final = batch_created.get("id", "") # Get model from request body (batch response doesn't include model) request_body = request_body or {} diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index e1e8864678e..751559b91d0 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -30,6 +30,7 @@ from pydantic import TypeAdapter from starlette.datastructures import UploadFile as StarletteUploadFile from starlette.routing import BaseRoute, Route from starlette.websockets import WebSocketState +from typing_extensions import ReadOnly from websockets.asyncio.client import connect from websockets.exceptions import ( ConnectionClosedError, @@ -168,8 +169,17 @@ def _is_mergeable_client_metadata(key: str, value: object) -> bool: return not key.startswith("user_api_key_") and (key != "model_info" or isinstance(value, Mapping)) +class _RegisteredPassThroughRoute(TypedDict): + endpoint_id: ReadOnly[str] + path: ReadOnly[str | None] + type: ReadOnly[str] + methods: ReadOnly[str | bool | list[str] | tuple[str, ...] | Mapping[str, object]] + auth: ReadOnly[bool] + passthrough_params: ReadOnly[Mapping[str, object]] + + # Global registry to track registered pass-through routes and prevent memory leaks -_registered_pass_through_routes: Final[dict[str, dict[str, str | bool | list[str] | Mapping[str, object]]]] = {} +_registered_pass_through_routes: Final[dict[str, _RegisteredPassThroughRoute]] = {} def get_response_body(response: httpx.Response) -> dict | None: @@ -2208,7 +2218,7 @@ def create_pass_through_route( status_code=status.HTTP_405_METHOD_NOT_ALLOWED, detail=f"Method {request.method} is not allowed for pass-through endpoint {path}.", ) - target_params: Final = { + target_params: Final[dict[str, object]] = { "target": target, "custom_headers": custom_headers, "forward_headers": _forward_headers, @@ -2240,7 +2250,9 @@ def create_pass_through_route( # Ensure custom_headers is a dict. Botocore returns a HeadersDict # for SigV4-prepared requests, which is a Mapping but not a dict. - headers_dict: Final = dict(param_custom_headers) if isinstance(param_custom_headers, Mapping) else {} + headers_dict: Final[dict[str, object]] = ( + {**param_custom_headers} if isinstance(param_custom_headers, Mapping) else {} + ) # Ensure query_params and custom_body are dicts or None final_query_params: Final = query_params_data if isinstance(query_params_data, dict) else {} @@ -3072,9 +3084,7 @@ class SafeRouteAdder: return f"{path}/{{subpath:path}}" return None - def registered_route_methods( - route_info: Mapping[str, str | bool | Sequence[str] | Mapping[str, object]], - ) -> tuple[tuple[str, str], ...]: + def registered_route_methods(route_info: _RegisteredPassThroughRoute) -> tuple[tuple[str, str], ...]: starlette_path: Final = registered_route_path(route_info) methods: Final = route_info.get("methods") if starlette_path is None or not isinstance(methods, (list, tuple)): @@ -3428,7 +3438,7 @@ class InitPassThroughEndpointHelpers: return (comparison_route, *forwarded) @staticmethod - def get_registered_pass_through_route(route: str, method: str | None = None) -> dict[str, Any] | None: + def get_registered_pass_through_route(route: str, method: str | None = None) -> _RegisteredPassThroughRoute | None: """Get passthrough params for a given route and optionally filter by HTTP method""" comparison_route: Final = InitPassThroughEndpointHelpers._route_for_registry_lookup(route) for key in _registered_pass_through_routes: diff --git a/litellm/proxy/spend_tracking/key_metadata_recovery.py b/litellm/proxy/spend_tracking/key_metadata_recovery.py index 5442c0f3f81..8fd4fbfcb8b 100644 --- a/litellm/proxy/spend_tracking/key_metadata_recovery.py +++ b/litellm/proxy/spend_tracking/key_metadata_recovery.py @@ -6,7 +6,7 @@ from datetime import datetime, timedelta from types import MappingProxyType from typing import Final, Literal, TypeVar -from pydantic import TypeAdapter +from pydantic import ConfigDict, TypeAdapter from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger @@ -155,6 +155,7 @@ class _DailyUserSpendOwnerRow(LiteLLMBaseModel): last_owner: str | None = None +_RAW_ROWS: Final = TypeAdapter(Sequence[dict[str, object]], config=ConfigDict(strict=True)) _TOKEN_DIGEST_ROWS: Final = TypeAdapter(tuple[_TokenDigestRow, ...]) _SPEND_LOG_DIGEST_ROWS: Final = TypeAdapter(tuple[_SpendLogDigestRow, ...]) _DAILY_USER_SPEND_OWNER_ROWS: Final = TypeAdapter(tuple[_DailyUserSpendOwnerRow, ...]) @@ -194,7 +195,7 @@ async def _rows_within_the_statement_timeout( await transaction.execute_raw(_SPEND_LOG_STATEMENT_TIMEOUT_SQL) for setting in planner_settings: await transaction.execute_raw(setting) - return await transaction.query_raw(sql, *params) + return _RAW_ROWS.validate_python(await transaction.query_raw(sql, *params)) async def _reverse_hash_key_metadata( diff --git a/litellm/secret_managers/aws_secret_manager_v2.py b/litellm/secret_managers/aws_secret_manager_v2.py index 98d12e497da..de8566e6fdc 100644 --- a/litellm/secret_managers/aws_secret_manager_v2.py +++ b/litellm/secret_managers/aws_secret_manager_v2.py @@ -21,6 +21,8 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Final import httpx +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger @@ -41,6 +43,15 @@ if TYPE_CHECKING: from botocore.awsrequest import HTTPHeaders +@with_config(ConfigDict(extra="allow", strict=True)) +class _SecretStringResponse(TypedDict): + SecretString: ReadOnly[str] + + +_SECRET_STRING_RESPONSE: Final = TypeAdapter(_SecretStringResponse) +_AWS_SECRET_MANAGER_RESPONSE: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True)) + + class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): def __init__( self, @@ -154,7 +165,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): try: response: Final = await async_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8")) response.raise_for_status() - return response.json()["SecretString"] + return _SECRET_STRING_RESPONSE.validate_python(response.json())["SecretString"] except httpx.TimeoutException: raise ValueError("Timeout error occurred") except Exception as e: @@ -204,7 +215,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): try: response: Final = sync_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8")) - return response.json()["SecretString"] + return _SECRET_STRING_RESPONSE.validate_python(response.json())["SecretString"] except httpx.TimeoutException: raise ValueError("Timeout error occurred") except httpx.HTTPStatusError as e: @@ -424,7 +435,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): try: response: Final = await async_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8")) response.raise_for_status() - return response.json() + return _AWS_SECRET_MANAGER_RESPONSE.validate_python(response.json()) except httpx.HTTPStatusError as err: raise ValueError(f"HTTP error occurred: {err.response.text}") except httpx.TimeoutException: @@ -481,7 +492,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): try: response: Final = await async_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8")) response.raise_for_status() - return response.json() + return _AWS_SECRET_MANAGER_RESPONSE.validate_python(response.json()) except httpx.HTTPStatusError as err: raise ValueError(f"HTTP error occurred: {err.response.text}") except httpx.TimeoutException: @@ -533,7 +544,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): try: response: Final = await async_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8")) response.raise_for_status() - return response.json() + return _AWS_SECRET_MANAGER_RESPONSE.validate_python(response.json()) except httpx.HTTPStatusError as err: raise ValueError(f"HTTP error occurred: {err.response.text}") except httpx.TimeoutException: @@ -613,7 +624,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): try: response: Final = await async_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8")) response.raise_for_status() - return response.json() + return _AWS_SECRET_MANAGER_RESPONSE.validate_python(response.json()) except httpx.HTTPStatusError as err: raise ValueError(f"HTTP error occurred: {err.response.text}") except httpx.TimeoutException: diff --git a/litellm/types/integrations/rag/bedrock_knowledgebase.py b/litellm/types/integrations/rag/bedrock_knowledgebase.py index 7156d8101e1..7288eca03b4 100644 --- a/litellm/types/integrations/rag/bedrock_knowledgebase.py +++ b/litellm/types/integrations/rag/bedrock_knowledgebase.py @@ -1,4 +1,4 @@ -from typing import Any, Literal +from typing import Literal from typing_extensions import ReadOnly, TypedDict @@ -40,7 +40,7 @@ class BedrockKBRetrievalResult(TypedDict, total=False): content: BedrockKBContent | None location: BedrockKBLocation | None score: float | None - metadata: dict[str, Any] | None + metadata: dict[str, object] | None class BedrockKBResponse(TypedDict, total=False): @@ -87,21 +87,21 @@ class BedrockKBMetadataConfiguration(TypedDict, total=False): class BedrockKBModelConfiguration(TypedDict, total=False): """Model configuration for reranking.""" - additionalModelRequestFields: dict[str, Any] | None + additionalModelRequestFields: dict[str, object] | None modelArn: str | None class BedrockKBRerankingConfiguration(TypedDict, total=False): """Configuration for reranking in vector search.""" - bedrockRerankingConfiguration: dict[str, Any] | None # This could be further typed if needed + bedrockRerankingConfiguration: dict[str, object] | None # This could be further typed if needed type: str | None class BedrockKBVectorSearchConfiguration(TypedDict, total=False): """Configuration for vector search.""" - filter: dict[str, Any] | None + filter: dict[str, object] | None implicitFilterConfiguration: BedrockKBImplicitFilterConfiguration | None numberOfResults: int | None overrideSearchType: str | None diff --git a/litellm/types/llms/bedrock_invoke_agents.py b/litellm/types/llms/bedrock_invoke_agents.py index 62f3c18ca7a..b334c986d6b 100644 --- a/litellm/types/llms/bedrock_invoke_agents.py +++ b/litellm/types/llms/bedrock_invoke_agents.py @@ -4,7 +4,7 @@ Type definitions for AWS Bedrock Invoke Agent API responses. https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agent-runtime_InvokeAgent.html """ -from typing import Any, Final +from typing import Final from typing_extensions import TypedDict @@ -39,7 +39,7 @@ class InvokeAgentModelInvocationInput(TypedDict, total=False): """Model invocation input details.""" foundationModel: str | None - inferenceConfiguration: dict[str, Any] | None + inferenceConfiguration: dict[str, object] | None text: str | None traceId: str | None type: str | None @@ -49,9 +49,9 @@ class InvokeAgentModelInvocationOutput(TypedDict, total=False): """Model invocation output details.""" metadata: InvokeAgentMetadata | None - parsedResponse: dict[str, Any] | None - rawResponse: dict[str, Any] | None - reasoningContent: dict[str, Any] | None + parsedResponse: dict[str, object] | None + rawResponse: dict[str, object] | None + reasoningContent: dict[str, object] | None traceId: str | None diff --git a/litellm/types/llms/gemini.py b/litellm/types/llms/gemini.py index c300c26bcd3..2ad81e64e05 100644 --- a/litellm/types/llms/gemini.py +++ b/litellm/types/llms/gemini.py @@ -1,5 +1,6 @@ +from collections.abc import Mapping from enum import Enum -from typing import Any, Literal +from typing import Literal from typing_extensions import Required, TypedDict @@ -235,7 +236,7 @@ class GeminiVideoGenerationInstance(TypedDict, total=False): """Instance data for Gemini video generation request""" prompt: Required[str] - image: dict[str, Any] + image: Mapping[str, object] class GeminiVideoGenerationParameters(LiteLLMBaseModel): @@ -263,7 +264,7 @@ class GeminiVideoGenerationParameters(LiteLLMBaseModel): negativePrompt: str | None = None """Text describing what not to include in the video.""" - lastFrame: Any | None = None + lastFrame: object | None = None """ The final image for interpolation video to transition. Should be used with the 'image' parameter. @@ -275,7 +276,7 @@ class GeminiVideoGenerationParameters(LiteLLMBaseModel): Only supported in Veo 3.1 (list of VideoGenerationReferenceImage objects). """ - video: Any | None = None + video: object | None = None """ Video to be used for video extension (Video object). Only supported in Veo 3.1 & Veo 3 Fast. @@ -354,5 +355,5 @@ class GeminiLongRunningOperationResponse(LiteLLMBaseModel): response: GeminiOperationResponse | None = None """Response object when operation is complete""" - error: dict[str, Any] | None = None + error: dict[str, object] | None = None """Error details if operation failed""" diff --git a/litellm/types/llms/langgraph.py b/litellm/types/llms/langgraph.py index 7e34329992d..83f7bb090d1 100644 --- a/litellm/types/llms/langgraph.py +++ b/litellm/types/llms/langgraph.py @@ -4,7 +4,8 @@ Type definitions for LangGraph API. LangGraph provides a streaming and non-streaming API for running agents. """ -from typing import Any, Literal +from collections.abc import Mapping +from typing import Literal from typing_extensions import TypedDict @@ -29,8 +30,8 @@ class LangGraphRequest(TypedDict, total=False): assistant_id: str input: LangGraphInput stream_mode: str | None - config: dict[str, Any] | None - metadata: dict[str, Any] | None + config: Mapping[str, object] | None + metadata: Mapping[str, object] | None # Response Types - Streaming @@ -38,7 +39,7 @@ class LangGraphStreamEvent(TypedDict, total=False): """Single event in a LangGraph stream response.""" event: str - data: Any + data: object # Response Types - Non-streaming @@ -55,7 +56,7 @@ class LangGraphResponse(TypedDict, total=False): """Non-streaming response structure from LangGraph.""" messages: list[LangGraphResponseMessage] - values: dict[str, Any] + values: dict[str, object] # Parsed response for internal use diff --git a/litellm/types/mcp_server/tool_registry.py b/litellm/types/mcp_server/tool_registry.py index 738d37c9503..080e27b6ed3 100644 --- a/litellm/types/mcp_server/tool_registry.py +++ b/litellm/types/mcp_server/tool_registry.py @@ -1,5 +1,5 @@ from collections.abc import Callable -from typing import Any, ClassVar +from typing import ClassVar from pydantic import ConfigDict, Field @@ -10,7 +10,7 @@ class MCPTool(LiteLLMBaseModel): model_config: ClassVar[ConfigDict] = ConfigDict(arbitrary_types_allowed=True) name: str description: str - input_schema: dict[str, Any] + input_schema: dict[str, object] handler: Callable server_id: str | None = Field(default=None, frozen=True) @@ -18,18 +18,18 @@ class MCPTool(LiteLLMBaseModel): class ToolSchema(LiteLLMBaseModel): name: str description: str - inputSchema: dict[str, Any] + inputSchema: dict[str, object] class ListToolsResponse(LiteLLMBaseModel): tools: list[ToolSchema] nextCursor: str | None = None - _meta: dict[str, Any] | None = None + _meta: dict[str, object] | None = None class CallToolRequest(LiteLLMBaseModel): method: str = "tools/call" - params: dict[str, Any] + params: dict[str, object] class ContentItem(LiteLLMBaseModel): diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/enkryptai.py b/litellm/types/proxy/guardrails/guardrail_hooks/enkryptai.py index d1eba1dd4a9..f936c9d85cf 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/enkryptai.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/enkryptai.py @@ -1,5 +1,3 @@ -from typing import Any - from pydantic import Field from typing_extensions import TypedDict @@ -20,7 +18,7 @@ class EnkryptAIPolicyViolationDetail(TypedDict, total=False): class EnkryptAIPIIDetail(TypedDict, total=False): """Details for PII detection.""" - pii: dict[str, Any] + pii: dict[str, object] class EnkryptAIToxicityDetail(TypedDict, total=False): @@ -85,8 +83,8 @@ class EnkryptAIResponseDetails(TypedDict, total=False): toxicity: EnkryptAIToxicityDetail | None keyword_detected: EnkryptAIKeywordDetail | None bias: EnkryptAIBiasDetail | None - prompt_injection: dict[str, Any] | None - jailbreak: dict[str, Any] | None + prompt_injection: dict[str, object] | None + jailbreak: dict[str, object] | None class EnkryptAIResponse(TypedDict, total=False): @@ -107,7 +105,7 @@ class EnkryptAIProcessedResult(TypedDict): | EnkryptAIToxicityDetail | EnkryptAIKeywordDetail | EnkryptAIBiasDetail - | dict[str, Any], + | dict[str, object], ]