This commit is contained in:
Mateo Wang 2026-10-24 08:25:25 +00:00 • committed by GitHub
commit 2b1f81bd2b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
50 changed files with 472 additions and 191 deletions

View file

@ -19,7 +19,7 @@ import importlib
import sys
from collections.abc import Callable, Mapping
from types import ModuleType
from typing import TYPE_CHECKING, Any, Final, cast
from typing import TYPE_CHECKING, Any, Final
from typing_extensions import ReadOnly, TypedDict
@ -456,7 +456,7 @@ def _lazy_import_http_handlers(name: str) -> object:
params: Final = {"timeout": async_timeout, "client_alias": "module level aclient"}
# Create the client instance
provider_id: Final = cast(Any, "litellm_module_level_client")
provider_id: Final = "litellm_module_level_client"
async_client: Final = get_async_httpx_client(
llm_provider=provider_id,
params=params,

View file

@ -173,6 +173,7 @@ class _MetricKwargs(TypedDict, total=False):
response_cost: ReadOnly[float | None]
completion_start_time: ReadOnly[datetime | float | str | None]
api_call_start_time: ReadOnly[datetime | float | str | None]
end_time: ReadOnly[datetime | float | str | None]
def resolve_error_type(kwargs: Mapping[str, Any]) -> str:
@ -414,13 +415,11 @@ class GenAIMetricRecorder:
if generation_time > 0:
self._metrics.time_per_output_token.record(generation_time / completion_tokens, attributes=common_attrs)
def _record_response_duration(self, kwargs: Mapping[str, Any], end_time: datetime, common_attrs: dict) -> None:
def _record_response_duration(self, kwargs: _MetricKwargs, end_time: datetime, common_attrs: dict) -> None:
api_call_start_time: Final = kwargs.get("api_call_start_time")
if api_call_start_time is None:
return
_end_time = kwargs.get("end_time") or end_time
if _end_time is None:
_end_time = datetime.now()
api_call_start: Final = to_seconds(api_call_start_time)
end_ts: Final = to_seconds(_end_time)
if api_call_start is None or end_ts is None:

View file

@ -3437,7 +3437,7 @@ class Logging(LiteLLMLoggingBaseClass):
start_time=start_time,
end_time=end_time,
print_verbose=print_verbose,
level=LogfireLevel.INFO.value,
level=LogfireLevel.INFO,
)
if callback == "lunary" and lunaryLogger is not None:
@ -4394,7 +4394,7 @@ class Logging(LiteLLMLoggingBaseClass):
response_obj=result,
start_time=start_time,
end_time=end_time,
level=LogfireLevel.ERROR.value,
level=LogfireLevel.ERROR,
print_verbose=print_verbose,
)

View file

@ -3182,10 +3182,8 @@ def anthropic_messages_pt(
def extract_between_tags(tag: str, string: str, strip: bool = False) -> list[str]:
ext_list = re.findall(f"<{tag}>(.+?)</{tag}>", string, re.DOTALL)
if strip:
ext_list = [e.strip() for e in ext_list]
return ext_list
matches: Final = re.finditer(f"<{tag}>(.+?)</{tag}>", string, re.DOTALL)
return [match.group(1).strip() if strip else match.group(1) for match in matches]
def contains_tag(tag: str, string: str) -> bool:
@ -6028,7 +6026,7 @@ def _parse_tool_call_arguments(raw: object, tool_name: str | None, context: str)
return ({},)
normalized_raw: Final = "{}" if raw == REDACTED_BY_LITELLM else raw
try:
parsed: Final = parse_tool_call_arguments(normalized_raw, tool_name=tool_name, context=context)
parsed: Final[object] = parse_tool_call_arguments(normalized_raw, tool_name=tool_name, context=context)
except ValueError as e:
salvaged: Final = salvage_concatenated_tool_arguments(normalized_raw)
if salvaged:

View file

@ -34,7 +34,7 @@ def safe_json_structure(
def _transform(key: str | None, value: str) -> str:
return value if value_transform is None else value_transform(key, value)
def _serialize(obj: object, seen: set[int], depth: int, key: str | None = None) -> Any:
def _serialize(obj: object, seen: set[int], depth: int, key: str | None = None) -> object:
# Check for maximum depth.
if depth > max_depth:
return "MaxDepthExceeded"
@ -48,32 +48,32 @@ def safe_json_structure(
if id(obj) in seen:
return "CircularReference Detected"
seen.add(id(obj))
result: dict | list | tuple | set | str
if isinstance(obj, dict):
result = {}
for k, v in obj.items():
if isinstance(k, (str)):
clean_k = k.replace("\x00", "") if "\x00" in k else k
result[clean_k] = _serialize(v, seen, depth + 1, clean_k)
mapping_result: Final = {
(clean_k := k.replace("\x00", "") if "\x00" in k else k): _serialize(v, seen, depth + 1, clean_k)
for k, v in obj.items()
if isinstance(k, str)
}
seen.remove(id(obj))
return result
return mapping_result
elif isinstance(obj, list):
result = [_serialize(item, seen, depth + 1, key) for item in obj]
list_result: Final = [_serialize(item, seen, depth + 1, key) for item in obj]
seen.remove(id(obj))
return result
return list_result
elif isinstance(obj, tuple):
result = tuple(_serialize(item, seen, depth + 1, key) for item in obj)
tuple_result: Final = tuple(_serialize(item, seen, depth + 1, key) for item in obj)
seen.remove(id(obj))
return result
return tuple_result
elif isinstance(obj, set):
result = sorted([_serialize(item, seen, depth + 1, key) for item in obj])
sortable_values: Final[list[Any]] = [_serialize(item, seen, depth + 1, key) for item in obj]
set_result: Final = sorted(sortable_values)
seen.remove(id(obj))
return result
return set_result
elif isinstance(obj, BaseModel):
dumped: Final = obj.model_dump()
result = _serialize(dumped, seen, depth + 1, key)
model_result: Final = _serialize(dumped, seen, depth + 1, key)
seen.remove(id(obj))
return result
return model_result
else:
# Fall back to string conversion for non-serializable objects.
try:

View file

@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Final, Optional, Union
from urllib.parse import quote
import httpx
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm._uuid import uuid
@ -52,6 +53,9 @@ else:
AsyncHTTPHandler = Any
_JSON_VALUE: Final = TypeAdapter(object)
class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
def __init__(self, **kwargs):
BaseConfig.__init__(self, **kwargs)
@ -314,7 +318,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
return None
try:
data: Final[object] = json.loads(json_str)
data: Final = _JSON_VALUE.validate_python(json.loads(json_str))
# Skip non-dict data (some lines contain JSON strings)
return data if isinstance(data, dict) else None
except json.JSONDecodeError:
@ -482,7 +486,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
if "application/json" in content_type:
# Direct JSON response
verbose_logger.debug("Parsing JSON response")
response_json: Final[object] = raw_response.json()
response_json: Final = _JSON_VALUE.validate_python(raw_response.json())
verbose_logger.debug("Response JSON: %s", response_json)
return self._parse_json_response(response_json)
else:
@ -566,7 +570,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
continue
try:
data_obj = json.loads(json_str)
data_obj = _JSON_VALUE.validate_python(json.loads(json_str))
if not isinstance(data_obj, dict):
continue
@ -706,7 +710,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
)
try:
body: Final = response.read()
response_json: Final[object] = json.loads(body)
response_json: Final = _JSON_VALUE.validate_python(json.loads(body))
except (json.JSONDecodeError, Exception) as e:
raise BedrockError(
status_code=response.status_code,
@ -788,7 +792,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
continue
try:
data_obj = json.loads(json_str)
data_obj = _JSON_VALUE.validate_python(json.loads(json_str))
if not isinstance(data_obj, dict):
continue
@ -928,7 +932,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
)
try:
body: Final = await response.aread()
response_json: Final[object] = json.loads(body)
response_json: Final = _JSON_VALUE.validate_python(json.loads(body))
except (json.JSONDecodeError, Exception) as e:
raise BedrockError(
status_code=response.status_code,

View file

@ -12,7 +12,8 @@ from itertools import chain
from typing import TYPE_CHECKING, Final, Literal, cast, overload
import httpx
from pydantic import TypeAdapter, ValidationError
from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
@ -129,6 +130,16 @@ from ..common_utils import (
normalize_bedrock_opus_output_config_effort,
)
@with_config(ConfigDict(extra="allow", strict=True))
class _JsonContentToolCall(TypedDict):
type: ReadOnly[NotRequired[object]]
name: ReadOnly[NotRequired[str]]
arguments: ReadOnly[NotRequired[str | dict[str, object] | None]]
_JSON_CONTENT_TOOL_CALL: Final = TypeAdapter(_JsonContentToolCall)
if TYPE_CHECKING:
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
@ -2453,7 +2464,7 @@ class AmazonConverseConfig(BaseConfig):
if message.content is not None:
try:
tool_call_names: Final = self.get_tool_call_names(tools)
json_content: Final = json.loads(message.content)
json_content: Final = _JSON_CONTENT_TOOL_CALL.validate_python(json.loads(message.content))
if json_content.get("type") == "function" and json_content.get("name") in tool_call_names:
tool_calls: Final = [ChatCompletionMessageToolCall(function=Function(**json_content))]

View file

@ -10,6 +10,8 @@ import time
from typing import TYPE_CHECKING, Any, Final
import httpx
from pydantic import ConfigDict, TypeAdapter, with_config
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
@ -34,6 +36,15 @@ else:
LiteLLMLoggingObj = Any
@with_config(ConfigDict(extra="allow", strict=True))
class _PegasusResponse(TypedDict):
message: ReadOnly[NotRequired[str]]
finishReason: ReadOnly[NotRequired[str]]
_PEGASUS_RESPONSE: Final = TypeAdapter(_PegasusResponse)
class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
"""
Handles transforming OpenAI-style requests into Bedrock InvokeModel requests for
@ -207,7 +218,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
ModelResponse with choices[0].message.content and finish_reason
"""
try:
completion_response: Final = raw_response.json()
completion_response: Final = _PEGASUS_RESPONSE.validate_python(raw_response.json())
except Exception as e:
raise BedrockError(
message=f"Error parsing response: {raw_response.text}, error: {e}",

View file

@ -1,11 +1,12 @@
import copy
import json
import time
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, cast, get_args
import httpx
from pydantic import TypeAdapter, ValidationError
from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
@ -48,6 +49,27 @@ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
_GUARDRAIL_CONFIG_VALIDATOR: Final["TypeAdapter[GuardrailConfigBlock]"] = TypeAdapter(GuardrailConfigBlock)
@with_config(ConfigDict(extra="allow", strict=True))
class _CohereGeneration(TypedDict):
text: ReadOnly[str | None]
finish_reason: ReadOnly[str]
@with_config(ConfigDict(extra="allow", strict=True))
class _CohereInvokeResponse(TypedDict):
text: NotRequired[ReadOnly[str | None]]
generations: NotRequired[ReadOnly[Sequence[_CohereGeneration]]]
@with_config(ConfigDict(extra="allow", strict=True))
class _LlamaInvokeResponse(TypedDict):
generation: ReadOnly[str | None]
_COHERE_INVOKE_RESPONSE: Final = TypeAdapter(_CohereInvokeResponse)
_LLAMA_INVOKE_RESPONSE: Final = TypeAdapter(_LlamaInvokeResponse)
_GUARDRAIL_CONFIG_EXPECTED_FORMAT: Final = (
"{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}"
)
@ -310,12 +332,13 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
outputText: str | None = None
try:
if provider == "cohere":
if "text" in completion_response:
outputText = completion_response["text"]
elif "generations" in completion_response:
outputText = completion_response["generations"][0]["text"]
cohere_response: Final = _COHERE_INVOKE_RESPONSE.validate_python(completion_response)
if "text" in cohere_response:
outputText = cohere_response["text"]
elif "generations" in cohere_response:
outputText = cohere_response["generations"][0]["text"]
model_response.choices[0].finish_reason = map_finish_reason(
completion_response["generations"][0]["finish_reason"]
cohere_response["generations"][0]["finish_reason"]
)
elif provider == "anthropic":
return litellm.AmazonAnthropicClaudeConfig().transform_response(
@ -361,7 +384,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
elif provider == "ai21":
outputText = completion_response.get("completions")[0].get("data").get("text")
elif provider == "meta" or provider == "llama" or provider == "deepseek_r1":
outputText = completion_response["generation"]
outputText = _LLAMA_INVOKE_RESPONSE.validate_python(completion_response)["generation"]
elif provider == "mistral":
outputText = litellm.AmazonMistralConfig.get_outputText(completion_response, model_response)
else: # amazon titan

View file

@ -10,7 +10,7 @@ from itertools import accumulate, chain
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Optional, Protocol, TypeAlias, TypeVar
from pydantic import InstanceOf, TypeAdapter, ValidationError
from pydantic import ConfigDict, InstanceOf, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
@ -768,6 +768,7 @@ class InvokeStreamFrame:
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
_STRICT_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(strict=True))
_Frame: Final = TypeVar("_Frame", ConverseStreamFrame, InvokeStreamFrame)
@ -1217,7 +1218,7 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
texts: Sequence[tuple[_StreamGroupKey, str]] = ()
if event_type == "contentBlockDelta":
try:
payload_dict: dict[str, object] = json.loads(payload_bytes)
payload_dict = _STRICT_JSON_OBJECT.validate_python(json.loads(payload_bytes))
texts = _delta_frame_texts(payload_dict)
except Exception as e:
verbose_proxy_logger.debug(
@ -1255,7 +1256,9 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
headers_bytes = frame_raw[12 : 12 + orig_hdrs_len]
try:
payload_dict = json.loads(frame_raw[12 + orig_hdrs_len : orig_total - 4])
payload_dict = _STRICT_JSON_OBJECT.validate_python(
json.loads(frame_raw[12 + orig_hdrs_len : orig_total - 4])
)
for local_idx, (_, container, key) in enumerate(
_collect_stream_delta_text_holders(payload_dict.get("delta"))
):

View file

@ -53,6 +53,7 @@ from litellm.types.realtime import (
RealtimeResponseTypedDict,
)
_JSON_OBJECT_MESSAGE: Final = TypeAdapter(dict[str, JsonValue], config=ConfigDict(strict=True))
_JSON_VALUE: Final = TypeAdapter(object)
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
@ -856,7 +857,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
List of Bedrock format messages (JSON strings)
"""
try:
json_message: Final[dict[str, JsonValue]] = json.loads(message)
json_message: Final = _JSON_OBJECT_MESSAGE.validate_python(json.loads(message))
except json.JSONDecodeError:
verbose_logger.warning("Invalid JSON message: %s", message[:200])
return []

View file

@ -51,6 +51,7 @@ class _BflPolledResult(TypedDict):
_BFL_POLLED_RESULT_ADAPTER: Final = TypeAdapter(_BflPolledResult)
_JSON_VALUE: Final = TypeAdapter(object)
_BFL_REQUEST_PARAMS: Final = (
@ -316,7 +317,7 @@ class BlackForestLabsImageEditConfig(BaseImageEditConfig):
The response contains: {"status": "Ready", "result": {"sample": "https://..."}}
"""
try:
raw_response_data: Final[object] = raw_response.json()
raw_response_data: Final = _JSON_VALUE.validate_python(raw_response.json())
except Exception as e:
raise BlackForestLabsError(
status_code=raw_response.status_code,

View file

@ -11,7 +11,7 @@ from __future__ import annotations
import itertools
import json
from collections.abc import Mapping, Sequence
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass, field
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final
@ -231,12 +231,16 @@ def _system_events(event: Mapping[str, object], state: ClaudeCodeStreamState) ->
return event_list()
def _record_result(event: Mapping[str, Any], state: ClaudeCodeStreamState) -> Sequence[Event]:
def _record_result(event: Mapping[str, object], state: ClaudeCodeStreamState) -> Sequence[Event]:
state.result_seen = True
state.is_error = bool(event.get("is_error", False))
result = event.get("result")
state.result_text = result if isinstance(result, str) else None
state.errors = [str(e) for e in event.get("errors") or ()] # mutable-ok: mirrors the JSON errors array
errors: Final = event.get("errors") or ()
if not isinstance(errors, Iterable):
raise TypeError(f"'{type(errors).__name__}' object is not iterable")
error_items: Final[Iterable[object]] = errors
state.errors = [str(e) for e in error_items] # mutable-ok: mirrors the JSON errors array
state.structured_output = event.get("structured_output")
if event.get("session_id"):
state.session_id = str(event["session_id"])
@ -301,7 +305,7 @@ class ClaudeCodeHarnessConfig(BaseCLIHarnessConfig):
options: ClaudeCodeOptions = self.get_options(ctx)
model = ctx.model
# Background calls (titles, summaries) use the same model group, like OpenCode.
model_env: Final = (
model_env: Final[Mapping[str, str]] = (
MappingProxyType({"ANTHROPIC_MODEL": model, "ANTHROPIC_SMALL_FAST_MODEL": model})
if model
else MappingProxyType({})

View file

@ -14,6 +14,8 @@ Docs - https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api
from typing import Final
import httpx
from pydantic import ConfigDict, TypeAdapter, with_config
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.url_utils import ensure_api_route
@ -28,6 +30,23 @@ from ..common_utils import DashScopeError
DEFAULT_API_BASE: Final = "https://dashscope.aliyuncs.com/compatible-mode/v1"
@with_config(ConfigDict(extra="allow", strict=True))
class _DashScopeUsage(TypedDict, total=False):
prompt_tokens: NotRequired[ReadOnly[int]]
total_tokens: NotRequired[ReadOnly[int]]
@with_config(ConfigDict(extra="allow", strict=True))
class _DashScopeEmbeddingResponse(TypedDict, total=False):
data: NotRequired[ReadOnly[list[object]]]
model: NotRequired[ReadOnly[str | None]]
usage: NotRequired[ReadOnly[_DashScopeUsage | None]]
id: NotRequired[ReadOnly[object]]
_DASHSCOPE_EMBEDDING_RESPONSE: Final = TypeAdapter(_DashScopeEmbeddingResponse)
class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
"""
Reference: https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api
@ -152,11 +171,12 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
message=message,
)
parsed_response: Final = _DASHSCOPE_EMBEDDING_RESPONSE.validate_python(response_json)
model_response.object = "list"
model_response.data = response_json.get("data", [])
model_response.model = response_json.get("model", model)
model_response.data = parsed_response.get("data", [])
model_response.model = parsed_response.get("model", model)
usage: Final = response_json.get("usage") or {}
usage: Final = parsed_response.get("usage") or {}
prompt_tokens: Final = usage.get("prompt_tokens", 0)
total_tokens: Final = usage.get("total_tokens", prompt_tokens)
setattr(
@ -169,8 +189,8 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
),
)
if "id" in response_json:
setattr(model_response, "id", response_json["id"])
if "id" in parsed_response:
setattr(model_response, "id", parsed_response["id"])
return model_response

View file

@ -1,6 +1,8 @@
import json
from typing import Final
from pydantic import ConfigDict, TypeAdapter
import litellm
from litellm import verbose_logger
from litellm.types.llms.openai import (
@ -23,6 +25,9 @@ def _usage_block(usage: Usage) -> ChatCompletionUsageBlock:
)
_JSON_CHUNK: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
class ModelResponseIterator:
def __init__(self, streaming_response, sync_stream: bool):
self.streaming_response = streaming_response
@ -99,7 +104,7 @@ class ModelResponseIterator:
self.response_iterator = self.streaming_response
chunk: Final = self._next_sse_data()
try:
json_chunk: Final = json.loads(chunk)
json_chunk: Final = _JSON_CHUNK.validate_python(json.loads(chunk))
return self.chunk_parser(chunk=json_chunk)
except ValueError as e:
verbose_logger.debug(
@ -126,7 +131,7 @@ class ModelResponseIterator:
if chunk == "[DONE]":
raise StopAsyncIteration
try:
json_chunk: Final = json.loads(chunk)
json_chunk: Final = _JSON_CHUNK.validate_python(json.loads(chunk))
return self.chunk_parser(chunk=json_chunk)
except ValueError as e:
verbose_logger.debug(

View file

@ -8,6 +8,7 @@ Docs: https://www.edenai.co/docs/v3/llms/embeddings
from typing import TYPE_CHECKING, Final
import httpx
from pydantic import ConfigDict, TypeAdapter
from litellm.litellm_core_utils.core_helpers import set_response_cost_in_hidden_params
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@ -23,6 +24,8 @@ if TYPE_CHECKING:
_SUPPORTED_PARAMS: Final = ("dimensions", "encoding_format", "user")
_EMBEDDING_BODY: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
class EdenAIEmbeddingConfig(BaseEmbeddingConfig):
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: inherited contract
@ -80,7 +83,7 @@ class EdenAIEmbeddingConfig(BaseEmbeddingConfig):
optional_params: dict[str, object], # mutable-ok: inherited contract
litellm_params: dict[str, object], # mutable-ok: inherited contract
) -> EmbeddingResponse:
body: Final = raw_response.json()
body: Final = _EMBEDDING_BODY.validate_python(raw_response.json())
logging_obj.post_call(original_response=body)
response: Final[EmbeddingResponse] = convert_to_model_response_object(
response_object=body, model_response_object=model_response, response_type="embedding"

View file

@ -12,6 +12,8 @@ from types import MappingProxyType
from typing import Final
import httpx
from pydantic import ConfigDict, TypeAdapter, with_config
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm._logging import verbose_logger
from litellm.caching.caching import InMemoryCache
@ -37,6 +39,17 @@ _token_cache: Final = InMemoryCache()
_NO_LITELLM_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
@with_config(ConfigDict(extra="allow", strict=True))
class _GigaChatTokenResponse(TypedDict):
tok: NotRequired[ReadOnly[str | None]]
access_token: NotRequired[ReadOnly[str | None]]
exp: NotRequired[ReadOnly[int | float | str | None]]
expires_at: NotRequired[ReadOnly[int | float | str | None]]
_TOKEN_RESPONSE: Final = TypeAdapter(_GigaChatTokenResponse)
class GigaChatAuthError(BaseLLMException):
"""GigaChat authentication error."""
@ -230,7 +243,7 @@ async def _request_token_async(
def _parse_token_response(response: httpx.Response) -> tuple[str, int]:
"""Parse OAuth token response."""
data: Final = response.json()
data: Final = _TOKEN_RESPONSE.validate_python(response.json())
# GigaChat returns either 'tok'/'exp' or 'access_token'/'expires_at'
access_token: Final = data.get("tok") or data.get("access_token")

View file

@ -5,6 +5,7 @@ from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Final
import httpx
from pydantic import ConfigDict, TypeAdapter
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
from litellm.llms.gigachat.authenticator import get_access_token
@ -20,6 +21,8 @@ if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.types.utils import CostResponseTypes
_STREAM_CHUNK: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
class GigaChatPassthroughConfig(BasePassthroughConfig):
def is_streaming_request(self, endpoint: str, request_data: Mapping[str, object]) -> bool:
@ -161,7 +164,7 @@ class GigaChatPassthroughConfig(BasePassthroughConfig):
continue
chunk = chunk.removeprefix("data: ")
try:
message = json.loads(chunk)
message = _STREAM_CHUNK.validate_python(json.loads(chunk))
except json.JSONDecodeError:
continue

View file

@ -5,7 +5,7 @@ This requires websockets, and is currently only supported on LiteLLM Proxy.
"""
import ssl
from typing import Any, Final, cast
from typing import TYPE_CHECKING, Final, cast
from litellm._logging import redact_string, verbose_logger
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
@ -23,6 +23,9 @@ from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context
from ..common_utils import is_openai_backed_api_base
from ..openai import OpenAIChatCompletion
if TYPE_CHECKING:
from fastapi import WebSocket
class OpenAIRealtime(OpenAIChatCompletion):
"""
@ -131,7 +134,7 @@ class OpenAIRealtime(OpenAIChatCompletion):
async def async_realtime(
self,
model: str,
websocket: Any,
websocket: "WebSocket",
logging_obj: LiteLLMLogging,
api_base: str | None = None,
api_key: str | None = None,

View file

@ -12,12 +12,14 @@ This module decodes them into float arrays for OpenAI-compatible responses.
"""
import base64
import builtins
import struct
from collections.abc import Mapping
from typing import Final
from collections.abc import Mapping, Sequence
from typing import Final, Literal
import httpx
from pydantic import ConfigDict, TypeAdapter
from pydantic import ConfigDict, TypeAdapter, with_config
from typing_extensions import ReadOnly, TypedDict
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.url_utils import join_api_base
@ -31,6 +33,17 @@ _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_i
_INT: Final = TypeAdapter(int)
@with_config(ConfigDict(extra="allow", strict=True))
class _PerplexityEmbeddingResponse(TypedDict, total=False):
model: ReadOnly[str | None]
object: ReadOnly[Literal["list"]]
data: ReadOnly[Sequence[builtins.object] | Mapping[str, builtins.object] | Literal[""]]
usage: ReadOnly[builtins.object]
_EMBEDDING_RESPONSE_ADAPTER: Final = TypeAdapter(_PerplexityEmbeddingResponse)
class PerplexityEmbeddingError(BaseLLMException):
def __init__(
self,
@ -153,14 +166,18 @@ class PerplexityEmbeddingConfig(BaseEmbeddingConfig):
litellm_params: dict = {},
) -> EmbeddingResponse:
try:
raw_response_json: Final = raw_response.json()
raw_response_json: Final[object] = raw_response.json()
except Exception:
raise PerplexityEmbeddingError(message=raw_response.text, status_code=raw_response.status_code)
model_response.model = raw_response_json.get("model", model)
model_response.object = raw_response_json.get("object", "list")
response_data: Final[_PerplexityEmbeddingResponse] = _EMBEDDING_RESPONSE_ADAPTER.validate_python(
raw_response_json
)
raw_data: Final = raw_response_json.get("data", [])
model_response.model = response_data.get("model", model)
model_response.object = response_data.get("object", "list")
raw_data: Final = response_data.get("data", [])
decoded_data: Final[list[dict[str, object]]] = []
for item in raw_data:
decoded_item = dict(_JSON_OBJECT.validate_python(item))
@ -168,7 +185,7 @@ class PerplexityEmbeddingConfig(BaseEmbeddingConfig):
decoded_data.append(decoded_item)
model_response.data = decoded_data
usage_data: Final = _JSON_OBJECT.validate_python(raw_response_json.get("usage", {}))
usage_data: Final = _JSON_OBJECT.validate_python(response_data.get("usage", {}))
usage: Final = Usage(
prompt_tokens=_INT.validate_python(
usage_data.get("prompt_tokens", 0) or usage_data.get("total_tokens", 0) or 0

View file

@ -6,11 +6,12 @@ Maps OpenAI TTS spec to RunwayML Text-to-Speech API
import asyncio
import time
from collections.abc import Coroutine, Sequence
from typing import TYPE_CHECKING, Any, Final, TypedDict, Union
from collections.abc import Coroutine
from typing import TYPE_CHECKING, Any, Final, Union
import httpx
from typing_extensions import ReadOnly
from pydantic import ConfigDict, TypeAdapter, with_config
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
@ -33,12 +34,17 @@ else:
HttpxBinaryResponseContent = Any
@with_config(ConfigDict(extra="allow", strict=True))
class _RunwayTtsTaskResponse(TypedDict, total=False):
id: ReadOnly[str]
status: ReadOnly[str]
output: ReadOnly[Sequence[object]]
failure: ReadOnly[str]
failureCode: ReadOnly[str]
output: ReadOnly[object]
failure: ReadOnly[object]
failureCode: ReadOnly[object]
_RUNWAY_TTS_TASK_RESPONSE_ADAPTER: Final = TypeAdapter(_RunwayTtsTaskResponse)
_RUNWAY_TTS_OUTPUT_ADAPTER: Final = TypeAdapter(list[object], config=ConfigDict(strict=True))
class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
@ -321,7 +327,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
response = client.get(url=task_url, headers=headers)
response.raise_for_status()
response_data: _RunwayTtsTaskResponse = response.json()
response_data = _RUNWAY_TTS_TASK_RESPONSE_ADAPTER.validate_python(response.json())
# Check task status
status = self._check_task_status(response_data=response_data)
@ -368,7 +374,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
response = await client.get(url=task_url, headers=headers)
response.raise_for_status()
response_data: _RunwayTtsTaskResponse = response.json()
response_data = _RUNWAY_TTS_TASK_RESPONSE_ADAPTER.validate_python(response.json())
# Check task status
status = self._check_task_status(response_data=response_data)
@ -459,7 +465,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
from litellm.types.llms.openai import HttpxBinaryResponseContent
try:
response_data: Final[_RunwayTtsTaskResponse] = raw_response.json()
response_data: Final = _RUNWAY_TTS_TASK_RESPONSE_ADAPTER.validate_python(raw_response.json())
except Exception as e:
raise self.get_error_class(
error_message=f"Error parsing RunwayML TTS response: {e}",
@ -489,16 +495,20 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
)
# Get the completed task data
task_data: Final[_RunwayTtsTaskResponse] = polled_response.json()
task_data: Final = _RUNWAY_TTS_TASK_RESPONSE_ADAPTER.validate_python(polled_response.json())
verbose_logger.debug("RunwayML TTS polling complete, downloading audio")
# Get audio URL from output
output: Final = task_data.get("output", [])
if not output or not isinstance(output, list) or len(output) == 0:
if not output or not isinstance(output, list):
raise ValueError("RunwayML TTS response missing audio URL in output")
audio_url: Final = output[0]
output_items: Final = _RUNWAY_TTS_OUTPUT_ADAPTER.validate_python(output)
if len(output_items) == 0:
raise ValueError("RunwayML TTS response missing audio URL in output")
audio_url: Final = output_items[0]
if not isinstance(audio_url, str):
raise ValueError(f"RunwayML TTS audio URL is not a string: {audio_url}")
@ -528,7 +538,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
from litellm.types.llms.openai import HttpxBinaryResponseContent
try:
response_data: Final[_RunwayTtsTaskResponse] = raw_response.json()
response_data: Final = _RUNWAY_TTS_TASK_RESPONSE_ADAPTER.validate_python(raw_response.json())
except Exception as e:
raise self.get_error_class(
error_message=f"Error parsing RunwayML TTS response: {e}",
@ -558,16 +568,20 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
)
# Get the completed task data
task_data: Final[_RunwayTtsTaskResponse] = polled_response.json()
task_data: Final = _RUNWAY_TTS_TASK_RESPONSE_ADAPTER.validate_python(polled_response.json())
verbose_logger.debug("RunwayML TTS polling complete (async), downloading audio")
# Get audio URL from output
output: Final = task_data.get("output", [])
if not output or not isinstance(output, list) or len(output) == 0:
if not output or not isinstance(output, list):
raise ValueError("RunwayML TTS response missing audio URL in output")
audio_url: Final = output[0]
output_items: Final = _RUNWAY_TTS_OUTPUT_ADAPTER.validate_python(output)
if len(output_items) == 0:
raise ValueError("RunwayML TTS response missing audio URL in output")
audio_url: Final = output_items[0]
if not isinstance(audio_url, str):
raise ValueError(f"RunwayML TTS audio URL is not a string: {audio_url}")

View file

@ -1,6 +1,9 @@
from collections.abc import Sequence
from typing import Final
import httpx
from pydantic import ConfigDict, TypeAdapter, with_config
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.llms.base_llm.chat.transformation import AllMessageValues, BaseLLMException
from litellm.llms.base_llm.embedding.transformation import (
@ -14,6 +17,21 @@ from litellm.utils import token_counter
from ..common_utils import TritonError
@with_config(ConfigDict(extra="allow", strict=True))
class _TritonEmbeddingOutput(TypedDict):
shape: ReadOnly[Sequence[int]]
data: ReadOnly[Sequence[int | float]]
@with_config(ConfigDict(extra="allow", strict=True))
class _TritonEmbeddingResponse(TypedDict):
outputs: ReadOnly[Sequence[_TritonEmbeddingOutput]]
model_name: ReadOnly[NotRequired[str | None]]
_TRITON_EMBEDDING_RESPONSE: Final = TypeAdapter(_TritonEmbeddingResponse)
class TritonEmbeddingConfig(BaseEmbeddingConfig):
"""
Transformations for triton /embeddings endpoint (This is a trtllm model)
@ -83,9 +101,10 @@ class TritonEmbeddingConfig(BaseEmbeddingConfig):
except Exception:
raise TritonError(message=raw_response.text, status_code=raw_response.status_code)
parsed_response: Final = _TRITON_EMBEDDING_RESPONSE.validate_python(raw_response_json)
_embedding_output: Final = []
_outputs: Final = raw_response_json["outputs"]
_outputs: Final = parsed_response["outputs"]
for output in _outputs:
_shape = output["shape"]
_data = output["data"]
@ -100,7 +119,7 @@ class TritonEmbeddingConfig(BaseEmbeddingConfig):
}
)
model_response.model = raw_response_json.get("model_name", "None")
model_response.model = parsed_response.get("model_name", "None")
model_response.data = _embedding_output
model_response.usage = self._build_embedding_usage(model=model, request_data=request_data)
return model_response
@ -134,7 +153,7 @@ class TritonEmbeddingConfig(BaseEmbeddingConfig):
return TritonError(message=error_message, status_code=status_code, headers=headers)
@staticmethod
def split_embedding_by_shape(data: list[float], shape: list[int]) -> list[list[float]]:
def split_embedding_by_shape(data: Sequence[int | float], shape: Sequence[int]) -> Sequence[Sequence[int | float]]:
if len(shape) != 2:
raise ValueError("Shape must be of length 2.")
embedding_size: Final = shape[1]

View file

@ -20,6 +20,7 @@ if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
_PREDICTIONS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True))
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
class VertexImageGeneration(VertexLLM):
@ -168,7 +169,7 @@ class VertexImageGeneration(VertexLLM):
if response.status_code != 200:
raise Exception(f"Error: {response.status_code} {response.text}")
json_response: Final = response.json()
json_response: Final = _JSON_OBJECT.validate_python(response.json())
return self.process_image_generation_response(json_response, model_response, model)
async def aimage_generation(
@ -272,7 +273,7 @@ class VertexImageGeneration(VertexLLM):
if response.status_code != 200:
raise Exception(f"Error: {response.status_code} {response.text}")
json_response: Final = response.json()
json_response: Final = _JSON_OBJECT.validate_python(response.json())
return self.process_image_generation_response(json_response, model_response, model)
def is_image_generation_response(self, json_response: dict[str, Any]) -> bool:

View file

@ -9,10 +9,11 @@ The actual message transformation reuses OpenAIGPTConfig since Gemma uses OpenAI
"""
from collections.abc import Callable
from typing import TYPE_CHECKING, Any, Final, cast
from typing import TYPE_CHECKING, Final, cast
import httpx
from pydantic import ValidationError
from pydantic import ConfigDict, TypeAdapter, ValidationError
from typing_extensions import NotRequired, ReadOnly, TypeAliasType, TypedDict
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.custom_httpx.http_handler import (
@ -32,6 +33,17 @@ if TYPE_CHECKING:
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
class _PredictionsEnvelope(TypedDict):
predictions: NotRequired[ReadOnly[dict[str, object]]]
_PredictionsEnvelopeType = TypeAliasType("_PredictionsEnvelopeType", _PredictionsEnvelope)
_PREDICTIONS_ENVELOPE: Final[TypeAdapter[_PredictionsEnvelope]] = TypeAdapter(
_PredictionsEnvelopeType,
config=ConfigDict(strict=True, extra="allow"),
)
def parse_vertex_gemma_container_error(predictions: object) -> VertexGemmaContainerError | None:
try:
return VertexGemmaContainerError.model_validate(predictions)
@ -142,7 +154,7 @@ class VertexGemmaConfig(OpenAIGPTConfig):
def _unwrap_predictions_response(
self,
response_json: dict[str, Any],
response_json: _PredictionsEnvelope,
) -> dict[str, object]:
"""
Unwrap the Vertex Gemma predictions format to OpenAI format.
@ -354,7 +366,7 @@ class VertexGemmaConfig(OpenAIGPTConfig):
message=f"Request failed: {response.text}",
)
response_json: Final = response.json()
response_json: Final = _PREDICTIONS_ENVELOPE.validate_python(response.json())
# Unwrap predictions to get OpenAI-compatible response
openai_response: Final = self._unwrap_predictions_response(response_json)
@ -449,7 +461,7 @@ class VertexGemmaConfig(OpenAIGPTConfig):
message=f"Request failed: {response.text}",
)
response_json: Final = response.json()
response_json: Final = _PREDICTIONS_ENVELOPE.validate_python(response.json())
# Unwrap predictions to get OpenAI-compatible response
openai_response: Final = self._unwrap_predictions_response(response_json)

View file

@ -3,6 +3,7 @@ from types import UnionType
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, Union, get_args, get_origin
import httpx
from pydantic import ConfigDict, TypeAdapter
from pydantic import fields as pyd_fields
import litellm
@ -37,6 +38,8 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObj = Any
_RESPONSE_BODY: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
class _EventModelClass(Protocol):
@property
@ -415,7 +418,7 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig):
@staticmethod
def _parsed_response_body(raw_response: httpx.Response) -> dict[str, object]:
return raw_response.json()
return _RESPONSE_BODY.validate_python(raw_response.json())
@staticmethod
def _annotation_origin(annotation: object) -> object:

View file

@ -300,12 +300,13 @@ def decode_state_hash(encrypted_state: str) -> dict:
if decrypted_json is None:
raise ValueError("Failed to decrypt state parameter")
state_data: Final = json.loads(decrypted_json)
state_data: Final = _DECODED_STATE.validate_python(json.loads(decrypted_json))
return state_data
_MANAGED_AUTH_CODE_PREFIX: Final = "llm_scope_"
_DECODED_JSON: Final = TypeAdapter(object)
_DECODED_STATE: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
class _ManagedAuthorizationCode(LiteLLMBaseModel):

View file

@ -11,6 +11,7 @@ from collections.abc import Mapping
from typing import TYPE_CHECKING, Final
import httpx
from pydantic import TypeAdapter
from litellm._internal_context import with_service_target
from litellm._logging import verbose_logger
@ -42,6 +43,9 @@ if TYPE_CHECKING:
from litellm.types.mcp_server.mcp_server_manager import MCPServer
_JSON_VALUE: Final = TypeAdapter(object)
class MCPOAuth2TokenCache(InMemoryCache):
"""
In-memory cache for OAuth2 client_credentials tokens, keyed by the identity of the token
@ -160,7 +164,7 @@ class MCPOAuth2TokenCache(InMemoryCache):
f"failed with status {exc.response.status_code}"
) from exc
body: Final = response.json()
body: Final = _JSON_VALUE.validate_python(response.json())
if not isinstance(body, dict):
raise ValueError(

View file

@ -18,7 +18,8 @@ from typing import Final, Literal, Protocol, TypeAlias
import jwt
from fastapi import HTTPException
from jwt.types import Options
from typing_extensions import assert_never
from pydantic import ConfigDict, TypeAdapter, with_config
from typing_extensions import ReadOnly, TypedDict, assert_never
from litellm._logging import verbose_logger
from litellm.caching.in_memory_cache import InMemoryCache
@ -38,6 +39,20 @@ _ALLOWED_ID_TOKEN_ALGORITHMS: Final = (
"PS512",
)
_JWKS_CACHE_TTL_SECONDS: Final = 3600
@with_config(ConfigDict(extra="allow", strict=True))
class _JwksDocument(TypedDict, total=False):
keys: ReadOnly[Sequence[Mapping[str, object]]]
@with_config(ConfigDict(extra="allow", strict=True))
class _OidcDiscoveryDocument(TypedDict, total=False):
jwks_uri: ReadOnly[object]
_JWKS_DOCUMENT: Final = TypeAdapter(_JwksDocument)
_OIDC_DISCOVERY_DOCUMENT: Final = TypeAdapter(_OidcDiscoveryDocument)
_jwks_cache: Final = InMemoryCache(default_ttl=_JWKS_CACHE_TTL_SECONDS)
JwksFetcher: TypeAlias = Callable[
@ -103,8 +118,8 @@ async def _fetch_issuer_jwks(binding: MCPOAuthIdentityBinding) -> Sequence[Mappi
client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.Oauth2Check)
response: Final = await client.get(jwks_url)
response.raise_for_status()
document: Final = response.json()
keys: Final = document.get("keys") if isinstance(document, dict) else None
document: Final = _JWKS_DOCUMENT.validate_python(response.json())
keys: Final = document.get("keys")
if not isinstance(keys, list):
raise TypeError(f"JWKS document at {jwks_url} has no 'keys' array")
await _jwks_cache.async_set_cache(jwks_url, keys, ttl=_JWKS_CACHE_TTL_SECONDS)
@ -116,8 +131,8 @@ async def _discover_jwks_url(issuer: str) -> str:
client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.Oauth2Check)
response: Final = await client.get(discovery_url)
response.raise_for_status()
metadata: Final = response.json()
jwks_uri: Final = metadata.get("jwks_uri") if isinstance(metadata, dict) else None
metadata: Final = _OIDC_DISCOVERY_DOCUMENT.validate_python(response.json())
jwks_uri: Final = metadata.get("jwks_uri")
if not isinstance(jwks_uri, str) or not jwks_uri:
raise ValueError(f"OIDC discovery at {discovery_url} returned no jwks_uri")
return jwks_uri

View file

@ -6,7 +6,9 @@ Filters MCP tools semantically for /chat/completions and /responses endpoints.
import asyncio
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Final
from typing_extensions import TypeVar
from litellm._logging import verbose_logger
from litellm.exceptions import ContextWindowExceededError
@ -21,6 +23,9 @@ if TYPE_CHECKING:
from litellm.router import Router
_ToolT: Final = TypeVar("_ToolT", default=object)
class SemanticToolFilterContextWindowError(Exception):
"""Raised when the embedding model exceeds its context window, so semantic filtering cannot run."""
@ -265,9 +270,9 @@ class SemanticMCPToolFilter:
async def filter_tools(
self,
query: str,
available_tools: list[Any],
available_tools: list[_ToolT],
top_k: int | None = None,
) -> list[Any]:
) -> list[_ToolT]:
"""
Filter tools semantically based on query.
@ -390,7 +395,7 @@ class SemanticMCPToolFilter:
separator: Final = client_name[-len(canonical) - 1]
return separator in ("_", "-")
def _get_tools_by_names(self, tool_names: Sequence[str], available_tools: Sequence[object]) -> list[object]:
def _get_tools_by_names(self, tool_names: Sequence[str], available_tools: Sequence[_ToolT]) -> list[_ToolT]:
"""
Get tools from available_tools by their names, preserving the
semantic router's ordering.
@ -406,13 +411,13 @@ class SemanticMCPToolFilter:
# Exact matches win over suffix matches when both are present, and
# each incoming tool is returned at most once even if two canonical
# names happen to be tail-compatible with the same incoming name.
available_by_name: Final[dict[str, object]] = {}
available_by_name: Final[dict[str, _ToolT]] = {}
for tool in available_tools:
client_name, _ = self.extract_tool_info(tool)
if client_name and client_name not in available_by_name:
available_by_name[client_name] = tool
matched: Final[list[object]] = []
matched: Final[list[_ToolT]] = []
used_ids: Final[set[int]] = set()
for canonical in tool_names:
tool = available_by_name.get(canonical)

View file

@ -3,9 +3,12 @@ from collections.abc import Mapping
from typing import Any, Final
import requests
from pydantic import ConfigDict, TypeAdapter
from .exceptions import NotFoundError, UnauthorizedError
_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
class ModelsManagementClient:
def __init__(self, base_url: str, api_key: str | None = None, timeout: int = 30):
@ -109,7 +112,7 @@ class ModelsManagementClient:
try:
response: Final = session.send(request.prepare(), timeout=self._timeout)
response.raise_for_status()
return response.json()
return _JSON_OBJECT.validate_python(response.json())
except requests.exceptions.HTTPError as e:
if e.response.status_code == 401:
raise UnauthorizedError(e)
@ -145,7 +148,7 @@ class ModelsManagementClient:
try:
response: Final = session.send(request.prepare(), timeout=self._timeout)
response.raise_for_status()
return response.json()
return _JSON_OBJECT.validate_python(response.json())
except requests.exceptions.HTTPError as e:
if e.response.status_code == 401:
raise UnauthorizedError(e)
@ -287,7 +290,7 @@ class ModelsManagementClient:
try:
response: Final = session.send(request.prepare(), timeout=self._timeout)
response.raise_for_status()
return response.json()
return _JSON_OBJECT.validate_python(response.json())
except requests.exceptions.HTTPError as e:
if e.response.status_code == 401:
raise UnauthorizedError(e)

View file

@ -9,7 +9,7 @@ import json
import os
import uuid
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict
from typing import TYPE_CHECKING, Any, Final, Literal
try:
import ulid
@ -28,6 +28,8 @@ except ImportError:
HTTPX_AVAILABLE = False
from fastapi import HTTPException
from pydantic import ConfigDict, TypeAdapter, with_config
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm import DualCache
@ -51,13 +53,17 @@ from litellm.proxy.guardrails._content_utils import (
from litellm.types.guardrails import GuardrailEventHooks
class LassoResponse(TypedDict):
@with_config(ConfigDict(extra="allow", strict=True))
class LassoResponse(TypedDict, total=False):
"""Type definition for Lasso API response."""
violations_detected: bool
deputies: dict[str, bool]
findings: dict[str, list[dict[str, object]]]
messages: list[dict[str, str]] | None
violations_detected: NotRequired[ReadOnly[object]]
deputies: NotRequired[ReadOnly[dict[str, object]]]
findings: NotRequired[ReadOnly[dict[str, object]]]
messages: NotRequired[ReadOnly[list[dict[str, object]] | None]]
_LASSO_RESPONSE: Final = TypeAdapter(LassoResponse)
if TYPE_CHECKING:
@ -467,7 +473,9 @@ class LassoGuardrail(CustomGuardrail):
# Also update data["input"] for Responses-API payloads so the
# unredacted text doesn't leak through that field.
if isinstance(data.get("input"), str):
text_parts = [msg["content"] for msg in masked_for_input if isinstance(msg.get("content"), str)]
text_parts: Final = [
content for msg in masked_for_input if isinstance(content := msg.get("content"), str)
]
if text_parts:
data["input"] = "\n".join(text_parts)
self._log_masking_applied(message_type, dict(response))
@ -820,7 +828,7 @@ class LassoGuardrail(CustomGuardrail):
timeout=self.timeout if self.timeout is not None else 10.0,
)
response.raise_for_status()
return response.json()
return _LASSO_RESPONSE.validate_python(response.json())
def _process_lasso_response(self, response: LassoResponse) -> None:
"""

View file

@ -13,6 +13,8 @@ from datetime import datetime
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, TypeAlias, cast
from urllib.parse import urlparse
from pydantic import ConfigDict, TypeAdapter
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_guardrail import (
CustomGuardrail,
@ -37,6 +39,7 @@ if TYPE_CHECKING:
_DEFAULT_API_BASE: Final = "https://api.noma.security/"
_AIDR_SCAN_ENDPOINT: Final = "/litellm/guardrail"
_AIDR_RESPONSE: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
_INTERVENED_INPUT_FIELDS: Final = ("texts", "images", "tools", "tool_calls")
_DEFAULT_API_BASE_HOSTNAME: Final = urlparse(_DEFAULT_API_BASE).hostname
@ -257,7 +260,7 @@ class NomaV2Guardrail(CustomGuardrail):
response.text,
)
response.raise_for_status()
response_json: Final[dict[str, object]] = response.json()
response_json: Final = _AIDR_RESPONSE.validate_python(response.json())
verbose_proxy_logger.debug(
"Noma v2 AIDR response parsed: %s",
json.dumps(response_json, default=str),

View file

@ -164,7 +164,7 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
from litellm.types.llms.openai import OpenAIModerationResponse
return OpenAIModerationResponse(**response.json())
return OpenAIModerationResponse.model_validate(response.json())
def _check_moderation_result(self, moderation_response: "OpenAIModerationResponse") -> None:
"""

View file

@ -10,6 +10,8 @@ import os
from typing import Any, Final, Literal
from fastapi import HTTPException
from pydantic import ConfigDict, TypeAdapter, with_config
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_guardrail import (
@ -31,6 +33,26 @@ GUARDRAIL_NAME: Final = "qualifire"
DEFAULT_QUALIFIRE_API_BASE: Final = "https://proxy.qualifire.ai"
@with_config(ConfigDict(extra="allow", strict=True))
class _QualifireFlaggedResult(TypedDict, total=False):
flagged: NotRequired[ReadOnly[object]]
@with_config(ConfigDict(extra="allow", strict=True))
class _QualifireEvaluationResult(TypedDict, total=False):
results: NotRequired[ReadOnly[list[_QualifireFlaggedResult] | None]]
@with_config(ConfigDict(extra="allow", strict=True))
class _QualifireResponse(TypedDict, total=False):
score: NotRequired[ReadOnly[object]]
status: NotRequired[ReadOnly[object]]
evaluationResults: NotRequired[ReadOnly[list[_QualifireEvaluationResult] | None]]
_QUALIFIRE_RESPONSE: Final = TypeAdapter(_QualifireResponse)
class QualifireGuardrail(CustomGuardrail):
@classmethod
def get_supported_event_hooks(cls) -> list[GuardrailEventHooks]:
@ -244,7 +266,7 @@ class QualifireGuardrail(CustomGuardrail):
return api_tools if api_tools else None
def _check_if_flagged(self, result: dict[str, Any]) -> bool:
def _check_if_flagged(self, result: _QualifireResponse) -> bool:
"""
Check if the Qualifire evaluation result indicates flagged content.
@ -381,7 +403,7 @@ class QualifireGuardrail(CustomGuardrail):
timeout=self.timeout,
)
response.raise_for_status()
result: Final = response.json()
result: Final = _QUALIFIRE_RESPONSE.validate_python(response.json())
# Extract response info for logging
qualifire_response: Final[dict[str, object]] = {

View file

@ -101,7 +101,7 @@ if TYPE_CHECKING:
from litellm.types.agents import AgentResponse
from litellm.types.caching import RedisPipelineIncrementOperation
Span = _Span | Any
Span = _Span
InternalUsageCache = _InternalUsageCache
else:
Span = Any

View file

@ -649,7 +649,7 @@ def _metadata_string_tags(metadata_objects: Iterable[Mapping[str, object]]) -> t
def _strip_untrusted_request_header_controls(
headers: Any,
headers: object,
*,
allow_client_message_redaction_opt_out: bool = False,
) -> None:
@ -1579,7 +1579,7 @@ class LiteLLMProxyRequestSetup:
@staticmethod
def add_internal_user_from_user_mapping(
general_settings: dict | None,
general_settings: Mapping[str, object] | None,
user_api_key_dict: UserAPIKeyAuth,
headers: dict,
) -> UserAPIKeyAuth:
@ -1598,7 +1598,7 @@ class LiteLLMProxyRequestSetup:
return user_api_key_dict
@staticmethod
def get_user_from_headers(headers: dict, general_settings: dict | None = None) -> str | None:
def get_user_from_headers(headers: dict, general_settings: Mapping[str, object] | None = None) -> str | None:
"""
Get the user from the specified header if `general_settings.user_header_name` is set.
"""
@ -1619,7 +1619,9 @@ class LiteLLMProxyRequestSetup:
return user
@staticmethod
def get_openai_org_id_from_headers(headers: dict, general_settings: dict | None = None) -> str | None:
def get_openai_org_id_from_headers(
headers: dict, general_settings: Mapping[str, object] | None = None
) -> str | None:
"""
Get the OpenAI Org ID from the headers.
"""
@ -1710,7 +1712,7 @@ class LiteLLMProxyRequestSetup:
headers: dict,
request_data: Mapping[str, object],
user_api_key_dict: UserAPIKeyAuth,
general_settings: dict[str, object] | None = None,
general_settings: Mapping[str, object] | None = None,
) -> LitellmDataForBackendLLMCall:
"""
- Adds user from headers

View file

@ -647,8 +647,22 @@ async def update_access_group(
_require_proxy_admin(user_api_key_dict)
prisma_client: Final = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value)
update_fields: Final = data.model_dump(exclude_unset=True)
update_data: Final[dict] = {"updated_by": user_api_key_dict.user_id}
update_fields: Final = {
field: value
for field, value in {
"access_group_name": data.access_group_name,
"description": data.description,
"access_model_names": data.access_model_names,
"access_mcp_server_ids": data.access_mcp_server_ids,
"mcp_tool_permissions": data.mcp_tool_permissions,
"access_agent_ids": data.access_agent_ids,
"access_skill_names": data.access_skill_names,
"assigned_team_ids": data.assigned_team_ids,
"assigned_key_ids": data.assigned_key_ids,
}.items()
if field in data.model_fields_set
}
update_data: Final[dict[str, object]] = {"updated_by": user_api_key_dict.user_id}
for field, value in update_fields.items():
if field in (
"assigned_team_ids",
@ -687,10 +701,10 @@ async def update_access_group(
old_team_ids: Final[set[str]] = set(attached[access_group_id])
old_key_ids: Final[set[str]] = set(existing.assigned_key_ids or [])
new_team_ids: Final[set[str]] = (
set(update_fields["assigned_team_ids"] or []) if "assigned_team_ids" in update_fields else old_team_ids
set(data.assigned_team_ids or []) if "assigned_team_ids" in update_fields else old_team_ids
)
new_key_ids: Final[set[str]] = (
set(update_fields["assigned_key_ids"] or []) if "assigned_key_ids" in update_fields else old_key_ids
set(data.assigned_key_ids or []) if "assigned_key_ids" in update_fields else old_key_ids
)
teams_to_add = list(new_team_ids - old_team_ids)

View file

@ -216,9 +216,12 @@ def _shadow_eval_attempts(prisma_client: "PrismaClient") -> _ShadowEvalAttemptTa
return prisma_client.db.litellm_shadowevalattempt
_RAW_ROWS: Final = TypeAdapter(Sequence[dict[str, object]], config=ConfigDict(strict=True))
async def _query_raw(prisma_client: "PrismaClient", query: str, *args: object) -> Sequence[Mapping[str, object]]:
async with db_span("auto_router_report_query", sql_relation(query)):
return await prisma_client.db.query_raw(query, *args)
return _RAW_ROWS.validate_python(await prisma_client.db.query_raw(query, *args))
async def _authorize_router_dry_run(user_api_key_dict: UserAPIKeyAuth, team_id: str | None) -> LiteLLM_TeamTable | None:

View file

@ -10,7 +10,7 @@ GET /router/fields - Get router settings field definitions without values (for U
import inspect
from collections.abc import Mapping
from types import MappingProxyType
from typing import Any, Final, get_args
from typing import Final, get_args
from fastapi import APIRouter, Depends
from pydantic import Field
@ -32,7 +32,7 @@ router: Final = APIRouter()
class RouterSettingsResponse(LiteLLMBaseModel):
fields: list[RouterSettingsField] = Field(description="List of all configurable router settings with metadata")
current_values: dict[str, Any] = Field(description="Current values of router settings")
current_values: dict[str, object] = Field(description="Current values of router settings")
routing_strategy_descriptions: dict[str, str] = Field(description="Descriptions for each routing strategy option")
routing_group_strategies: tuple[str, ...] = Field(
description="Strategies supported when constructing a routing group"

View file

@ -40,6 +40,7 @@ from litellm.proxy.management_helpers.access_group_team_sync import (
)
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.repositories.chunked_in import delete_many_in, find_many_in, update_many_in
from litellm.repositories.team_repository import TeamRepository
if TYPE_CHECKING:
from prisma import Prisma
@ -144,7 +145,7 @@ def _changes_nothing(plan: RosterPlan) -> bool:
async def _unchanged_roster(prisma_client: PrismaClient, team_id: str) -> RosterSync | TeamGone:
team_row: Final = await prisma_client.writer_db.litellm_teamtable.find_unique(where={"team_id": team_id})
team_row: Final = await TeamRepository(prisma_client, use_writer=True).table.find_unique(where={"team_id": team_id})
if team_row is None:
return TeamGone(team_id=team_id)
team: Final = LiteLLM_TeamTable.model_validate(team_row.model_dump())

View file

@ -5,7 +5,8 @@ from datetime import datetime
from typing import TYPE_CHECKING, Any, Final, cast
import httpx
from pydantic import ConfigDict, TypeAdapter
from pydantic import ConfigDict, TypeAdapter, with_config
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_proxy_logger
@ -53,6 +54,14 @@ else:
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
@with_config(ConfigDict(extra="allow", strict=True))
class _AnthropicBatchCreated(TypedDict):
id: ReadOnly[NotRequired[str]]
_ANTHROPIC_BATCH_CREATED: Final = TypeAdapter(_AnthropicBatchCreated)
class AnthropicPassthroughLoggingHandler:
@staticmethod
def anthropic_passthrough_handler(
@ -552,6 +561,7 @@ class AnthropicPassthroughLoggingHandler:
# Only handle successful batch job creation (POST requests with 201 status)
if httpx_response.status_code == 200 and "id" in _json_response:
batch_created: Final = _ANTHROPIC_BATCH_CREATED.validate_python(_json_response)
# Transform Anthropic response to LiteLLM batch format
anthropic_batches_config: Final = AnthropicBatchesConfig()
litellm_batch_response: Final = anthropic_batches_config.transform_retrieve_batch_response(
@ -565,7 +575,7 @@ class AnthropicPassthroughLoggingHandler:
litellm_batch_response.status = "validating"
# Extract batch ID from the response
batch_id: Final = _json_response.get("id", "")
batch_id: Final = batch_created.get("id", "")
# Get model from request body (batch response doesn't include model)
request_body = request_body or {}

View file

@ -30,6 +30,7 @@ from pydantic import TypeAdapter
from starlette.datastructures import UploadFile as StarletteUploadFile
from starlette.routing import BaseRoute, Route
from starlette.websockets import WebSocketState
from typing_extensions import ReadOnly
from websockets.asyncio.client import connect
from websockets.exceptions import (
ConnectionClosedError,
@ -168,8 +169,17 @@ def _is_mergeable_client_metadata(key: str, value: object) -> bool:
return not key.startswith("user_api_key_") and (key != "model_info" or isinstance(value, Mapping))
class _RegisteredPassThroughRoute(TypedDict):
endpoint_id: ReadOnly[str]
path: ReadOnly[str | None]
type: ReadOnly[str]
methods: ReadOnly[str | bool | list[str] | tuple[str, ...] | Mapping[str, object]]
auth: ReadOnly[bool]
passthrough_params: ReadOnly[Mapping[str, object]]
# Global registry to track registered pass-through routes and prevent memory leaks
_registered_pass_through_routes: Final[dict[str, dict[str, str | bool | list[str] | Mapping[str, object]]]] = {}
_registered_pass_through_routes: Final[dict[str, _RegisteredPassThroughRoute]] = {}
def get_response_body(response: httpx.Response) -> dict | None:
@ -2208,7 +2218,7 @@ def create_pass_through_route(
status_code=status.HTTP_405_METHOD_NOT_ALLOWED,
detail=f"Method {request.method} is not allowed for pass-through endpoint {path}.",
)
target_params: Final = {
target_params: Final[dict[str, object]] = {
"target": target,
"custom_headers": custom_headers,
"forward_headers": _forward_headers,
@ -2240,7 +2250,9 @@ def create_pass_through_route(
# Ensure custom_headers is a dict. Botocore returns a HeadersDict
# for SigV4-prepared requests, which is a Mapping but not a dict.
headers_dict: Final = dict(param_custom_headers) if isinstance(param_custom_headers, Mapping) else {}
headers_dict: Final[dict[str, object]] = (
{**param_custom_headers} if isinstance(param_custom_headers, Mapping) else {}
)
# Ensure query_params and custom_body are dicts or None
final_query_params: Final = query_params_data if isinstance(query_params_data, dict) else {}
@ -3072,9 +3084,7 @@ class SafeRouteAdder:
return f"{path}/{{subpath:path}}"
return None
def registered_route_methods(
route_info: Mapping[str, str | bool | Sequence[str] | Mapping[str, object]],
) -> tuple[tuple[str, str], ...]:
def registered_route_methods(route_info: _RegisteredPassThroughRoute) -> tuple[tuple[str, str], ...]:
starlette_path: Final = registered_route_path(route_info)
methods: Final = route_info.get("methods")
if starlette_path is None or not isinstance(methods, (list, tuple)):
@ -3428,7 +3438,7 @@ class InitPassThroughEndpointHelpers:
return (comparison_route, *forwarded)
@staticmethod
def get_registered_pass_through_route(route: str, method: str | None = None) -> dict[str, Any] | None:
def get_registered_pass_through_route(route: str, method: str | None = None) -> _RegisteredPassThroughRoute | None:
"""Get passthrough params for a given route and optionally filter by HTTP method"""
comparison_route: Final = InitPassThroughEndpointHelpers._route_for_registry_lookup(route)
for key in _registered_pass_through_routes:

View file

@ -6,7 +6,7 @@ from datetime import datetime, timedelta
from types import MappingProxyType
from typing import Final, Literal, TypeVar
from pydantic import TypeAdapter
from pydantic import ConfigDict, TypeAdapter
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
@ -155,6 +155,7 @@ class _DailyUserSpendOwnerRow(LiteLLMBaseModel):
last_owner: str | None = None
_RAW_ROWS: Final = TypeAdapter(Sequence[dict[str, object]], config=ConfigDict(strict=True))
_TOKEN_DIGEST_ROWS: Final = TypeAdapter(tuple[_TokenDigestRow, ...])
_SPEND_LOG_DIGEST_ROWS: Final = TypeAdapter(tuple[_SpendLogDigestRow, ...])
_DAILY_USER_SPEND_OWNER_ROWS: Final = TypeAdapter(tuple[_DailyUserSpendOwnerRow, ...])
@ -194,7 +195,7 @@ async def _rows_within_the_statement_timeout(
await transaction.execute_raw(_SPEND_LOG_STATEMENT_TIMEOUT_SQL)
for setting in planner_settings:
await transaction.execute_raw(setting)
return await transaction.query_raw(sql, *params)
return _RAW_ROWS.validate_python(await transaction.query_raw(sql, *params))
async def _reverse_hash_key_metadata(

View file

@ -21,6 +21,8 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Final
import httpx
from pydantic import ConfigDict, TypeAdapter, with_config
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
@ -41,6 +43,15 @@ if TYPE_CHECKING:
from botocore.awsrequest import HTTPHeaders
@with_config(ConfigDict(extra="allow", strict=True))
class _SecretStringResponse(TypedDict):
SecretString: ReadOnly[str]
_SECRET_STRING_RESPONSE: Final = TypeAdapter(_SecretStringResponse)
_AWS_SECRET_MANAGER_RESPONSE: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager):
def __init__(
self,
@ -154,7 +165,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager):
try:
response: Final = await async_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8"))
response.raise_for_status()
return response.json()["SecretString"]
return _SECRET_STRING_RESPONSE.validate_python(response.json())["SecretString"]
except httpx.TimeoutException:
raise ValueError("Timeout error occurred")
except Exception as e:
@ -204,7 +215,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager):
try:
response: Final = sync_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8"))
return response.json()["SecretString"]
return _SECRET_STRING_RESPONSE.validate_python(response.json())["SecretString"]
except httpx.TimeoutException:
raise ValueError("Timeout error occurred")
except httpx.HTTPStatusError as e:
@ -424,7 +435,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager):
try:
response: Final = await async_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8"))
response.raise_for_status()
return response.json()
return _AWS_SECRET_MANAGER_RESPONSE.validate_python(response.json())
except httpx.HTTPStatusError as err:
raise ValueError(f"HTTP error occurred: {err.response.text}")
except httpx.TimeoutException:
@ -481,7 +492,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager):
try:
response: Final = await async_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8"))
response.raise_for_status()
return response.json()
return _AWS_SECRET_MANAGER_RESPONSE.validate_python(response.json())
except httpx.HTTPStatusError as err:
raise ValueError(f"HTTP error occurred: {err.response.text}")
except httpx.TimeoutException:
@ -533,7 +544,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager):
try:
response: Final = await async_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8"))
response.raise_for_status()
return response.json()
return _AWS_SECRET_MANAGER_RESPONSE.validate_python(response.json())
except httpx.HTTPStatusError as err:
raise ValueError(f"HTTP error occurred: {err.response.text}")
except httpx.TimeoutException:
@ -613,7 +624,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager):
try:
response: Final = await async_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8"))
response.raise_for_status()
return response.json()
return _AWS_SECRET_MANAGER_RESPONSE.validate_python(response.json())
except httpx.HTTPStatusError as err:
raise ValueError(f"HTTP error occurred: {err.response.text}")
except httpx.TimeoutException:

View file

@ -1,4 +1,4 @@
from typing import Any, Literal
from typing import Literal
from typing_extensions import ReadOnly, TypedDict
@ -40,7 +40,7 @@ class BedrockKBRetrievalResult(TypedDict, total=False):
content: BedrockKBContent | None
location: BedrockKBLocation | None
score: float | None
metadata: dict[str, Any] | None
metadata: dict[str, object] | None
class BedrockKBResponse(TypedDict, total=False):
@ -87,21 +87,21 @@ class BedrockKBMetadataConfiguration(TypedDict, total=False):
class BedrockKBModelConfiguration(TypedDict, total=False):
"""Model configuration for reranking."""
additionalModelRequestFields: dict[str, Any] | None
additionalModelRequestFields: dict[str, object] | None
modelArn: str | None
class BedrockKBRerankingConfiguration(TypedDict, total=False):
"""Configuration for reranking in vector search."""
bedrockRerankingConfiguration: dict[str, Any] | None # This could be further typed if needed
bedrockRerankingConfiguration: dict[str, object] | None # This could be further typed if needed
type: str | None
class BedrockKBVectorSearchConfiguration(TypedDict, total=False):
"""Configuration for vector search."""
filter: dict[str, Any] | None
filter: dict[str, object] | None
implicitFilterConfiguration: BedrockKBImplicitFilterConfiguration | None
numberOfResults: int | None
overrideSearchType: str | None

View file

@ -4,7 +4,7 @@ Type definitions for AWS Bedrock Invoke Agent API responses.
https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agent-runtime_InvokeAgent.html
"""
from typing import Any, Final
from typing import Final
from typing_extensions import TypedDict
@ -39,7 +39,7 @@ class InvokeAgentModelInvocationInput(TypedDict, total=False):
"""Model invocation input details."""
foundationModel: str | None
inferenceConfiguration: dict[str, Any] | None
inferenceConfiguration: dict[str, object] | None
text: str | None
traceId: str | None
type: str | None
@ -49,9 +49,9 @@ class InvokeAgentModelInvocationOutput(TypedDict, total=False):
"""Model invocation output details."""
metadata: InvokeAgentMetadata | None
parsedResponse: dict[str, Any] | None
rawResponse: dict[str, Any] | None
reasoningContent: dict[str, Any] | None
parsedResponse: dict[str, object] | None
rawResponse: dict[str, object] | None
reasoningContent: dict[str, object] | None
traceId: str | None

View file

@ -1,5 +1,6 @@
from collections.abc import Mapping
from enum import Enum
from typing import Any, Literal
from typing import Literal
from typing_extensions import Required, TypedDict
@ -235,7 +236,7 @@ class GeminiVideoGenerationInstance(TypedDict, total=False):
"""Instance data for Gemini video generation request"""
prompt: Required[str]
image: dict[str, Any]
image: Mapping[str, object]
class GeminiVideoGenerationParameters(LiteLLMBaseModel):
@ -263,7 +264,7 @@ class GeminiVideoGenerationParameters(LiteLLMBaseModel):
negativePrompt: str | None = None
"""Text describing what not to include in the video."""
lastFrame: Any | None = None
lastFrame: object | None = None
"""
The final image for interpolation video to transition.
Should be used with the 'image' parameter.
@ -275,7 +276,7 @@ class GeminiVideoGenerationParameters(LiteLLMBaseModel):
Only supported in Veo 3.1 (list of VideoGenerationReferenceImage objects).
"""
video: Any | None = None
video: object | None = None
"""
Video to be used for video extension (Video object).
Only supported in Veo 3.1 & Veo 3 Fast.
@ -354,5 +355,5 @@ class GeminiLongRunningOperationResponse(LiteLLMBaseModel):
response: GeminiOperationResponse | None = None
"""Response object when operation is complete"""
error: dict[str, Any] | None = None
error: dict[str, object] | None = None
"""Error details if operation failed"""

View file

@ -4,7 +4,8 @@ Type definitions for LangGraph API.
LangGraph provides a streaming and non-streaming API for running agents.
"""
from typing import Any, Literal
from collections.abc import Mapping
from typing import Literal
from typing_extensions import TypedDict
@ -29,8 +30,8 @@ class LangGraphRequest(TypedDict, total=False):
assistant_id: str
input: LangGraphInput
stream_mode: str | None
config: dict[str, Any] | None
metadata: dict[str, Any] | None
config: Mapping[str, object] | None
metadata: Mapping[str, object] | None
# Response Types - Streaming
@ -38,7 +39,7 @@ class LangGraphStreamEvent(TypedDict, total=False):
"""Single event in a LangGraph stream response."""
event: str
data: Any
data: object
# Response Types - Non-streaming
@ -55,7 +56,7 @@ class LangGraphResponse(TypedDict, total=False):
"""Non-streaming response structure from LangGraph."""
messages: list[LangGraphResponseMessage]
values: dict[str, Any]
values: dict[str, object]
# Parsed response for internal use

View file

@ -1,5 +1,5 @@
from collections.abc import Callable
from typing import Any, ClassVar
from typing import ClassVar
from pydantic import ConfigDict, Field
@ -10,7 +10,7 @@ class MCPTool(LiteLLMBaseModel):
model_config: ClassVar[ConfigDict] = ConfigDict(arbitrary_types_allowed=True)
name: str
description: str
input_schema: dict[str, Any]
input_schema: dict[str, object]
handler: Callable
server_id: str | None = Field(default=None, frozen=True)
@ -18,18 +18,18 @@ class MCPTool(LiteLLMBaseModel):
class ToolSchema(LiteLLMBaseModel):
name: str
description: str
inputSchema: dict[str, Any]
inputSchema: dict[str, object]
class ListToolsResponse(LiteLLMBaseModel):
tools: list[ToolSchema]
nextCursor: str | None = None
_meta: dict[str, Any] | None = None
_meta: dict[str, object] | None = None
class CallToolRequest(LiteLLMBaseModel):
method: str = "tools/call"
params: dict[str, Any]
params: dict[str, object]
class ContentItem(LiteLLMBaseModel):

View file

@ -1,5 +1,3 @@
from typing import Any
from pydantic import Field
from typing_extensions import TypedDict
@ -20,7 +18,7 @@ class EnkryptAIPolicyViolationDetail(TypedDict, total=False):
class EnkryptAIPIIDetail(TypedDict, total=False):
"""Details for PII detection."""
pii: dict[str, Any]
pii: dict[str, object]
class EnkryptAIToxicityDetail(TypedDict, total=False):
@ -85,8 +83,8 @@ class EnkryptAIResponseDetails(TypedDict, total=False):
toxicity: EnkryptAIToxicityDetail | None
keyword_detected: EnkryptAIKeywordDetail | None
bias: EnkryptAIBiasDetail | None
prompt_injection: dict[str, Any] | None
jailbreak: dict[str, Any] | None
prompt_injection: dict[str, object] | None
jailbreak: dict[str, object] | None
class EnkryptAIResponse(TypedDict, total=False):
@ -107,7 +105,7 @@ class EnkryptAIProcessedResult(TypedDict):
| EnkryptAIToxicityDetail
| EnkryptAIKeywordDetail
| EnkryptAIBiasDetail
| dict[str, Any],
| dict[str, object],
]