mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-21 10:15:47 +00:00
Merge 2e4c8e0259 into e04e89c701
This commit is contained in:
commit
2b1f81bd2b
50 changed files with 472 additions and 191 deletions
|
|
@ -19,7 +19,7 @@ import importlib
|
|||
import sys
|
||||
from collections.abc import Callable, Mapping
|
||||
from types import ModuleType
|
||||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
|
|
@ -456,7 +456,7 @@ def _lazy_import_http_handlers(name: str) -> object:
|
|||
params: Final = {"timeout": async_timeout, "client_alias": "module level aclient"}
|
||||
|
||||
# Create the client instance
|
||||
provider_id: Final = cast(Any, "litellm_module_level_client")
|
||||
provider_id: Final = "litellm_module_level_client"
|
||||
async_client: Final = get_async_httpx_client(
|
||||
llm_provider=provider_id,
|
||||
params=params,
|
||||
|
|
|
|||
|
|
@ -173,6 +173,7 @@ class _MetricKwargs(TypedDict, total=False):
|
|||
response_cost: ReadOnly[float | None]
|
||||
completion_start_time: ReadOnly[datetime | float | str | None]
|
||||
api_call_start_time: ReadOnly[datetime | float | str | None]
|
||||
end_time: ReadOnly[datetime | float | str | None]
|
||||
|
||||
|
||||
def resolve_error_type(kwargs: Mapping[str, Any]) -> str:
|
||||
|
|
@ -414,13 +415,11 @@ class GenAIMetricRecorder:
|
|||
if generation_time > 0:
|
||||
self._metrics.time_per_output_token.record(generation_time / completion_tokens, attributes=common_attrs)
|
||||
|
||||
def _record_response_duration(self, kwargs: Mapping[str, Any], end_time: datetime, common_attrs: dict) -> None:
|
||||
def _record_response_duration(self, kwargs: _MetricKwargs, end_time: datetime, common_attrs: dict) -> None:
|
||||
api_call_start_time: Final = kwargs.get("api_call_start_time")
|
||||
if api_call_start_time is None:
|
||||
return
|
||||
_end_time = kwargs.get("end_time") or end_time
|
||||
if _end_time is None:
|
||||
_end_time = datetime.now()
|
||||
api_call_start: Final = to_seconds(api_call_start_time)
|
||||
end_ts: Final = to_seconds(_end_time)
|
||||
if api_call_start is None or end_ts is None:
|
||||
|
|
|
|||
|
|
@ -3437,7 +3437,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
print_verbose=print_verbose,
|
||||
level=LogfireLevel.INFO.value,
|
||||
level=LogfireLevel.INFO,
|
||||
)
|
||||
|
||||
if callback == "lunary" and lunaryLogger is not None:
|
||||
|
|
@ -4394,7 +4394,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
response_obj=result,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
level=LogfireLevel.ERROR.value,
|
||||
level=LogfireLevel.ERROR,
|
||||
print_verbose=print_verbose,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -3182,10 +3182,8 @@ def anthropic_messages_pt(
|
|||
|
||||
|
||||
def extract_between_tags(tag: str, string: str, strip: bool = False) -> list[str]:
|
||||
ext_list = re.findall(f"<{tag}>(.+?)</{tag}>", string, re.DOTALL)
|
||||
if strip:
|
||||
ext_list = [e.strip() for e in ext_list]
|
||||
return ext_list
|
||||
matches: Final = re.finditer(f"<{tag}>(.+?)</{tag}>", string, re.DOTALL)
|
||||
return [match.group(1).strip() if strip else match.group(1) for match in matches]
|
||||
|
||||
|
||||
def contains_tag(tag: str, string: str) -> bool:
|
||||
|
|
@ -6028,7 +6026,7 @@ def _parse_tool_call_arguments(raw: object, tool_name: str | None, context: str)
|
|||
return ({},)
|
||||
normalized_raw: Final = "{}" if raw == REDACTED_BY_LITELLM else raw
|
||||
try:
|
||||
parsed: Final = parse_tool_call_arguments(normalized_raw, tool_name=tool_name, context=context)
|
||||
parsed: Final[object] = parse_tool_call_arguments(normalized_raw, tool_name=tool_name, context=context)
|
||||
except ValueError as e:
|
||||
salvaged: Final = salvage_concatenated_tool_arguments(normalized_raw)
|
||||
if salvaged:
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ def safe_json_structure(
|
|||
def _transform(key: str | None, value: str) -> str:
|
||||
return value if value_transform is None else value_transform(key, value)
|
||||
|
||||
def _serialize(obj: object, seen: set[int], depth: int, key: str | None = None) -> Any:
|
||||
def _serialize(obj: object, seen: set[int], depth: int, key: str | None = None) -> object:
|
||||
# Check for maximum depth.
|
||||
if depth > max_depth:
|
||||
return "MaxDepthExceeded"
|
||||
|
|
@ -48,32 +48,32 @@ def safe_json_structure(
|
|||
if id(obj) in seen:
|
||||
return "CircularReference Detected"
|
||||
seen.add(id(obj))
|
||||
result: dict | list | tuple | set | str
|
||||
if isinstance(obj, dict):
|
||||
result = {}
|
||||
for k, v in obj.items():
|
||||
if isinstance(k, (str)):
|
||||
clean_k = k.replace("\x00", "") if "\x00" in k else k
|
||||
result[clean_k] = _serialize(v, seen, depth + 1, clean_k)
|
||||
mapping_result: Final = {
|
||||
(clean_k := k.replace("\x00", "") if "\x00" in k else k): _serialize(v, seen, depth + 1, clean_k)
|
||||
for k, v in obj.items()
|
||||
if isinstance(k, str)
|
||||
}
|
||||
seen.remove(id(obj))
|
||||
return result
|
||||
return mapping_result
|
||||
elif isinstance(obj, list):
|
||||
result = [_serialize(item, seen, depth + 1, key) for item in obj]
|
||||
list_result: Final = [_serialize(item, seen, depth + 1, key) for item in obj]
|
||||
seen.remove(id(obj))
|
||||
return result
|
||||
return list_result
|
||||
elif isinstance(obj, tuple):
|
||||
result = tuple(_serialize(item, seen, depth + 1, key) for item in obj)
|
||||
tuple_result: Final = tuple(_serialize(item, seen, depth + 1, key) for item in obj)
|
||||
seen.remove(id(obj))
|
||||
return result
|
||||
return tuple_result
|
||||
elif isinstance(obj, set):
|
||||
result = sorted([_serialize(item, seen, depth + 1, key) for item in obj])
|
||||
sortable_values: Final[list[Any]] = [_serialize(item, seen, depth + 1, key) for item in obj]
|
||||
set_result: Final = sorted(sortable_values)
|
||||
seen.remove(id(obj))
|
||||
return result
|
||||
return set_result
|
||||
elif isinstance(obj, BaseModel):
|
||||
dumped: Final = obj.model_dump()
|
||||
result = _serialize(dumped, seen, depth + 1, key)
|
||||
model_result: Final = _serialize(dumped, seen, depth + 1, key)
|
||||
seen.remove(id(obj))
|
||||
return result
|
||||
return model_result
|
||||
else:
|
||||
# Fall back to string conversion for non-serializable objects.
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Final, Optional, Union
|
|||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm._uuid import uuid
|
||||
|
|
@ -52,6 +53,9 @@ else:
|
|||
AsyncHTTPHandler = Any
|
||||
|
||||
|
||||
_JSON_VALUE: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
||||
def __init__(self, **kwargs):
|
||||
BaseConfig.__init__(self, **kwargs)
|
||||
|
|
@ -314,7 +318,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
|||
return None
|
||||
|
||||
try:
|
||||
data: Final[object] = json.loads(json_str)
|
||||
data: Final = _JSON_VALUE.validate_python(json.loads(json_str))
|
||||
# Skip non-dict data (some lines contain JSON strings)
|
||||
return data if isinstance(data, dict) else None
|
||||
except json.JSONDecodeError:
|
||||
|
|
@ -482,7 +486,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
|||
if "application/json" in content_type:
|
||||
# Direct JSON response
|
||||
verbose_logger.debug("Parsing JSON response")
|
||||
response_json: Final[object] = raw_response.json()
|
||||
response_json: Final = _JSON_VALUE.validate_python(raw_response.json())
|
||||
verbose_logger.debug("Response JSON: %s", response_json)
|
||||
return self._parse_json_response(response_json)
|
||||
else:
|
||||
|
|
@ -566,7 +570,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
|||
continue
|
||||
|
||||
try:
|
||||
data_obj = json.loads(json_str)
|
||||
data_obj = _JSON_VALUE.validate_python(json.loads(json_str))
|
||||
if not isinstance(data_obj, dict):
|
||||
continue
|
||||
|
||||
|
|
@ -706,7 +710,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
|||
)
|
||||
try:
|
||||
body: Final = response.read()
|
||||
response_json: Final[object] = json.loads(body)
|
||||
response_json: Final = _JSON_VALUE.validate_python(json.loads(body))
|
||||
except (json.JSONDecodeError, Exception) as e:
|
||||
raise BedrockError(
|
||||
status_code=response.status_code,
|
||||
|
|
@ -788,7 +792,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
|||
continue
|
||||
|
||||
try:
|
||||
data_obj = json.loads(json_str)
|
||||
data_obj = _JSON_VALUE.validate_python(json.loads(json_str))
|
||||
if not isinstance(data_obj, dict):
|
||||
continue
|
||||
|
||||
|
|
@ -928,7 +932,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
|||
)
|
||||
try:
|
||||
body: Final = await response.aread()
|
||||
response_json: Final[object] = json.loads(body)
|
||||
response_json: Final = _JSON_VALUE.validate_python(json.loads(body))
|
||||
except (json.JSONDecodeError, Exception) as e:
|
||||
raise BedrockError(
|
||||
status_code=response.status_code,
|
||||
|
|
|
|||
|
|
@ -12,7 +12,8 @@ from itertools import chain
|
|||
from typing import TYPE_CHECKING, Final, Literal, cast, overload
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -129,6 +130,16 @@ from ..common_utils import (
|
|||
normalize_bedrock_opus_output_config_effort,
|
||||
)
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _JsonContentToolCall(TypedDict):
|
||||
type: ReadOnly[NotRequired[object]]
|
||||
name: ReadOnly[NotRequired[str]]
|
||||
arguments: ReadOnly[NotRequired[str | dict[str, object] | None]]
|
||||
|
||||
|
||||
_JSON_CONTENT_TOOL_CALL: Final = TypeAdapter(_JsonContentToolCall)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
|
@ -2453,7 +2464,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
if message.content is not None:
|
||||
try:
|
||||
tool_call_names: Final = self.get_tool_call_names(tools)
|
||||
json_content: Final = json.loads(message.content)
|
||||
json_content: Final = _JSON_CONTENT_TOOL_CALL.validate_python(json.loads(message.content))
|
||||
if json_content.get("type") == "function" and json_content.get("name") in tool_call_names:
|
||||
tool_calls: Final = [ChatCompletionMessageToolCall(function=Function(**json_content))]
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ import time
|
|||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter, with_config
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -34,6 +36,15 @@ else:
|
|||
LiteLLMLoggingObj = Any
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _PegasusResponse(TypedDict):
|
||||
message: ReadOnly[NotRequired[str]]
|
||||
finishReason: ReadOnly[NotRequired[str]]
|
||||
|
||||
|
||||
_PEGASUS_RESPONSE: Final = TypeAdapter(_PegasusResponse)
|
||||
|
||||
|
||||
class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
|
||||
"""
|
||||
Handles transforming OpenAI-style requests into Bedrock InvokeModel requests for
|
||||
|
|
@ -207,7 +218,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
|
|||
ModelResponse with choices[0].message.content and finish_reason
|
||||
"""
|
||||
try:
|
||||
completion_response: Final = raw_response.json()
|
||||
completion_response: Final = _PEGASUS_RESPONSE.validate_python(raw_response.json())
|
||||
except Exception as e:
|
||||
raise BedrockError(
|
||||
message=f"Error parsing response: {raw_response.text}, error: {e}",
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
import copy
|
||||
import json
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, cast, get_args
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -48,6 +49,27 @@ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
|||
|
||||
_GUARDRAIL_CONFIG_VALIDATOR: Final["TypeAdapter[GuardrailConfigBlock]"] = TypeAdapter(GuardrailConfigBlock)
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _CohereGeneration(TypedDict):
|
||||
text: ReadOnly[str | None]
|
||||
finish_reason: ReadOnly[str]
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _CohereInvokeResponse(TypedDict):
|
||||
text: NotRequired[ReadOnly[str | None]]
|
||||
generations: NotRequired[ReadOnly[Sequence[_CohereGeneration]]]
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _LlamaInvokeResponse(TypedDict):
|
||||
generation: ReadOnly[str | None]
|
||||
|
||||
|
||||
_COHERE_INVOKE_RESPONSE: Final = TypeAdapter(_CohereInvokeResponse)
|
||||
_LLAMA_INVOKE_RESPONSE: Final = TypeAdapter(_LlamaInvokeResponse)
|
||||
|
||||
_GUARDRAIL_CONFIG_EXPECTED_FORMAT: Final = (
|
||||
"{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}"
|
||||
)
|
||||
|
|
@ -310,12 +332,13 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
|
|||
outputText: str | None = None
|
||||
try:
|
||||
if provider == "cohere":
|
||||
if "text" in completion_response:
|
||||
outputText = completion_response["text"]
|
||||
elif "generations" in completion_response:
|
||||
outputText = completion_response["generations"][0]["text"]
|
||||
cohere_response: Final = _COHERE_INVOKE_RESPONSE.validate_python(completion_response)
|
||||
if "text" in cohere_response:
|
||||
outputText = cohere_response["text"]
|
||||
elif "generations" in cohere_response:
|
||||
outputText = cohere_response["generations"][0]["text"]
|
||||
model_response.choices[0].finish_reason = map_finish_reason(
|
||||
completion_response["generations"][0]["finish_reason"]
|
||||
cohere_response["generations"][0]["finish_reason"]
|
||||
)
|
||||
elif provider == "anthropic":
|
||||
return litellm.AmazonAnthropicClaudeConfig().transform_response(
|
||||
|
|
@ -361,7 +384,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
|
|||
elif provider == "ai21":
|
||||
outputText = completion_response.get("completions")[0].get("data").get("text")
|
||||
elif provider == "meta" or provider == "llama" or provider == "deepseek_r1":
|
||||
outputText = completion_response["generation"]
|
||||
outputText = _LLAMA_INVOKE_RESPONSE.validate_python(completion_response)["generation"]
|
||||
elif provider == "mistral":
|
||||
outputText = litellm.AmazonMistralConfig.get_outputText(completion_response, model_response)
|
||||
else: # amazon titan
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from itertools import accumulate, chain
|
|||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Optional, Protocol, TypeAlias, TypeVar
|
||||
|
||||
from pydantic import InstanceOf, TypeAdapter, ValidationError
|
||||
from pydantic import ConfigDict, InstanceOf, TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
|
@ -768,6 +768,7 @@ class InvokeStreamFrame:
|
|||
|
||||
|
||||
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
|
||||
_STRICT_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(strict=True))
|
||||
_Frame: Final = TypeVar("_Frame", ConverseStreamFrame, InvokeStreamFrame)
|
||||
|
||||
|
||||
|
|
@ -1217,7 +1218,7 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
|
|||
texts: Sequence[tuple[_StreamGroupKey, str]] = ()
|
||||
if event_type == "contentBlockDelta":
|
||||
try:
|
||||
payload_dict: dict[str, object] = json.loads(payload_bytes)
|
||||
payload_dict = _STRICT_JSON_OBJECT.validate_python(json.loads(payload_bytes))
|
||||
texts = _delta_frame_texts(payload_dict)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.debug(
|
||||
|
|
@ -1255,7 +1256,9 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
|
|||
headers_bytes = frame_raw[12 : 12 + orig_hdrs_len]
|
||||
|
||||
try:
|
||||
payload_dict = json.loads(frame_raw[12 + orig_hdrs_len : orig_total - 4])
|
||||
payload_dict = _STRICT_JSON_OBJECT.validate_python(
|
||||
json.loads(frame_raw[12 + orig_hdrs_len : orig_total - 4])
|
||||
)
|
||||
for local_idx, (_, container, key) in enumerate(
|
||||
_collect_stream_delta_text_holders(payload_dict.get("delta"))
|
||||
):
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@ from litellm.types.realtime import (
|
|||
RealtimeResponseTypedDict,
|
||||
)
|
||||
|
||||
_JSON_OBJECT_MESSAGE: Final = TypeAdapter(dict[str, JsonValue], config=ConfigDict(strict=True))
|
||||
_JSON_VALUE: Final = TypeAdapter(object)
|
||||
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
|
@ -856,7 +857,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
|
|||
List of Bedrock format messages (JSON strings)
|
||||
"""
|
||||
try:
|
||||
json_message: Final[dict[str, JsonValue]] = json.loads(message)
|
||||
json_message: Final = _JSON_OBJECT_MESSAGE.validate_python(json.loads(message))
|
||||
except json.JSONDecodeError:
|
||||
verbose_logger.warning("Invalid JSON message: %s", message[:200])
|
||||
return []
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ class _BflPolledResult(TypedDict):
|
|||
|
||||
|
||||
_BFL_POLLED_RESULT_ADAPTER: Final = TypeAdapter(_BflPolledResult)
|
||||
_JSON_VALUE: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
_BFL_REQUEST_PARAMS: Final = (
|
||||
|
|
@ -316,7 +317,7 @@ class BlackForestLabsImageEditConfig(BaseImageEditConfig):
|
|||
The response contains: {"status": "Ready", "result": {"sample": "https://..."}}
|
||||
"""
|
||||
try:
|
||||
raw_response_data: Final[object] = raw_response.json()
|
||||
raw_response_data: Final = _JSON_VALUE.validate_python(raw_response.json())
|
||||
except Exception as e:
|
||||
raise BlackForestLabsError(
|
||||
status_code=raw_response.status_code,
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from __future__ import annotations
|
|||
|
||||
import itertools
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
|
@ -231,12 +231,16 @@ def _system_events(event: Mapping[str, object], state: ClaudeCodeStreamState) ->
|
|||
return event_list()
|
||||
|
||||
|
||||
def _record_result(event: Mapping[str, Any], state: ClaudeCodeStreamState) -> Sequence[Event]:
|
||||
def _record_result(event: Mapping[str, object], state: ClaudeCodeStreamState) -> Sequence[Event]:
|
||||
state.result_seen = True
|
||||
state.is_error = bool(event.get("is_error", False))
|
||||
result = event.get("result")
|
||||
state.result_text = result if isinstance(result, str) else None
|
||||
state.errors = [str(e) for e in event.get("errors") or ()] # mutable-ok: mirrors the JSON errors array
|
||||
errors: Final = event.get("errors") or ()
|
||||
if not isinstance(errors, Iterable):
|
||||
raise TypeError(f"'{type(errors).__name__}' object is not iterable")
|
||||
error_items: Final[Iterable[object]] = errors
|
||||
state.errors = [str(e) for e in error_items] # mutable-ok: mirrors the JSON errors array
|
||||
state.structured_output = event.get("structured_output")
|
||||
if event.get("session_id"):
|
||||
state.session_id = str(event["session_id"])
|
||||
|
|
@ -301,7 +305,7 @@ class ClaudeCodeHarnessConfig(BaseCLIHarnessConfig):
|
|||
options: ClaudeCodeOptions = self.get_options(ctx)
|
||||
model = ctx.model
|
||||
# Background calls (titles, summaries) use the same model group, like OpenCode.
|
||||
model_env: Final = (
|
||||
model_env: Final[Mapping[str, str]] = (
|
||||
MappingProxyType({"ANTHROPIC_MODEL": model, "ANTHROPIC_SMALL_FAST_MODEL": model})
|
||||
if model
|
||||
else MappingProxyType({})
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@ Docs - https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter, with_config
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route
|
||||
|
|
@ -28,6 +30,23 @@ from ..common_utils import DashScopeError
|
|||
DEFAULT_API_BASE: Final = "https://dashscope.aliyuncs.com/compatible-mode/v1"
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _DashScopeUsage(TypedDict, total=False):
|
||||
prompt_tokens: NotRequired[ReadOnly[int]]
|
||||
total_tokens: NotRequired[ReadOnly[int]]
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _DashScopeEmbeddingResponse(TypedDict, total=False):
|
||||
data: NotRequired[ReadOnly[list[object]]]
|
||||
model: NotRequired[ReadOnly[str | None]]
|
||||
usage: NotRequired[ReadOnly[_DashScopeUsage | None]]
|
||||
id: NotRequired[ReadOnly[object]]
|
||||
|
||||
|
||||
_DASHSCOPE_EMBEDDING_RESPONSE: Final = TypeAdapter(_DashScopeEmbeddingResponse)
|
||||
|
||||
|
||||
class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
|
||||
"""
|
||||
Reference: https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api
|
||||
|
|
@ -152,11 +171,12 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
|
|||
message=message,
|
||||
)
|
||||
|
||||
parsed_response: Final = _DASHSCOPE_EMBEDDING_RESPONSE.validate_python(response_json)
|
||||
model_response.object = "list"
|
||||
model_response.data = response_json.get("data", [])
|
||||
model_response.model = response_json.get("model", model)
|
||||
model_response.data = parsed_response.get("data", [])
|
||||
model_response.model = parsed_response.get("model", model)
|
||||
|
||||
usage: Final = response_json.get("usage") or {}
|
||||
usage: Final = parsed_response.get("usage") or {}
|
||||
prompt_tokens: Final = usage.get("prompt_tokens", 0)
|
||||
total_tokens: Final = usage.get("total_tokens", prompt_tokens)
|
||||
setattr(
|
||||
|
|
@ -169,8 +189,8 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
|
|||
),
|
||||
)
|
||||
|
||||
if "id" in response_json:
|
||||
setattr(model_response, "id", response_json["id"])
|
||||
if "id" in parsed_response:
|
||||
setattr(model_response, "id", parsed_response["id"])
|
||||
|
||||
return model_response
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm import verbose_logger
|
||||
from litellm.types.llms.openai import (
|
||||
|
|
@ -23,6 +25,9 @@ def _usage_block(usage: Usage) -> ChatCompletionUsageBlock:
|
|||
)
|
||||
|
||||
|
||||
_JSON_CHUNK: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
|
||||
|
||||
|
||||
class ModelResponseIterator:
|
||||
def __init__(self, streaming_response, sync_stream: bool):
|
||||
self.streaming_response = streaming_response
|
||||
|
|
@ -99,7 +104,7 @@ class ModelResponseIterator:
|
|||
self.response_iterator = self.streaming_response
|
||||
chunk: Final = self._next_sse_data()
|
||||
try:
|
||||
json_chunk: Final = json.loads(chunk)
|
||||
json_chunk: Final = _JSON_CHUNK.validate_python(json.loads(chunk))
|
||||
return self.chunk_parser(chunk=json_chunk)
|
||||
except ValueError as e:
|
||||
verbose_logger.debug(
|
||||
|
|
@ -126,7 +131,7 @@ class ModelResponseIterator:
|
|||
if chunk == "[DONE]":
|
||||
raise StopAsyncIteration
|
||||
try:
|
||||
json_chunk: Final = json.loads(chunk)
|
||||
json_chunk: Final = _JSON_CHUNK.validate_python(json.loads(chunk))
|
||||
return self.chunk_parser(chunk=json_chunk)
|
||||
except ValueError as e:
|
||||
verbose_logger.debug(
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ Docs: https://www.edenai.co/docs/v3/llms/embeddings
|
|||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import set_response_cost_in_hidden_params
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
|
@ -23,6 +24,8 @@ if TYPE_CHECKING:
|
|||
|
||||
_SUPPORTED_PARAMS: Final = ("dimensions", "encoding_format", "user")
|
||||
|
||||
_EMBEDDING_BODY: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
|
||||
|
||||
|
||||
class EdenAIEmbeddingConfig(BaseEmbeddingConfig):
|
||||
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: inherited contract
|
||||
|
|
@ -80,7 +83,7 @@ class EdenAIEmbeddingConfig(BaseEmbeddingConfig):
|
|||
optional_params: dict[str, object], # mutable-ok: inherited contract
|
||||
litellm_params: dict[str, object], # mutable-ok: inherited contract
|
||||
) -> EmbeddingResponse:
|
||||
body: Final = raw_response.json()
|
||||
body: Final = _EMBEDDING_BODY.validate_python(raw_response.json())
|
||||
logging_obj.post_call(original_response=body)
|
||||
response: Final[EmbeddingResponse] = convert_to_model_response_object(
|
||||
response_object=body, model_response_object=model_response, response_type="embedding"
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ from types import MappingProxyType
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter, with_config
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.caching.caching import InMemoryCache
|
||||
|
|
@ -37,6 +39,17 @@ _token_cache: Final = InMemoryCache()
|
|||
_NO_LITELLM_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _GigaChatTokenResponse(TypedDict):
|
||||
tok: NotRequired[ReadOnly[str | None]]
|
||||
access_token: NotRequired[ReadOnly[str | None]]
|
||||
exp: NotRequired[ReadOnly[int | float | str | None]]
|
||||
expires_at: NotRequired[ReadOnly[int | float | str | None]]
|
||||
|
||||
|
||||
_TOKEN_RESPONSE: Final = TypeAdapter(_GigaChatTokenResponse)
|
||||
|
||||
|
||||
class GigaChatAuthError(BaseLLMException):
|
||||
"""GigaChat authentication error."""
|
||||
|
||||
|
|
@ -230,7 +243,7 @@ async def _request_token_async(
|
|||
|
||||
def _parse_token_response(response: httpx.Response) -> tuple[str, int]:
|
||||
"""Parse OAuth token response."""
|
||||
data: Final = response.json()
|
||||
data: Final = _TOKEN_RESPONSE.validate_python(response.json())
|
||||
|
||||
# GigaChat returns either 'tok'/'exp' or 'access_token'/'expires_at'
|
||||
access_token: Final = data.get("tok") or data.get("access_token")
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from collections.abc import Mapping, Sequence
|
|||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
|
||||
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
|
||||
from litellm.llms.gigachat.authenticator import get_access_token
|
||||
|
|
@ -20,6 +21,8 @@ if TYPE_CHECKING:
|
|||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.types.utils import CostResponseTypes
|
||||
|
||||
_STREAM_CHUNK: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
|
||||
|
||||
|
||||
class GigaChatPassthroughConfig(BasePassthroughConfig):
|
||||
def is_streaming_request(self, endpoint: str, request_data: Mapping[str, object]) -> bool:
|
||||
|
|
@ -161,7 +164,7 @@ class GigaChatPassthroughConfig(BasePassthroughConfig):
|
|||
continue
|
||||
chunk = chunk.removeprefix("data: ")
|
||||
try:
|
||||
message = json.loads(chunk)
|
||||
message = _STREAM_CHUNK.validate_python(json.loads(chunk))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ This requires websockets, and is currently only supported on LiteLLM Proxy.
|
|||
"""
|
||||
|
||||
import ssl
|
||||
from typing import Any, Final, cast
|
||||
from typing import TYPE_CHECKING, Final, cast
|
||||
|
||||
from litellm._logging import redact_string, verbose_logger
|
||||
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
|
||||
|
|
@ -23,6 +23,9 @@ from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context
|
|||
from ..common_utils import is_openai_backed_api_base
|
||||
from ..openai import OpenAIChatCompletion
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fastapi import WebSocket
|
||||
|
||||
|
||||
class OpenAIRealtime(OpenAIChatCompletion):
|
||||
"""
|
||||
|
|
@ -131,7 +134,7 @@ class OpenAIRealtime(OpenAIChatCompletion):
|
|||
async def async_realtime(
|
||||
self,
|
||||
model: str,
|
||||
websocket: Any,
|
||||
websocket: "WebSocket",
|
||||
logging_obj: LiteLLMLogging,
|
||||
api_base: str | None = None,
|
||||
api_key: str | None = None,
|
||||
|
|
|
|||
|
|
@ -12,12 +12,14 @@ This module decodes them into float arrays for OpenAI-compatible responses.
|
|||
"""
|
||||
|
||||
import base64
|
||||
import builtins
|
||||
import struct
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final, Literal
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
from pydantic import ConfigDict, TypeAdapter, with_config
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
|
|
@ -31,6 +33,17 @@ _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_i
|
|||
_INT: Final = TypeAdapter(int)
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _PerplexityEmbeddingResponse(TypedDict, total=False):
|
||||
model: ReadOnly[str | None]
|
||||
object: ReadOnly[Literal["list"]]
|
||||
data: ReadOnly[Sequence[builtins.object] | Mapping[str, builtins.object] | Literal[""]]
|
||||
usage: ReadOnly[builtins.object]
|
||||
|
||||
|
||||
_EMBEDDING_RESPONSE_ADAPTER: Final = TypeAdapter(_PerplexityEmbeddingResponse)
|
||||
|
||||
|
||||
class PerplexityEmbeddingError(BaseLLMException):
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -153,14 +166,18 @@ class PerplexityEmbeddingConfig(BaseEmbeddingConfig):
|
|||
litellm_params: dict = {},
|
||||
) -> EmbeddingResponse:
|
||||
try:
|
||||
raw_response_json: Final = raw_response.json()
|
||||
raw_response_json: Final[object] = raw_response.json()
|
||||
except Exception:
|
||||
raise PerplexityEmbeddingError(message=raw_response.text, status_code=raw_response.status_code)
|
||||
|
||||
model_response.model = raw_response_json.get("model", model)
|
||||
model_response.object = raw_response_json.get("object", "list")
|
||||
response_data: Final[_PerplexityEmbeddingResponse] = _EMBEDDING_RESPONSE_ADAPTER.validate_python(
|
||||
raw_response_json
|
||||
)
|
||||
|
||||
raw_data: Final = raw_response_json.get("data", [])
|
||||
model_response.model = response_data.get("model", model)
|
||||
model_response.object = response_data.get("object", "list")
|
||||
|
||||
raw_data: Final = response_data.get("data", [])
|
||||
decoded_data: Final[list[dict[str, object]]] = []
|
||||
for item in raw_data:
|
||||
decoded_item = dict(_JSON_OBJECT.validate_python(item))
|
||||
|
|
@ -168,7 +185,7 @@ class PerplexityEmbeddingConfig(BaseEmbeddingConfig):
|
|||
decoded_data.append(decoded_item)
|
||||
model_response.data = decoded_data
|
||||
|
||||
usage_data: Final = _JSON_OBJECT.validate_python(raw_response_json.get("usage", {}))
|
||||
usage_data: Final = _JSON_OBJECT.validate_python(response_data.get("usage", {}))
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=_INT.validate_python(
|
||||
usage_data.get("prompt_tokens", 0) or usage_data.get("total_tokens", 0) or 0
|
||||
|
|
|
|||
|
|
@ -6,11 +6,12 @@ Maps OpenAI TTS spec to RunwayML Text-to-Speech API
|
|||
|
||||
import asyncio
|
||||
import time
|
||||
from collections.abc import Coroutine, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, TypedDict, Union
|
||||
from collections.abc import Coroutine
|
||||
from typing import TYPE_CHECKING, Any, Final, Union
|
||||
|
||||
import httpx
|
||||
from typing_extensions import ReadOnly
|
||||
from pydantic import ConfigDict, TypeAdapter, with_config
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -33,12 +34,17 @@ else:
|
|||
HttpxBinaryResponseContent = Any
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _RunwayTtsTaskResponse(TypedDict, total=False):
|
||||
id: ReadOnly[str]
|
||||
status: ReadOnly[str]
|
||||
output: ReadOnly[Sequence[object]]
|
||||
failure: ReadOnly[str]
|
||||
failureCode: ReadOnly[str]
|
||||
output: ReadOnly[object]
|
||||
failure: ReadOnly[object]
|
||||
failureCode: ReadOnly[object]
|
||||
|
||||
|
||||
_RUNWAY_TTS_TASK_RESPONSE_ADAPTER: Final = TypeAdapter(_RunwayTtsTaskResponse)
|
||||
_RUNWAY_TTS_OUTPUT_ADAPTER: Final = TypeAdapter(list[object], config=ConfigDict(strict=True))
|
||||
|
||||
|
||||
class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
|
||||
|
|
@ -321,7 +327,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
|
|||
response = client.get(url=task_url, headers=headers)
|
||||
response.raise_for_status()
|
||||
|
||||
response_data: _RunwayTtsTaskResponse = response.json()
|
||||
response_data = _RUNWAY_TTS_TASK_RESPONSE_ADAPTER.validate_python(response.json())
|
||||
|
||||
# Check task status
|
||||
status = self._check_task_status(response_data=response_data)
|
||||
|
|
@ -368,7 +374,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
|
|||
response = await client.get(url=task_url, headers=headers)
|
||||
response.raise_for_status()
|
||||
|
||||
response_data: _RunwayTtsTaskResponse = response.json()
|
||||
response_data = _RUNWAY_TTS_TASK_RESPONSE_ADAPTER.validate_python(response.json())
|
||||
|
||||
# Check task status
|
||||
status = self._check_task_status(response_data=response_data)
|
||||
|
|
@ -459,7 +465,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
|
|||
from litellm.types.llms.openai import HttpxBinaryResponseContent
|
||||
|
||||
try:
|
||||
response_data: Final[_RunwayTtsTaskResponse] = raw_response.json()
|
||||
response_data: Final = _RUNWAY_TTS_TASK_RESPONSE_ADAPTER.validate_python(raw_response.json())
|
||||
except Exception as e:
|
||||
raise self.get_error_class(
|
||||
error_message=f"Error parsing RunwayML TTS response: {e}",
|
||||
|
|
@ -489,16 +495,20 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
|
|||
)
|
||||
|
||||
# Get the completed task data
|
||||
task_data: Final[_RunwayTtsTaskResponse] = polled_response.json()
|
||||
task_data: Final = _RUNWAY_TTS_TASK_RESPONSE_ADAPTER.validate_python(polled_response.json())
|
||||
|
||||
verbose_logger.debug("RunwayML TTS polling complete, downloading audio")
|
||||
|
||||
# Get audio URL from output
|
||||
output: Final = task_data.get("output", [])
|
||||
if not output or not isinstance(output, list) or len(output) == 0:
|
||||
if not output or not isinstance(output, list):
|
||||
raise ValueError("RunwayML TTS response missing audio URL in output")
|
||||
|
||||
audio_url: Final = output[0]
|
||||
output_items: Final = _RUNWAY_TTS_OUTPUT_ADAPTER.validate_python(output)
|
||||
if len(output_items) == 0:
|
||||
raise ValueError("RunwayML TTS response missing audio URL in output")
|
||||
|
||||
audio_url: Final = output_items[0]
|
||||
if not isinstance(audio_url, str):
|
||||
raise ValueError(f"RunwayML TTS audio URL is not a string: {audio_url}")
|
||||
|
||||
|
|
@ -528,7 +538,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
|
|||
from litellm.types.llms.openai import HttpxBinaryResponseContent
|
||||
|
||||
try:
|
||||
response_data: Final[_RunwayTtsTaskResponse] = raw_response.json()
|
||||
response_data: Final = _RUNWAY_TTS_TASK_RESPONSE_ADAPTER.validate_python(raw_response.json())
|
||||
except Exception as e:
|
||||
raise self.get_error_class(
|
||||
error_message=f"Error parsing RunwayML TTS response: {e}",
|
||||
|
|
@ -558,16 +568,20 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
|
|||
)
|
||||
|
||||
# Get the completed task data
|
||||
task_data: Final[_RunwayTtsTaskResponse] = polled_response.json()
|
||||
task_data: Final = _RUNWAY_TTS_TASK_RESPONSE_ADAPTER.validate_python(polled_response.json())
|
||||
|
||||
verbose_logger.debug("RunwayML TTS polling complete (async), downloading audio")
|
||||
|
||||
# Get audio URL from output
|
||||
output: Final = task_data.get("output", [])
|
||||
if not output or not isinstance(output, list) or len(output) == 0:
|
||||
if not output or not isinstance(output, list):
|
||||
raise ValueError("RunwayML TTS response missing audio URL in output")
|
||||
|
||||
audio_url: Final = output[0]
|
||||
output_items: Final = _RUNWAY_TTS_OUTPUT_ADAPTER.validate_python(output)
|
||||
if len(output_items) == 0:
|
||||
raise ValueError("RunwayML TTS response missing audio URL in output")
|
||||
|
||||
audio_url: Final = output_items[0]
|
||||
if not isinstance(audio_url, str):
|
||||
raise ValueError(f"RunwayML TTS audio URL is not a string: {audio_url}")
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
from collections.abc import Sequence
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter, with_config
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
from litellm.llms.base_llm.chat.transformation import AllMessageValues, BaseLLMException
|
||||
from litellm.llms.base_llm.embedding.transformation import (
|
||||
|
|
@ -14,6 +17,21 @@ from litellm.utils import token_counter
|
|||
from ..common_utils import TritonError
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _TritonEmbeddingOutput(TypedDict):
|
||||
shape: ReadOnly[Sequence[int]]
|
||||
data: ReadOnly[Sequence[int | float]]
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _TritonEmbeddingResponse(TypedDict):
|
||||
outputs: ReadOnly[Sequence[_TritonEmbeddingOutput]]
|
||||
model_name: ReadOnly[NotRequired[str | None]]
|
||||
|
||||
|
||||
_TRITON_EMBEDDING_RESPONSE: Final = TypeAdapter(_TritonEmbeddingResponse)
|
||||
|
||||
|
||||
class TritonEmbeddingConfig(BaseEmbeddingConfig):
|
||||
"""
|
||||
Transformations for triton /embeddings endpoint (This is a trtllm model)
|
||||
|
|
@ -83,9 +101,10 @@ class TritonEmbeddingConfig(BaseEmbeddingConfig):
|
|||
except Exception:
|
||||
raise TritonError(message=raw_response.text, status_code=raw_response.status_code)
|
||||
|
||||
parsed_response: Final = _TRITON_EMBEDDING_RESPONSE.validate_python(raw_response_json)
|
||||
_embedding_output: Final = []
|
||||
|
||||
_outputs: Final = raw_response_json["outputs"]
|
||||
_outputs: Final = parsed_response["outputs"]
|
||||
for output in _outputs:
|
||||
_shape = output["shape"]
|
||||
_data = output["data"]
|
||||
|
|
@ -100,7 +119,7 @@ class TritonEmbeddingConfig(BaseEmbeddingConfig):
|
|||
}
|
||||
)
|
||||
|
||||
model_response.model = raw_response_json.get("model_name", "None")
|
||||
model_response.model = parsed_response.get("model_name", "None")
|
||||
model_response.data = _embedding_output
|
||||
model_response.usage = self._build_embedding_usage(model=model, request_data=request_data)
|
||||
return model_response
|
||||
|
|
@ -134,7 +153,7 @@ class TritonEmbeddingConfig(BaseEmbeddingConfig):
|
|||
return TritonError(message=error_message, status_code=status_code, headers=headers)
|
||||
|
||||
@staticmethod
|
||||
def split_embedding_by_shape(data: list[float], shape: list[int]) -> list[list[float]]:
|
||||
def split_embedding_by_shape(data: Sequence[int | float], shape: Sequence[int]) -> Sequence[Sequence[int | float]]:
|
||||
if len(shape) != 2:
|
||||
raise ValueError("Shape must be of length 2.")
|
||||
embedding_size: Final = shape[1]
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ if TYPE_CHECKING:
|
|||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
||||
_PREDICTIONS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True))
|
||||
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
|
||||
|
||||
|
||||
class VertexImageGeneration(VertexLLM):
|
||||
|
|
@ -168,7 +169,7 @@ class VertexImageGeneration(VertexLLM):
|
|||
if response.status_code != 200:
|
||||
raise Exception(f"Error: {response.status_code} {response.text}")
|
||||
|
||||
json_response: Final = response.json()
|
||||
json_response: Final = _JSON_OBJECT.validate_python(response.json())
|
||||
return self.process_image_generation_response(json_response, model_response, model)
|
||||
|
||||
async def aimage_generation(
|
||||
|
|
@ -272,7 +273,7 @@ class VertexImageGeneration(VertexLLM):
|
|||
if response.status_code != 200:
|
||||
raise Exception(f"Error: {response.status_code} {response.text}")
|
||||
|
||||
json_response: Final = response.json()
|
||||
json_response: Final = _JSON_OBJECT.validate_python(response.json())
|
||||
return self.process_image_generation_response(json_response, model_response, model)
|
||||
|
||||
def is_image_generation_response(self, json_response: dict[str, Any]) -> bool:
|
||||
|
|
|
|||
|
|
@ -9,10 +9,11 @@ The actual message transformation reuses OpenAIGPTConfig since Gemma uses OpenAI
|
|||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
from typing import TYPE_CHECKING, Final, cast
|
||||
|
||||
import httpx
|
||||
from pydantic import ValidationError
|
||||
from pydantic import ConfigDict, TypeAdapter, ValidationError
|
||||
from typing_extensions import NotRequired, ReadOnly, TypeAliasType, TypedDict
|
||||
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
|
|
@ -32,6 +33,17 @@ if TYPE_CHECKING:
|
|||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
class _PredictionsEnvelope(TypedDict):
|
||||
predictions: NotRequired[ReadOnly[dict[str, object]]]
|
||||
|
||||
|
||||
_PredictionsEnvelopeType = TypeAliasType("_PredictionsEnvelopeType", _PredictionsEnvelope)
|
||||
_PREDICTIONS_ENVELOPE: Final[TypeAdapter[_PredictionsEnvelope]] = TypeAdapter(
|
||||
_PredictionsEnvelopeType,
|
||||
config=ConfigDict(strict=True, extra="allow"),
|
||||
)
|
||||
|
||||
|
||||
def parse_vertex_gemma_container_error(predictions: object) -> VertexGemmaContainerError | None:
|
||||
try:
|
||||
return VertexGemmaContainerError.model_validate(predictions)
|
||||
|
|
@ -142,7 +154,7 @@ class VertexGemmaConfig(OpenAIGPTConfig):
|
|||
|
||||
def _unwrap_predictions_response(
|
||||
self,
|
||||
response_json: dict[str, Any],
|
||||
response_json: _PredictionsEnvelope,
|
||||
) -> dict[str, object]:
|
||||
"""
|
||||
Unwrap the Vertex Gemma predictions format to OpenAI format.
|
||||
|
|
@ -354,7 +366,7 @@ class VertexGemmaConfig(OpenAIGPTConfig):
|
|||
message=f"Request failed: {response.text}",
|
||||
)
|
||||
|
||||
response_json: Final = response.json()
|
||||
response_json: Final = _PREDICTIONS_ENVELOPE.validate_python(response.json())
|
||||
|
||||
# Unwrap predictions to get OpenAI-compatible response
|
||||
openai_response: Final = self._unwrap_predictions_response(response_json)
|
||||
|
|
@ -449,7 +461,7 @@ class VertexGemmaConfig(OpenAIGPTConfig):
|
|||
message=f"Request failed: {response.text}",
|
||||
)
|
||||
|
||||
response_json: Final = response.json()
|
||||
response_json: Final = _PREDICTIONS_ENVELOPE.validate_python(response.json())
|
||||
|
||||
# Unwrap predictions to get OpenAI-compatible response
|
||||
openai_response: Final = self._unwrap_predictions_response(response_json)
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ from types import UnionType
|
|||
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, Union, get_args, get_origin
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
from pydantic import fields as pyd_fields
|
||||
|
||||
import litellm
|
||||
|
|
@ -37,6 +38,8 @@ if TYPE_CHECKING:
|
|||
else:
|
||||
LiteLLMLoggingObj = Any
|
||||
|
||||
_RESPONSE_BODY: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
|
||||
|
||||
|
||||
class _EventModelClass(Protocol):
|
||||
@property
|
||||
|
|
@ -415,7 +418,7 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
|
||||
@staticmethod
|
||||
def _parsed_response_body(raw_response: httpx.Response) -> dict[str, object]:
|
||||
return raw_response.json()
|
||||
return _RESPONSE_BODY.validate_python(raw_response.json())
|
||||
|
||||
@staticmethod
|
||||
def _annotation_origin(annotation: object) -> object:
|
||||
|
|
|
|||
|
|
@ -300,12 +300,13 @@ def decode_state_hash(encrypted_state: str) -> dict:
|
|||
if decrypted_json is None:
|
||||
raise ValueError("Failed to decrypt state parameter")
|
||||
|
||||
state_data: Final = json.loads(decrypted_json)
|
||||
state_data: Final = _DECODED_STATE.validate_python(json.loads(decrypted_json))
|
||||
return state_data
|
||||
|
||||
|
||||
_MANAGED_AUTH_CODE_PREFIX: Final = "llm_scope_"
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
_DECODED_STATE: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
|
||||
|
||||
|
||||
class _ManagedAuthorizationCode(LiteLLMBaseModel):
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from collections.abc import Mapping
|
|||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._internal_context import with_service_target
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -42,6 +43,9 @@ if TYPE_CHECKING:
|
|||
from litellm.types.mcp_server.mcp_server_manager import MCPServer
|
||||
|
||||
|
||||
_JSON_VALUE: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class MCPOAuth2TokenCache(InMemoryCache):
|
||||
"""
|
||||
In-memory cache for OAuth2 client_credentials tokens, keyed by the identity of the token
|
||||
|
|
@ -160,7 +164,7 @@ class MCPOAuth2TokenCache(InMemoryCache):
|
|||
f"failed with status {exc.response.status_code}"
|
||||
) from exc
|
||||
|
||||
body: Final = response.json()
|
||||
body: Final = _JSON_VALUE.validate_python(response.json())
|
||||
|
||||
if not isinstance(body, dict):
|
||||
raise ValueError(
|
||||
|
|
|
|||
|
|
@ -18,7 +18,8 @@ from typing import Final, Literal, Protocol, TypeAlias
|
|||
import jwt
|
||||
from fastapi import HTTPException
|
||||
from jwt.types import Options
|
||||
from typing_extensions import assert_never
|
||||
from pydantic import ConfigDict, TypeAdapter, with_config
|
||||
from typing_extensions import ReadOnly, TypedDict, assert_never
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
|
|
@ -38,6 +39,20 @@ _ALLOWED_ID_TOKEN_ALGORITHMS: Final = (
|
|||
"PS512",
|
||||
)
|
||||
_JWKS_CACHE_TTL_SECONDS: Final = 3600
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _JwksDocument(TypedDict, total=False):
|
||||
keys: ReadOnly[Sequence[Mapping[str, object]]]
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _OidcDiscoveryDocument(TypedDict, total=False):
|
||||
jwks_uri: ReadOnly[object]
|
||||
|
||||
|
||||
_JWKS_DOCUMENT: Final = TypeAdapter(_JwksDocument)
|
||||
_OIDC_DISCOVERY_DOCUMENT: Final = TypeAdapter(_OidcDiscoveryDocument)
|
||||
_jwks_cache: Final = InMemoryCache(default_ttl=_JWKS_CACHE_TTL_SECONDS)
|
||||
|
||||
JwksFetcher: TypeAlias = Callable[
|
||||
|
|
@ -103,8 +118,8 @@ async def _fetch_issuer_jwks(binding: MCPOAuthIdentityBinding) -> Sequence[Mappi
|
|||
client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.Oauth2Check)
|
||||
response: Final = await client.get(jwks_url)
|
||||
response.raise_for_status()
|
||||
document: Final = response.json()
|
||||
keys: Final = document.get("keys") if isinstance(document, dict) else None
|
||||
document: Final = _JWKS_DOCUMENT.validate_python(response.json())
|
||||
keys: Final = document.get("keys")
|
||||
if not isinstance(keys, list):
|
||||
raise TypeError(f"JWKS document at {jwks_url} has no 'keys' array")
|
||||
await _jwks_cache.async_set_cache(jwks_url, keys, ttl=_JWKS_CACHE_TTL_SECONDS)
|
||||
|
|
@ -116,8 +131,8 @@ async def _discover_jwks_url(issuer: str) -> str:
|
|||
client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.Oauth2Check)
|
||||
response: Final = await client.get(discovery_url)
|
||||
response.raise_for_status()
|
||||
metadata: Final = response.json()
|
||||
jwks_uri: Final = metadata.get("jwks_uri") if isinstance(metadata, dict) else None
|
||||
metadata: Final = _OIDC_DISCOVERY_DOCUMENT.validate_python(response.json())
|
||||
jwks_uri: Final = metadata.get("jwks_uri")
|
||||
if not isinstance(jwks_uri, str) or not jwks_uri:
|
||||
raise ValueError(f"OIDC discovery at {discovery_url} returned no jwks_uri")
|
||||
return jwks_uri
|
||||
|
|
|
|||
|
|
@ -6,7 +6,9 @@ Filters MCP tools semantically for /chat/completions and /responses endpoints.
|
|||
|
||||
import asyncio
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from typing_extensions import TypeVar
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.exceptions import ContextWindowExceededError
|
||||
|
|
@ -21,6 +23,9 @@ if TYPE_CHECKING:
|
|||
from litellm.router import Router
|
||||
|
||||
|
||||
_ToolT: Final = TypeVar("_ToolT", default=object)
|
||||
|
||||
|
||||
class SemanticToolFilterContextWindowError(Exception):
|
||||
"""Raised when the embedding model exceeds its context window, so semantic filtering cannot run."""
|
||||
|
||||
|
|
@ -265,9 +270,9 @@ class SemanticMCPToolFilter:
|
|||
async def filter_tools(
|
||||
self,
|
||||
query: str,
|
||||
available_tools: list[Any],
|
||||
available_tools: list[_ToolT],
|
||||
top_k: int | None = None,
|
||||
) -> list[Any]:
|
||||
) -> list[_ToolT]:
|
||||
"""
|
||||
Filter tools semantically based on query.
|
||||
|
||||
|
|
@ -390,7 +395,7 @@ class SemanticMCPToolFilter:
|
|||
separator: Final = client_name[-len(canonical) - 1]
|
||||
return separator in ("_", "-")
|
||||
|
||||
def _get_tools_by_names(self, tool_names: Sequence[str], available_tools: Sequence[object]) -> list[object]:
|
||||
def _get_tools_by_names(self, tool_names: Sequence[str], available_tools: Sequence[_ToolT]) -> list[_ToolT]:
|
||||
"""
|
||||
Get tools from available_tools by their names, preserving the
|
||||
semantic router's ordering.
|
||||
|
|
@ -406,13 +411,13 @@ class SemanticMCPToolFilter:
|
|||
# Exact matches win over suffix matches when both are present, and
|
||||
# each incoming tool is returned at most once even if two canonical
|
||||
# names happen to be tail-compatible with the same incoming name.
|
||||
available_by_name: Final[dict[str, object]] = {}
|
||||
available_by_name: Final[dict[str, _ToolT]] = {}
|
||||
for tool in available_tools:
|
||||
client_name, _ = self.extract_tool_info(tool)
|
||||
if client_name and client_name not in available_by_name:
|
||||
available_by_name[client_name] = tool
|
||||
|
||||
matched: Final[list[object]] = []
|
||||
matched: Final[list[_ToolT]] = []
|
||||
used_ids: Final[set[int]] = set()
|
||||
for canonical in tool_names:
|
||||
tool = available_by_name.get(canonical)
|
||||
|
|
|
|||
|
|
@ -3,9 +3,12 @@ from collections.abc import Mapping
|
|||
from typing import Any, Final
|
||||
|
||||
import requests
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
|
||||
from .exceptions import NotFoundError, UnauthorizedError
|
||||
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
|
||||
|
||||
|
||||
class ModelsManagementClient:
|
||||
def __init__(self, base_url: str, api_key: str | None = None, timeout: int = 30):
|
||||
|
|
@ -109,7 +112,7 @@ class ModelsManagementClient:
|
|||
try:
|
||||
response: Final = session.send(request.prepare(), timeout=self._timeout)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
return _JSON_OBJECT.validate_python(response.json())
|
||||
except requests.exceptions.HTTPError as e:
|
||||
if e.response.status_code == 401:
|
||||
raise UnauthorizedError(e)
|
||||
|
|
@ -145,7 +148,7 @@ class ModelsManagementClient:
|
|||
try:
|
||||
response: Final = session.send(request.prepare(), timeout=self._timeout)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
return _JSON_OBJECT.validate_python(response.json())
|
||||
except requests.exceptions.HTTPError as e:
|
||||
if e.response.status_code == 401:
|
||||
raise UnauthorizedError(e)
|
||||
|
|
@ -287,7 +290,7 @@ class ModelsManagementClient:
|
|||
try:
|
||||
response: Final = session.send(request.prepare(), timeout=self._timeout)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
return _JSON_OBJECT.validate_python(response.json())
|
||||
except requests.exceptions.HTTPError as e:
|
||||
if e.response.status_code == 401:
|
||||
raise UnauthorizedError(e)
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import json
|
|||
import os
|
||||
import uuid
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal
|
||||
|
||||
try:
|
||||
import ulid
|
||||
|
|
@ -28,6 +28,8 @@ except ImportError:
|
|||
HTTPX_AVAILABLE = False
|
||||
|
||||
from fastapi import HTTPException
|
||||
from pydantic import ConfigDict, TypeAdapter, with_config
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm import DualCache
|
||||
|
|
@ -51,13 +53,17 @@ from litellm.proxy.guardrails._content_utils import (
|
|||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
|
||||
|
||||
class LassoResponse(TypedDict):
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class LassoResponse(TypedDict, total=False):
|
||||
"""Type definition for Lasso API response."""
|
||||
|
||||
violations_detected: bool
|
||||
deputies: dict[str, bool]
|
||||
findings: dict[str, list[dict[str, object]]]
|
||||
messages: list[dict[str, str]] | None
|
||||
violations_detected: NotRequired[ReadOnly[object]]
|
||||
deputies: NotRequired[ReadOnly[dict[str, object]]]
|
||||
findings: NotRequired[ReadOnly[dict[str, object]]]
|
||||
messages: NotRequired[ReadOnly[list[dict[str, object]] | None]]
|
||||
|
||||
|
||||
_LASSO_RESPONSE: Final = TypeAdapter(LassoResponse)
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -467,7 +473,9 @@ class LassoGuardrail(CustomGuardrail):
|
|||
# Also update data["input"] for Responses-API payloads so the
|
||||
# unredacted text doesn't leak through that field.
|
||||
if isinstance(data.get("input"), str):
|
||||
text_parts = [msg["content"] for msg in masked_for_input if isinstance(msg.get("content"), str)]
|
||||
text_parts: Final = [
|
||||
content for msg in masked_for_input if isinstance(content := msg.get("content"), str)
|
||||
]
|
||||
if text_parts:
|
||||
data["input"] = "\n".join(text_parts)
|
||||
self._log_masking_applied(message_type, dict(response))
|
||||
|
|
@ -820,7 +828,7 @@ class LassoGuardrail(CustomGuardrail):
|
|||
timeout=self.timeout if self.timeout is not None else 10.0,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
return _LASSO_RESPONSE.validate_python(response.json())
|
||||
|
||||
def _process_lasso_response(self, response: LassoResponse) -> None:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -13,6 +13,8 @@ from datetime import datetime
|
|||
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, TypeAlias, cast
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.integrations.custom_guardrail import (
|
||||
CustomGuardrail,
|
||||
|
|
@ -37,6 +39,7 @@ if TYPE_CHECKING:
|
|||
|
||||
_DEFAULT_API_BASE: Final = "https://api.noma.security/"
|
||||
_AIDR_SCAN_ENDPOINT: Final = "/litellm/guardrail"
|
||||
_AIDR_RESPONSE: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
|
||||
_INTERVENED_INPUT_FIELDS: Final = ("texts", "images", "tools", "tool_calls")
|
||||
_DEFAULT_API_BASE_HOSTNAME: Final = urlparse(_DEFAULT_API_BASE).hostname
|
||||
|
||||
|
|
@ -257,7 +260,7 @@ class NomaV2Guardrail(CustomGuardrail):
|
|||
response.text,
|
||||
)
|
||||
response.raise_for_status()
|
||||
response_json: Final[dict[str, object]] = response.json()
|
||||
response_json: Final = _AIDR_RESPONSE.validate_python(response.json())
|
||||
verbose_proxy_logger.debug(
|
||||
"Noma v2 AIDR response parsed: %s",
|
||||
json.dumps(response_json, default=str),
|
||||
|
|
|
|||
|
|
@ -164,7 +164,7 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
|
|||
|
||||
from litellm.types.llms.openai import OpenAIModerationResponse
|
||||
|
||||
return OpenAIModerationResponse(**response.json())
|
||||
return OpenAIModerationResponse.model_validate(response.json())
|
||||
|
||||
def _check_moderation_result(self, moderation_response: "OpenAIModerationResponse") -> None:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ import os
|
|||
from typing import Any, Final, Literal
|
||||
|
||||
from fastapi import HTTPException
|
||||
from pydantic import ConfigDict, TypeAdapter, with_config
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.integrations.custom_guardrail import (
|
||||
|
|
@ -31,6 +33,26 @@ GUARDRAIL_NAME: Final = "qualifire"
|
|||
DEFAULT_QUALIFIRE_API_BASE: Final = "https://proxy.qualifire.ai"
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _QualifireFlaggedResult(TypedDict, total=False):
|
||||
flagged: NotRequired[ReadOnly[object]]
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _QualifireEvaluationResult(TypedDict, total=False):
|
||||
results: NotRequired[ReadOnly[list[_QualifireFlaggedResult] | None]]
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _QualifireResponse(TypedDict, total=False):
|
||||
score: NotRequired[ReadOnly[object]]
|
||||
status: NotRequired[ReadOnly[object]]
|
||||
evaluationResults: NotRequired[ReadOnly[list[_QualifireEvaluationResult] | None]]
|
||||
|
||||
|
||||
_QUALIFIRE_RESPONSE: Final = TypeAdapter(_QualifireResponse)
|
||||
|
||||
|
||||
class QualifireGuardrail(CustomGuardrail):
|
||||
@classmethod
|
||||
def get_supported_event_hooks(cls) -> list[GuardrailEventHooks]:
|
||||
|
|
@ -244,7 +266,7 @@ class QualifireGuardrail(CustomGuardrail):
|
|||
|
||||
return api_tools if api_tools else None
|
||||
|
||||
def _check_if_flagged(self, result: dict[str, Any]) -> bool:
|
||||
def _check_if_flagged(self, result: _QualifireResponse) -> bool:
|
||||
"""
|
||||
Check if the Qualifire evaluation result indicates flagged content.
|
||||
|
||||
|
|
@ -381,7 +403,7 @@ class QualifireGuardrail(CustomGuardrail):
|
|||
timeout=self.timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
result: Final = response.json()
|
||||
result: Final = _QUALIFIRE_RESPONSE.validate_python(response.json())
|
||||
|
||||
# Extract response info for logging
|
||||
qualifire_response: Final[dict[str, object]] = {
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@ if TYPE_CHECKING:
|
|||
from litellm.types.agents import AgentResponse
|
||||
from litellm.types.caching import RedisPipelineIncrementOperation
|
||||
|
||||
Span = _Span | Any
|
||||
Span = _Span
|
||||
InternalUsageCache = _InternalUsageCache
|
||||
else:
|
||||
Span = Any
|
||||
|
|
|
|||
|
|
@ -649,7 +649,7 @@ def _metadata_string_tags(metadata_objects: Iterable[Mapping[str, object]]) -> t
|
|||
|
||||
|
||||
def _strip_untrusted_request_header_controls(
|
||||
headers: Any,
|
||||
headers: object,
|
||||
*,
|
||||
allow_client_message_redaction_opt_out: bool = False,
|
||||
) -> None:
|
||||
|
|
@ -1579,7 +1579,7 @@ class LiteLLMProxyRequestSetup:
|
|||
|
||||
@staticmethod
|
||||
def add_internal_user_from_user_mapping(
|
||||
general_settings: dict | None,
|
||||
general_settings: Mapping[str, object] | None,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
headers: dict,
|
||||
) -> UserAPIKeyAuth:
|
||||
|
|
@ -1598,7 +1598,7 @@ class LiteLLMProxyRequestSetup:
|
|||
return user_api_key_dict
|
||||
|
||||
@staticmethod
|
||||
def get_user_from_headers(headers: dict, general_settings: dict | None = None) -> str | None:
|
||||
def get_user_from_headers(headers: dict, general_settings: Mapping[str, object] | None = None) -> str | None:
|
||||
"""
|
||||
Get the user from the specified header if `general_settings.user_header_name` is set.
|
||||
"""
|
||||
|
|
@ -1619,7 +1619,9 @@ class LiteLLMProxyRequestSetup:
|
|||
return user
|
||||
|
||||
@staticmethod
|
||||
def get_openai_org_id_from_headers(headers: dict, general_settings: dict | None = None) -> str | None:
|
||||
def get_openai_org_id_from_headers(
|
||||
headers: dict, general_settings: Mapping[str, object] | None = None
|
||||
) -> str | None:
|
||||
"""
|
||||
Get the OpenAI Org ID from the headers.
|
||||
"""
|
||||
|
|
@ -1710,7 +1712,7 @@ class LiteLLMProxyRequestSetup:
|
|||
headers: dict,
|
||||
request_data: Mapping[str, object],
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
general_settings: dict[str, object] | None = None,
|
||||
general_settings: Mapping[str, object] | None = None,
|
||||
) -> LitellmDataForBackendLLMCall:
|
||||
"""
|
||||
- Adds user from headers
|
||||
|
|
|
|||
|
|
@ -647,8 +647,22 @@ async def update_access_group(
|
|||
_require_proxy_admin(user_api_key_dict)
|
||||
prisma_client: Final = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value)
|
||||
|
||||
update_fields: Final = data.model_dump(exclude_unset=True)
|
||||
update_data: Final[dict] = {"updated_by": user_api_key_dict.user_id}
|
||||
update_fields: Final = {
|
||||
field: value
|
||||
for field, value in {
|
||||
"access_group_name": data.access_group_name,
|
||||
"description": data.description,
|
||||
"access_model_names": data.access_model_names,
|
||||
"access_mcp_server_ids": data.access_mcp_server_ids,
|
||||
"mcp_tool_permissions": data.mcp_tool_permissions,
|
||||
"access_agent_ids": data.access_agent_ids,
|
||||
"access_skill_names": data.access_skill_names,
|
||||
"assigned_team_ids": data.assigned_team_ids,
|
||||
"assigned_key_ids": data.assigned_key_ids,
|
||||
}.items()
|
||||
if field in data.model_fields_set
|
||||
}
|
||||
update_data: Final[dict[str, object]] = {"updated_by": user_api_key_dict.user_id}
|
||||
for field, value in update_fields.items():
|
||||
if field in (
|
||||
"assigned_team_ids",
|
||||
|
|
@ -687,10 +701,10 @@ async def update_access_group(
|
|||
old_team_ids: Final[set[str]] = set(attached[access_group_id])
|
||||
old_key_ids: Final[set[str]] = set(existing.assigned_key_ids or [])
|
||||
new_team_ids: Final[set[str]] = (
|
||||
set(update_fields["assigned_team_ids"] or []) if "assigned_team_ids" in update_fields else old_team_ids
|
||||
set(data.assigned_team_ids or []) if "assigned_team_ids" in update_fields else old_team_ids
|
||||
)
|
||||
new_key_ids: Final[set[str]] = (
|
||||
set(update_fields["assigned_key_ids"] or []) if "assigned_key_ids" in update_fields else old_key_ids
|
||||
set(data.assigned_key_ids or []) if "assigned_key_ids" in update_fields else old_key_ids
|
||||
)
|
||||
|
||||
teams_to_add = list(new_team_ids - old_team_ids)
|
||||
|
|
|
|||
|
|
@ -216,9 +216,12 @@ def _shadow_eval_attempts(prisma_client: "PrismaClient") -> _ShadowEvalAttemptTa
|
|||
return prisma_client.db.litellm_shadowevalattempt
|
||||
|
||||
|
||||
_RAW_ROWS: Final = TypeAdapter(Sequence[dict[str, object]], config=ConfigDict(strict=True))
|
||||
|
||||
|
||||
async def _query_raw(prisma_client: "PrismaClient", query: str, *args: object) -> Sequence[Mapping[str, object]]:
|
||||
async with db_span("auto_router_report_query", sql_relation(query)):
|
||||
return await prisma_client.db.query_raw(query, *args)
|
||||
return _RAW_ROWS.validate_python(await prisma_client.db.query_raw(query, *args))
|
||||
|
||||
|
||||
async def _authorize_router_dry_run(user_api_key_dict: UserAPIKeyAuth, team_id: str | None) -> LiteLLM_TeamTable | None:
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ GET /router/fields - Get router settings field definitions without values (for U
|
|||
import inspect
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final, get_args
|
||||
from typing import Final, get_args
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import Field
|
||||
|
|
@ -32,7 +32,7 @@ router: Final = APIRouter()
|
|||
|
||||
class RouterSettingsResponse(LiteLLMBaseModel):
|
||||
fields: list[RouterSettingsField] = Field(description="List of all configurable router settings with metadata")
|
||||
current_values: dict[str, Any] = Field(description="Current values of router settings")
|
||||
current_values: dict[str, object] = Field(description="Current values of router settings")
|
||||
routing_strategy_descriptions: dict[str, str] = Field(description="Descriptions for each routing strategy option")
|
||||
routing_group_strategies: tuple[str, ...] = Field(
|
||||
description="Strategies supported when constructing a routing group"
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ from litellm.proxy.management_helpers.access_group_team_sync import (
|
|||
)
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging
|
||||
from litellm.repositories.chunked_in import delete_many_in, find_many_in, update_many_in
|
||||
from litellm.repositories.team_repository import TeamRepository
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from prisma import Prisma
|
||||
|
|
@ -144,7 +145,7 @@ def _changes_nothing(plan: RosterPlan) -> bool:
|
|||
|
||||
|
||||
async def _unchanged_roster(prisma_client: PrismaClient, team_id: str) -> RosterSync | TeamGone:
|
||||
team_row: Final = await prisma_client.writer_db.litellm_teamtable.find_unique(where={"team_id": team_id})
|
||||
team_row: Final = await TeamRepository(prisma_client, use_writer=True).table.find_unique(where={"team_id": team_id})
|
||||
if team_row is None:
|
||||
return TeamGone(team_id=team_id)
|
||||
team: Final = LiteLLM_TeamTable.model_validate(team_row.model_dump())
|
||||
|
|
|
|||
|
|
@ -5,7 +5,8 @@ from datetime import datetime
|
|||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
from pydantic import ConfigDict, TypeAdapter, with_config
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
|
@ -53,6 +54,14 @@ else:
|
|||
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _AnthropicBatchCreated(TypedDict):
|
||||
id: ReadOnly[NotRequired[str]]
|
||||
|
||||
|
||||
_ANTHROPIC_BATCH_CREATED: Final = TypeAdapter(_AnthropicBatchCreated)
|
||||
|
||||
|
||||
class AnthropicPassthroughLoggingHandler:
|
||||
@staticmethod
|
||||
def anthropic_passthrough_handler(
|
||||
|
|
@ -552,6 +561,7 @@ class AnthropicPassthroughLoggingHandler:
|
|||
|
||||
# Only handle successful batch job creation (POST requests with 201 status)
|
||||
if httpx_response.status_code == 200 and "id" in _json_response:
|
||||
batch_created: Final = _ANTHROPIC_BATCH_CREATED.validate_python(_json_response)
|
||||
# Transform Anthropic response to LiteLLM batch format
|
||||
anthropic_batches_config: Final = AnthropicBatchesConfig()
|
||||
litellm_batch_response: Final = anthropic_batches_config.transform_retrieve_batch_response(
|
||||
|
|
@ -565,7 +575,7 @@ class AnthropicPassthroughLoggingHandler:
|
|||
litellm_batch_response.status = "validating"
|
||||
|
||||
# Extract batch ID from the response
|
||||
batch_id: Final = _json_response.get("id", "")
|
||||
batch_id: Final = batch_created.get("id", "")
|
||||
|
||||
# Get model from request body (batch response doesn't include model)
|
||||
request_body = request_body or {}
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ from pydantic import TypeAdapter
|
|||
from starlette.datastructures import UploadFile as StarletteUploadFile
|
||||
from starlette.routing import BaseRoute, Route
|
||||
from starlette.websockets import WebSocketState
|
||||
from typing_extensions import ReadOnly
|
||||
from websockets.asyncio.client import connect
|
||||
from websockets.exceptions import (
|
||||
ConnectionClosedError,
|
||||
|
|
@ -168,8 +169,17 @@ def _is_mergeable_client_metadata(key: str, value: object) -> bool:
|
|||
return not key.startswith("user_api_key_") and (key != "model_info" or isinstance(value, Mapping))
|
||||
|
||||
|
||||
class _RegisteredPassThroughRoute(TypedDict):
|
||||
endpoint_id: ReadOnly[str]
|
||||
path: ReadOnly[str | None]
|
||||
type: ReadOnly[str]
|
||||
methods: ReadOnly[str | bool | list[str] | tuple[str, ...] | Mapping[str, object]]
|
||||
auth: ReadOnly[bool]
|
||||
passthrough_params: ReadOnly[Mapping[str, object]]
|
||||
|
||||
|
||||
# Global registry to track registered pass-through routes and prevent memory leaks
|
||||
_registered_pass_through_routes: Final[dict[str, dict[str, str | bool | list[str] | Mapping[str, object]]]] = {}
|
||||
_registered_pass_through_routes: Final[dict[str, _RegisteredPassThroughRoute]] = {}
|
||||
|
||||
|
||||
def get_response_body(response: httpx.Response) -> dict | None:
|
||||
|
|
@ -2208,7 +2218,7 @@ def create_pass_through_route(
|
|||
status_code=status.HTTP_405_METHOD_NOT_ALLOWED,
|
||||
detail=f"Method {request.method} is not allowed for pass-through endpoint {path}.",
|
||||
)
|
||||
target_params: Final = {
|
||||
target_params: Final[dict[str, object]] = {
|
||||
"target": target,
|
||||
"custom_headers": custom_headers,
|
||||
"forward_headers": _forward_headers,
|
||||
|
|
@ -2240,7 +2250,9 @@ def create_pass_through_route(
|
|||
|
||||
# Ensure custom_headers is a dict. Botocore returns a HeadersDict
|
||||
# for SigV4-prepared requests, which is a Mapping but not a dict.
|
||||
headers_dict: Final = dict(param_custom_headers) if isinstance(param_custom_headers, Mapping) else {}
|
||||
headers_dict: Final[dict[str, object]] = (
|
||||
{**param_custom_headers} if isinstance(param_custom_headers, Mapping) else {}
|
||||
)
|
||||
|
||||
# Ensure query_params and custom_body are dicts or None
|
||||
final_query_params: Final = query_params_data if isinstance(query_params_data, dict) else {}
|
||||
|
|
@ -3072,9 +3084,7 @@ class SafeRouteAdder:
|
|||
return f"{path}/{{subpath:path}}"
|
||||
return None
|
||||
|
||||
def registered_route_methods(
|
||||
route_info: Mapping[str, str | bool | Sequence[str] | Mapping[str, object]],
|
||||
) -> tuple[tuple[str, str], ...]:
|
||||
def registered_route_methods(route_info: _RegisteredPassThroughRoute) -> tuple[tuple[str, str], ...]:
|
||||
starlette_path: Final = registered_route_path(route_info)
|
||||
methods: Final = route_info.get("methods")
|
||||
if starlette_path is None or not isinstance(methods, (list, tuple)):
|
||||
|
|
@ -3428,7 +3438,7 @@ class InitPassThroughEndpointHelpers:
|
|||
return (comparison_route, *forwarded)
|
||||
|
||||
@staticmethod
|
||||
def get_registered_pass_through_route(route: str, method: str | None = None) -> dict[str, Any] | None:
|
||||
def get_registered_pass_through_route(route: str, method: str | None = None) -> _RegisteredPassThroughRoute | None:
|
||||
"""Get passthrough params for a given route and optionally filter by HTTP method"""
|
||||
comparison_route: Final = InitPassThroughEndpointHelpers._route_for_registry_lookup(route)
|
||||
for key in _registered_pass_through_routes:
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from datetime import datetime, timedelta
|
|||
from types import MappingProxyType
|
||||
from typing import Final, Literal, TypeVar
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
|
@ -155,6 +155,7 @@ class _DailyUserSpendOwnerRow(LiteLLMBaseModel):
|
|||
last_owner: str | None = None
|
||||
|
||||
|
||||
_RAW_ROWS: Final = TypeAdapter(Sequence[dict[str, object]], config=ConfigDict(strict=True))
|
||||
_TOKEN_DIGEST_ROWS: Final = TypeAdapter(tuple[_TokenDigestRow, ...])
|
||||
_SPEND_LOG_DIGEST_ROWS: Final = TypeAdapter(tuple[_SpendLogDigestRow, ...])
|
||||
_DAILY_USER_SPEND_OWNER_ROWS: Final = TypeAdapter(tuple[_DailyUserSpendOwnerRow, ...])
|
||||
|
|
@ -194,7 +195,7 @@ async def _rows_within_the_statement_timeout(
|
|||
await transaction.execute_raw(_SPEND_LOG_STATEMENT_TIMEOUT_SQL)
|
||||
for setting in planner_settings:
|
||||
await transaction.execute_raw(setting)
|
||||
return await transaction.query_raw(sql, *params)
|
||||
return _RAW_ROWS.validate_python(await transaction.query_raw(sql, *params))
|
||||
|
||||
|
||||
async def _reverse_hash_key_metadata(
|
||||
|
|
|
|||
|
|
@ -21,6 +21,8 @@ from types import MappingProxyType
|
|||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter, with_config
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -41,6 +43,15 @@ if TYPE_CHECKING:
|
|||
from botocore.awsrequest import HTTPHeaders
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _SecretStringResponse(TypedDict):
|
||||
SecretString: ReadOnly[str]
|
||||
|
||||
|
||||
_SECRET_STRING_RESPONSE: Final = TypeAdapter(_SecretStringResponse)
|
||||
_AWS_SECRET_MANAGER_RESPONSE: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
|
||||
|
||||
|
||||
class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager):
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -154,7 +165,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager):
|
|||
try:
|
||||
response: Final = await async_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8"))
|
||||
response.raise_for_status()
|
||||
return response.json()["SecretString"]
|
||||
return _SECRET_STRING_RESPONSE.validate_python(response.json())["SecretString"]
|
||||
except httpx.TimeoutException:
|
||||
raise ValueError("Timeout error occurred")
|
||||
except Exception as e:
|
||||
|
|
@ -204,7 +215,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager):
|
|||
|
||||
try:
|
||||
response: Final = sync_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8"))
|
||||
return response.json()["SecretString"]
|
||||
return _SECRET_STRING_RESPONSE.validate_python(response.json())["SecretString"]
|
||||
except httpx.TimeoutException:
|
||||
raise ValueError("Timeout error occurred")
|
||||
except httpx.HTTPStatusError as e:
|
||||
|
|
@ -424,7 +435,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager):
|
|||
try:
|
||||
response: Final = await async_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8"))
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
return _AWS_SECRET_MANAGER_RESPONSE.validate_python(response.json())
|
||||
except httpx.HTTPStatusError as err:
|
||||
raise ValueError(f"HTTP error occurred: {err.response.text}")
|
||||
except httpx.TimeoutException:
|
||||
|
|
@ -481,7 +492,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager):
|
|||
try:
|
||||
response: Final = await async_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8"))
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
return _AWS_SECRET_MANAGER_RESPONSE.validate_python(response.json())
|
||||
except httpx.HTTPStatusError as err:
|
||||
raise ValueError(f"HTTP error occurred: {err.response.text}")
|
||||
except httpx.TimeoutException:
|
||||
|
|
@ -533,7 +544,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager):
|
|||
try:
|
||||
response: Final = await async_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8"))
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
return _AWS_SECRET_MANAGER_RESPONSE.validate_python(response.json())
|
||||
except httpx.HTTPStatusError as err:
|
||||
raise ValueError(f"HTTP error occurred: {err.response.text}")
|
||||
except httpx.TimeoutException:
|
||||
|
|
@ -613,7 +624,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager):
|
|||
try:
|
||||
response: Final = await async_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8"))
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
return _AWS_SECRET_MANAGER_RESPONSE.validate_python(response.json())
|
||||
except httpx.HTTPStatusError as err:
|
||||
raise ValueError(f"HTTP error occurred: {err.response.text}")
|
||||
except httpx.TimeoutException:
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from typing import Any, Literal
|
||||
from typing import Literal
|
||||
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
|
|
@ -40,7 +40,7 @@ class BedrockKBRetrievalResult(TypedDict, total=False):
|
|||
content: BedrockKBContent | None
|
||||
location: BedrockKBLocation | None
|
||||
score: float | None
|
||||
metadata: dict[str, Any] | None
|
||||
metadata: dict[str, object] | None
|
||||
|
||||
|
||||
class BedrockKBResponse(TypedDict, total=False):
|
||||
|
|
@ -87,21 +87,21 @@ class BedrockKBMetadataConfiguration(TypedDict, total=False):
|
|||
class BedrockKBModelConfiguration(TypedDict, total=False):
|
||||
"""Model configuration for reranking."""
|
||||
|
||||
additionalModelRequestFields: dict[str, Any] | None
|
||||
additionalModelRequestFields: dict[str, object] | None
|
||||
modelArn: str | None
|
||||
|
||||
|
||||
class BedrockKBRerankingConfiguration(TypedDict, total=False):
|
||||
"""Configuration for reranking in vector search."""
|
||||
|
||||
bedrockRerankingConfiguration: dict[str, Any] | None # This could be further typed if needed
|
||||
bedrockRerankingConfiguration: dict[str, object] | None # This could be further typed if needed
|
||||
type: str | None
|
||||
|
||||
|
||||
class BedrockKBVectorSearchConfiguration(TypedDict, total=False):
|
||||
"""Configuration for vector search."""
|
||||
|
||||
filter: dict[str, Any] | None
|
||||
filter: dict[str, object] | None
|
||||
implicitFilterConfiguration: BedrockKBImplicitFilterConfiguration | None
|
||||
numberOfResults: int | None
|
||||
overrideSearchType: str | None
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ Type definitions for AWS Bedrock Invoke Agent API responses.
|
|||
https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agent-runtime_InvokeAgent.html
|
||||
"""
|
||||
|
||||
from typing import Any, Final
|
||||
from typing import Final
|
||||
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
|
|
@ -39,7 +39,7 @@ class InvokeAgentModelInvocationInput(TypedDict, total=False):
|
|||
"""Model invocation input details."""
|
||||
|
||||
foundationModel: str | None
|
||||
inferenceConfiguration: dict[str, Any] | None
|
||||
inferenceConfiguration: dict[str, object] | None
|
||||
text: str | None
|
||||
traceId: str | None
|
||||
type: str | None
|
||||
|
|
@ -49,9 +49,9 @@ class InvokeAgentModelInvocationOutput(TypedDict, total=False):
|
|||
"""Model invocation output details."""
|
||||
|
||||
metadata: InvokeAgentMetadata | None
|
||||
parsedResponse: dict[str, Any] | None
|
||||
rawResponse: dict[str, Any] | None
|
||||
reasoningContent: dict[str, Any] | None
|
||||
parsedResponse: dict[str, object] | None
|
||||
rawResponse: dict[str, object] | None
|
||||
reasoningContent: dict[str, object] | None
|
||||
traceId: str | None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from collections.abc import Mapping
|
||||
from enum import Enum
|
||||
from typing import Any, Literal
|
||||
from typing import Literal
|
||||
|
||||
from typing_extensions import Required, TypedDict
|
||||
|
||||
|
|
@ -235,7 +236,7 @@ class GeminiVideoGenerationInstance(TypedDict, total=False):
|
|||
"""Instance data for Gemini video generation request"""
|
||||
|
||||
prompt: Required[str]
|
||||
image: dict[str, Any]
|
||||
image: Mapping[str, object]
|
||||
|
||||
|
||||
class GeminiVideoGenerationParameters(LiteLLMBaseModel):
|
||||
|
|
@ -263,7 +264,7 @@ class GeminiVideoGenerationParameters(LiteLLMBaseModel):
|
|||
negativePrompt: str | None = None
|
||||
"""Text describing what not to include in the video."""
|
||||
|
||||
lastFrame: Any | None = None
|
||||
lastFrame: object | None = None
|
||||
"""
|
||||
The final image for interpolation video to transition.
|
||||
Should be used with the 'image' parameter.
|
||||
|
|
@ -275,7 +276,7 @@ class GeminiVideoGenerationParameters(LiteLLMBaseModel):
|
|||
Only supported in Veo 3.1 (list of VideoGenerationReferenceImage objects).
|
||||
"""
|
||||
|
||||
video: Any | None = None
|
||||
video: object | None = None
|
||||
"""
|
||||
Video to be used for video extension (Video object).
|
||||
Only supported in Veo 3.1 & Veo 3 Fast.
|
||||
|
|
@ -354,5 +355,5 @@ class GeminiLongRunningOperationResponse(LiteLLMBaseModel):
|
|||
response: GeminiOperationResponse | None = None
|
||||
"""Response object when operation is complete"""
|
||||
|
||||
error: dict[str, Any] | None = None
|
||||
error: dict[str, object] | None = None
|
||||
"""Error details if operation failed"""
|
||||
|
|
|
|||
|
|
@ -4,7 +4,8 @@ Type definitions for LangGraph API.
|
|||
LangGraph provides a streaming and non-streaming API for running agents.
|
||||
"""
|
||||
|
||||
from typing import Any, Literal
|
||||
from collections.abc import Mapping
|
||||
from typing import Literal
|
||||
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
|
|
@ -29,8 +30,8 @@ class LangGraphRequest(TypedDict, total=False):
|
|||
assistant_id: str
|
||||
input: LangGraphInput
|
||||
stream_mode: str | None
|
||||
config: dict[str, Any] | None
|
||||
metadata: dict[str, Any] | None
|
||||
config: Mapping[str, object] | None
|
||||
metadata: Mapping[str, object] | None
|
||||
|
||||
|
||||
# Response Types - Streaming
|
||||
|
|
@ -38,7 +39,7 @@ class LangGraphStreamEvent(TypedDict, total=False):
|
|||
"""Single event in a LangGraph stream response."""
|
||||
|
||||
event: str
|
||||
data: Any
|
||||
data: object
|
||||
|
||||
|
||||
# Response Types - Non-streaming
|
||||
|
|
@ -55,7 +56,7 @@ class LangGraphResponse(TypedDict, total=False):
|
|||
"""Non-streaming response structure from LangGraph."""
|
||||
|
||||
messages: list[LangGraphResponseMessage]
|
||||
values: dict[str, Any]
|
||||
values: dict[str, object]
|
||||
|
||||
|
||||
# Parsed response for internal use
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
from collections.abc import Callable
|
||||
from typing import Any, ClassVar
|
||||
from typing import ClassVar
|
||||
|
||||
from pydantic import ConfigDict, Field
|
||||
|
||||
|
|
@ -10,7 +10,7 @@ class MCPTool(LiteLLMBaseModel):
|
|||
model_config: ClassVar[ConfigDict] = ConfigDict(arbitrary_types_allowed=True)
|
||||
name: str
|
||||
description: str
|
||||
input_schema: dict[str, Any]
|
||||
input_schema: dict[str, object]
|
||||
handler: Callable
|
||||
server_id: str | None = Field(default=None, frozen=True)
|
||||
|
||||
|
|
@ -18,18 +18,18 @@ class MCPTool(LiteLLMBaseModel):
|
|||
class ToolSchema(LiteLLMBaseModel):
|
||||
name: str
|
||||
description: str
|
||||
inputSchema: dict[str, Any]
|
||||
inputSchema: dict[str, object]
|
||||
|
||||
|
||||
class ListToolsResponse(LiteLLMBaseModel):
|
||||
tools: list[ToolSchema]
|
||||
nextCursor: str | None = None
|
||||
_meta: dict[str, Any] | None = None
|
||||
_meta: dict[str, object] | None = None
|
||||
|
||||
|
||||
class CallToolRequest(LiteLLMBaseModel):
|
||||
method: str = "tools/call"
|
||||
params: dict[str, Any]
|
||||
params: dict[str, object]
|
||||
|
||||
|
||||
class ContentItem(LiteLLMBaseModel):
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
from typing import Any
|
||||
|
||||
from pydantic import Field
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
|
|
@ -20,7 +18,7 @@ class EnkryptAIPolicyViolationDetail(TypedDict, total=False):
|
|||
class EnkryptAIPIIDetail(TypedDict, total=False):
|
||||
"""Details for PII detection."""
|
||||
|
||||
pii: dict[str, Any]
|
||||
pii: dict[str, object]
|
||||
|
||||
|
||||
class EnkryptAIToxicityDetail(TypedDict, total=False):
|
||||
|
|
@ -85,8 +83,8 @@ class EnkryptAIResponseDetails(TypedDict, total=False):
|
|||
toxicity: EnkryptAIToxicityDetail | None
|
||||
keyword_detected: EnkryptAIKeywordDetail | None
|
||||
bias: EnkryptAIBiasDetail | None
|
||||
prompt_injection: dict[str, Any] | None
|
||||
jailbreak: dict[str, Any] | None
|
||||
prompt_injection: dict[str, object] | None
|
||||
jailbreak: dict[str, object] | None
|
||||
|
||||
|
||||
class EnkryptAIResponse(TypedDict, total=False):
|
||||
|
|
@ -107,7 +105,7 @@ class EnkryptAIProcessedResult(TypedDict):
|
|||
| EnkryptAIToxicityDetail
|
||||
| EnkryptAIKeywordDetail
|
||||
| EnkryptAIBiasDetail
|
||||
| dict[str, Any],
|
||||
| dict[str, object],
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue