mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-22 10:40:46 +00:00
refactor(typing): narrow Any sources across providers, proxy and guardrails (wave 1)
This commit is contained in:
parent
5145250d83
commit
ac9d475697
202 changed files with 1304 additions and 644 deletions
|
|
@ -15,7 +15,7 @@ sys.path.insert(
|
|||
) # Adds the parent directory to the system path
|
||||
import json
|
||||
import sys
|
||||
from typing import Any, List, Literal, Optional
|
||||
from typing import List, Literal, Optional
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
|
|
@ -68,7 +68,7 @@ class AporiaGuardrail(CustomGuardrail):
|
|||
async def prepare_aporia_request(
|
||||
self, new_messages: List[dict], response_string: Optional[str] = None
|
||||
) -> dict:
|
||||
data: dict[str, Any] = {}
|
||||
data: dict[str, object] = {}
|
||||
if new_messages is not None:
|
||||
data["messages"] = new_messages
|
||||
if response_string is not None:
|
||||
|
|
|
|||
|
|
@ -1,15 +1,20 @@
|
|||
# this is an example endpoint to receive data from litellm
|
||||
from typing import Final
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
_LOGGED_EVENT: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
@app.post("/log-event")
|
||||
async def log_event(request: Request):
|
||||
try:
|
||||
print("Received /log-event request") # noqa
|
||||
# Assuming the incoming request has JSON data
|
||||
data = await request.json()
|
||||
data: Final = _LOGGED_EVENT.validate_python(await request.json())
|
||||
print("Received request data:") # noqa
|
||||
print(data) # noqa
|
||||
|
||||
|
|
|
|||
|
|
@ -2652,7 +2652,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
# Continue with other files even if one fails
|
||||
continue
|
||||
|
||||
def _get_content_type_from_file_object(self, file_object: Optional[Any]) -> str:
|
||||
def _get_content_type_from_file_object(self, file_object: object) -> str:
|
||||
"""
|
||||
Determine content type from file object.
|
||||
|
||||
|
|
|
|||
|
|
@ -175,7 +175,7 @@ argilla_batch_size: Optional[int] = None
|
|||
datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload.
|
||||
gcs_pub_sub_use_v1: Optional[bool] = False # if you want to use v1 gcs pubsub logged payload
|
||||
generic_api_use_v1: Optional[bool] = False # if you want to use v1 generic api logged payload
|
||||
argilla_transformation_object: Optional[Dict[str, Any]] = None
|
||||
argilla_transformation_object: Mapping[str, str] | None = None
|
||||
_async_input_callback: List[Union[str, Callable[..., object], "CustomLogger"]] = ( # CustomLogger is lazy-loaded
|
||||
[]
|
||||
) # internal variable - async custom callbacks are routed here.
|
||||
|
|
@ -333,7 +333,7 @@ in_memory_llm_clients_cache: "LLMClientCache"
|
|||
safe_memory_mode: bool = False
|
||||
enable_azure_ad_token_refresh: Optional[bool] = False
|
||||
# Proxy Authentication - auto-obtain/refresh OAuth2/JWT tokens for LiteLLM Proxy
|
||||
proxy_auth: Optional[Any] = None
|
||||
proxy_auth: _ProxyAuthHandlerType | None = None
|
||||
### DEFAULT AZURE API VERSION ###
|
||||
AZURE_DEFAULT_API_VERSION = "2025-02-01-preview" # this is updated to the latest
|
||||
### DEFAULT WATSONX API VERSION ###
|
||||
|
|
@ -503,7 +503,7 @@ skill_search_embedding_model: Optional[str] = None
|
|||
# Supports both old format (Dict[str, str]) and new format (Dict[str, Dict[str, Any]])
|
||||
# New format: { "displayName": { "url": "...", "index": 0 } }
|
||||
# Old format: { "displayName": "url" } (for backward compatibility)
|
||||
public_model_groups_links: Dict[str, Union[str, Dict[str, Any]]] = {}
|
||||
public_model_groups_links: dict[str, str | dict[str, object]] = {}
|
||||
#### REQUEST PRIORITIZATION #######
|
||||
priority_reservation: Optional[Dict[str, Union[float, "PriorityReservationDict"]]] = None
|
||||
# priority_reservation_settings is lazy-loaded via __getattr__
|
||||
|
|
@ -2284,6 +2284,10 @@ if TYPE_CHECKING:
|
|||
LiteLLM_UpperboundKeyGenerateParams,
|
||||
)
|
||||
from litellm.utils import ModelResponseListIterator as _ModelResponseListIterator
|
||||
from collections.abc import Awaitable
|
||||
from litellm.types.utils import ModelResponse as _ModelResponseType
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper as _CustomStreamWrapperType
|
||||
from litellm.proxy_auth import ProxyAuthHandler as _ProxyAuthHandlerType
|
||||
|
||||
# Cost calculator functions
|
||||
cost_per_token: Callable[..., Tuple[float, float]]
|
||||
|
|
@ -2307,7 +2311,7 @@ if TYPE_CHECKING:
|
|||
supports_audio_output: Callable[..., bool]
|
||||
supports_system_messages: Callable[..., bool]
|
||||
supports_reasoning: Callable[..., bool]
|
||||
acreate: Callable[..., Any]
|
||||
acreate: Callable[..., Awaitable[_ModelResponseType | _CustomStreamWrapperType]]
|
||||
get_max_tokens: Callable[..., int]
|
||||
get_model_info: Callable[..., _ModelInfoType]
|
||||
register_prompt_template: Callable[..., None]
|
||||
|
|
@ -2605,8 +2609,10 @@ def __getattr__(name: str) -> Any:
|
|||
cancel_run,
|
||||
delete_run,
|
||||
)
|
||||
from types import MappingProxyType
|
||||
|
||||
return locals()[name]
|
||||
evals_exports: Final[Mapping[str, object]] = MappingProxyType(locals())
|
||||
return evals_exports[name]
|
||||
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ import importlib
|
|||
import sys
|
||||
from collections.abc import Callable, Mapping
|
||||
from types import ModuleType
|
||||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
|
|
@ -456,7 +456,7 @@ def _lazy_import_http_handlers(name: str) -> object:
|
|||
params: Final = {"timeout": async_timeout, "client_alias": "module level aclient"}
|
||||
|
||||
# Create the client instance
|
||||
provider_id: Final = cast(Any, "litellm_module_level_client")
|
||||
provider_id: Final = "litellm_module_level_client"
|
||||
async_client: Final = get_async_httpx_client(
|
||||
llm_provider=provider_id,
|
||||
params=params,
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ class A2ACompletionBridgeHandler:
|
|||
@staticmethod
|
||||
def _build_completion_params(
|
||||
params: dict[str, Any],
|
||||
litellm_params: Mapping[str, Any],
|
||||
litellm_params: Mapping[str, object],
|
||||
api_base: str | None,
|
||||
agent_extra_headers: Mapping[str, str] | None,
|
||||
*,
|
||||
|
|
|
|||
|
|
@ -84,7 +84,7 @@ class A2ARequestUtils:
|
|||
Returns:
|
||||
The message object/dict or None
|
||||
"""
|
||||
params: Final = getattr(request, "params", None)
|
||||
params: Final[object] = getattr(request, "params", None)
|
||||
if params is None:
|
||||
return None
|
||||
return getattr(params, "message", None)
|
||||
|
|
|
|||
|
|
@ -7,10 +7,14 @@ Similar to litellm/litellm_core_utils/exception_mapping_utils.py but for Anthrop
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
|
||||
from .exceptions import AnthropicErrorDetail, AnthropicErrorResponse, AnthropicErrorType
|
||||
|
||||
_PARSED_ERROR_BODY: Final = TypeAdapter(object)
|
||||
|
||||
# HTTP status code -> Anthropic error type
|
||||
# Source: https://docs.anthropic.com/en/api/errors
|
||||
ANTHROPIC_ERROR_TYPE_MAP: Final[dict[int, AnthropicErrorType]] = {
|
||||
|
|
@ -79,7 +83,7 @@ class AnthropicExceptionMapping:
|
|||
- Generic: {"message": "..."}
|
||||
- Plain strings
|
||||
"""
|
||||
parsed: Final = safe_json_loads(raw_message)
|
||||
parsed: Final = _PARSED_ERROR_BODY.validate_python(safe_json_loads(raw_message))
|
||||
if isinstance(parsed, dict):
|
||||
# Bedrock format
|
||||
if "detail" in parsed and isinstance(parsed["detail"], dict):
|
||||
|
|
@ -145,9 +149,8 @@ class AnthropicExceptionMapping:
|
|||
AnthropicErrorResponse dict
|
||||
"""
|
||||
# Try to parse as JSON once
|
||||
parsed: dict | None = safe_json_loads(raw_message)
|
||||
if not isinstance(parsed, dict):
|
||||
parsed = None
|
||||
decoded: Final = _PARSED_ERROR_BODY.validate_python(safe_json_loads(raw_message))
|
||||
parsed: Final[dict[str, object] | None] = decoded if isinstance(decoded, dict) else None
|
||||
|
||||
# If parsed and already in Anthropic format - passthrough
|
||||
if parsed is not None and AnthropicExceptionMapping._is_anthropic_error_dict(parsed):
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ def get_assistants(
|
|||
api_base: str | None = None,
|
||||
api_version: str | None = None,
|
||||
**kwargs,
|
||||
) -> SyncCursorPage[Assistant]:
|
||||
) -> SyncCursorPage[Assistant] | Coroutine[None, None, AsyncCursorPage[Assistant]]:
|
||||
aget_assistants: Final[bool | None] = kwargs.pop("aget_assistants", None)
|
||||
if aget_assistants is not None and not isinstance(aget_assistants, bool):
|
||||
raise Exception("Invalid value passed in for aget_assistants. Only bool or None allowed")
|
||||
|
|
@ -100,7 +100,7 @@ def get_assistants(
|
|||
elif timeout is None:
|
||||
timeout = 600.0
|
||||
|
||||
response: SyncCursorPage[Assistant] | None = None
|
||||
response: SyncCursorPage[Assistant] | Coroutine[None, None, AsyncCursorPage[Assistant]] | None = None
|
||||
if custom_llm_provider == "openai":
|
||||
api_base = (
|
||||
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
|
||||
|
|
@ -564,7 +564,7 @@ def create_thread(
|
|||
tool_resources: OpenAICreateThreadParamsToolResources | None = None,
|
||||
client: OpenAI | None = None,
|
||||
**kwargs,
|
||||
) -> Thread:
|
||||
) -> Thread | Coroutine[None, None, Thread]:
|
||||
"""
|
||||
- get the llm provider
|
||||
- if openai - route it there
|
||||
|
|
@ -610,7 +610,7 @@ def create_thread(
|
|||
api_base: str | None = None
|
||||
api_key: str | None = None
|
||||
|
||||
response: Thread | None = None
|
||||
response: Thread | Coroutine[None, None, Thread] | None = None
|
||||
if custom_llm_provider == "openai":
|
||||
api_base = (
|
||||
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
|
||||
|
|
@ -735,7 +735,7 @@ def get_thread(
|
|||
thread_id: str,
|
||||
client=None,
|
||||
**kwargs,
|
||||
) -> Thread:
|
||||
) -> Thread | Coroutine[None, None, Thread]:
|
||||
"""Get the thread object, given a thread_id"""
|
||||
aget_thread: Final = kwargs.pop("aget_thread", None)
|
||||
optional_params: Final = GenericLiteLLMParams(**kwargs)
|
||||
|
|
@ -757,7 +757,7 @@ def get_thread(
|
|||
timeout = 600.0
|
||||
api_base: str | None = None
|
||||
api_key: str | None = None
|
||||
response: Thread | None = None
|
||||
response: Thread | Coroutine[None, None, Thread] | None = None
|
||||
if custom_llm_provider == "openai":
|
||||
api_base = (
|
||||
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
|
||||
|
|
@ -903,7 +903,7 @@ def add_message(
|
|||
metadata: dict | None = None,
|
||||
client=None,
|
||||
**kwargs,
|
||||
) -> OpenAIMessage:
|
||||
) -> OpenAIMessage | Coroutine[None, None, OpenAIMessage]:
|
||||
### COMMON OBJECTS ###
|
||||
a_add_message: Final = kwargs.pop("a_add_message", None)
|
||||
_message_data: Final = MessageData(role=role, content=content, attachments=attachments, metadata=metadata)
|
||||
|
|
@ -935,7 +935,7 @@ def add_message(
|
|||
timeout = 600.0
|
||||
api_key: str | None = None
|
||||
api_base: str | None = None
|
||||
response: OpenAIMessage | None = None
|
||||
response: OpenAIMessage | Coroutine[None, None, OpenAIMessage] | None = None
|
||||
if custom_llm_provider == "openai":
|
||||
api_base = (
|
||||
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
|
||||
|
|
@ -1065,7 +1065,7 @@ def get_messages(
|
|||
thread_id: str,
|
||||
client: Any | None = None,
|
||||
**kwargs,
|
||||
) -> SyncCursorPage[OpenAIMessage]:
|
||||
) -> SyncCursorPage[OpenAIMessage] | Coroutine[None, None, AsyncCursorPage[OpenAIMessage]]:
|
||||
aget_messages: Final = kwargs.pop("aget_messages", None)
|
||||
optional_params: Final = GenericLiteLLMParams(**kwargs)
|
||||
litellm_params_dict: Final = get_litellm_params(**kwargs)
|
||||
|
|
@ -1086,7 +1086,7 @@ def get_messages(
|
|||
elif timeout is None:
|
||||
timeout = 600.0
|
||||
|
||||
response: SyncCursorPage[OpenAIMessage] | None = None
|
||||
response: SyncCursorPage[OpenAIMessage] | Coroutine[None, None, AsyncCursorPage[OpenAIMessage]] | None = None
|
||||
api_key: str | None = None
|
||||
api_base: str | None = None
|
||||
if custom_llm_provider == "openai":
|
||||
|
|
@ -1256,7 +1256,7 @@ def run_thread(
|
|||
client: object | None = None,
|
||||
event_handler: AssistantEventHandler | None = None, # for stream=True calls
|
||||
**kwargs,
|
||||
) -> Run:
|
||||
) -> Run | Coroutine[None, None, Run]:
|
||||
"""Run a given thread + assistant."""
|
||||
arun_thread: Final = kwargs.pop("arun_thread", None)
|
||||
optional_params: Final = GenericLiteLLMParams(**kwargs)
|
||||
|
|
@ -1278,7 +1278,7 @@ def run_thread(
|
|||
elif timeout is None:
|
||||
timeout = 600.0
|
||||
|
||||
response: Run | None = None
|
||||
response: Run | Coroutine[None, None, Run] | None = None
|
||||
if custom_llm_provider == "openai":
|
||||
api_base = (
|
||||
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
|
||||
|
|
|
|||
|
|
@ -13,10 +13,14 @@ import json
|
|||
from contextlib import suppress
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._logging import print_verbose, verbose_logger
|
||||
|
||||
from .base_cache import BaseCache
|
||||
|
||||
_DECODED_BLOB: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class AzureBlobCache(BaseCache):
|
||||
def __init__(self, account_url, container) -> None:
|
||||
|
|
@ -65,7 +69,7 @@ class AzureBlobCache(BaseCache):
|
|||
print_verbose(f"Get Azure Blob Cache: key: {key}")
|
||||
as_bytes: Final = self.container_client.download_blob(key).readall()
|
||||
as_str: Final = as_bytes.decode("utf-8")
|
||||
cached_response: Final[object] = json.loads(as_str)
|
||||
cached_response: Final = _DECODED_BLOB.validate_python(json.loads(as_str))
|
||||
|
||||
verbose_logger.debug(
|
||||
"Got Azure Blob Cache: key: %s, cached_response %s. Type Response %s",
|
||||
|
|
@ -86,7 +90,7 @@ class AzureBlobCache(BaseCache):
|
|||
blob: Final = await self.async_container_client.download_blob(key)
|
||||
as_bytes: Final = await blob.readall()
|
||||
as_str: Final = as_bytes.decode("utf-8")
|
||||
cached_response: Final[object] = json.loads(as_str)
|
||||
cached_response: Final = _DECODED_BLOB.validate_python(json.loads(as_str))
|
||||
verbose_logger.debug(
|
||||
"Got Azure Blob Cache: key: %s, cached_response %s. Type Response %s",
|
||||
key,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import json
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from .base_cache import BaseCache
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -10,6 +12,8 @@ if TYPE_CHECKING:
|
|||
else:
|
||||
Span = Any
|
||||
|
||||
_DECODED_DISK_VALUE: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class DiskCache(BaseCache):
|
||||
def __init__(self, disk_cache_dir: str | None = None):
|
||||
|
|
@ -44,7 +48,7 @@ class DiskCache(BaseCache):
|
|||
original_cached_response: Final = self.disk_cache.get(key)
|
||||
if original_cached_response:
|
||||
try:
|
||||
cached_response = json.loads(original_cached_response)
|
||||
cached_response = _DECODED_DISK_VALUE.validate_python(json.loads(original_cached_response))
|
||||
except Exception:
|
||||
cached_response = original_cached_response
|
||||
return cached_response
|
||||
|
|
|
|||
|
|
@ -2994,8 +2994,11 @@ def _batch_prompt_cost(
|
|||
)
|
||||
|
||||
|
||||
_ATTRIBUTE_VALUE: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
def _attribute_value(obj: object, name: str) -> object:
|
||||
return getattr(obj, name)
|
||||
return _ATTRIBUTE_VALUE.validate_python(getattr(obj, name))
|
||||
|
||||
|
||||
def _summable_prompt_token_fields(prompt_tokens_details: BaseModel) -> list[str]:
|
||||
|
|
|
|||
|
|
@ -1273,7 +1273,7 @@ class MidStreamFallbackError(ServiceUnavailableError):
|
|||
self.message = _saved_message
|
||||
self.args = (_saved_message,)
|
||||
|
||||
original_headers: Final = getattr(original_exception, "litellm_response_headers", None)
|
||||
original_headers: Final[object] = getattr(original_exception, "litellm_response_headers", None)
|
||||
if original_headers is not None:
|
||||
self.litellm_response_headers = original_headers
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import asyncio
|
|||
import contextvars
|
||||
import time
|
||||
import uuid as uuid_module
|
||||
from collections.abc import Coroutine
|
||||
from collections.abc import Awaitable, Coroutine
|
||||
from functools import partial
|
||||
from typing import Final, Literal, cast
|
||||
|
||||
|
|
@ -293,6 +293,14 @@ def create_file(
|
|||
raise e
|
||||
|
||||
|
||||
async def _await_retrieved_file(pending: Awaitable[FileObject | OpenAIFileObject]) -> FileObject | OpenAIFileObject:
|
||||
return await pending
|
||||
|
||||
|
||||
async def _await_deleted_file(pending: Awaitable[object]) -> object:
|
||||
return await pending
|
||||
|
||||
|
||||
@client
|
||||
async def afile_retrieve(
|
||||
file_id: str,
|
||||
|
|
@ -325,7 +333,7 @@ async def afile_retrieve(
|
|||
func_with_context: Final = partial(ctx.run, func)
|
||||
init_response: Final = await loop.run_in_executor(None, func_with_context)
|
||||
if asyncio.iscoroutine(init_response):
|
||||
response = await init_response
|
||||
response = await _await_retrieved_file(init_response)
|
||||
else:
|
||||
response = init_response
|
||||
|
||||
|
|
@ -505,7 +513,7 @@ async def afile_delete(
|
|||
func_with_context: Final = partial(ctx.run, func)
|
||||
init_response: Final = await loop.run_in_executor(None, func_with_context)
|
||||
if asyncio.iscoroutine(init_response):
|
||||
response = await init_response
|
||||
response = await _await_deleted_file(init_response)
|
||||
else:
|
||||
response = init_response
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import inspect
|
|||
import logging
|
||||
import os
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator, Callable, Coroutine, Generator, Mapping, Sequence
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Generator, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from types import MappingProxyType
|
||||
from typing import (
|
||||
|
|
@ -331,11 +331,15 @@ async def _put_end(queue: asyncio.Queue[Event | _End], end: _End) -> None:
|
|||
queue.put_nowait(end)
|
||||
|
||||
|
||||
async def _await_decision(pending: Awaitable[object]) -> object:
|
||||
return await pending
|
||||
|
||||
|
||||
async def call_approval_handler(handler: ApprovalHandler, approval: Approval) -> None:
|
||||
"""Run on_approval (sync in a worker thread, or async) and resolve approval."""
|
||||
try:
|
||||
if inspect.iscoroutinefunction(handler):
|
||||
decision: object = await handler(approval)
|
||||
decision: object = await _await_decision(handler(approval))
|
||||
else:
|
||||
decision = await asyncio.to_thread(handler, approval)
|
||||
if inspect.isawaitable(decision):
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import contextvars
|
|||
import importlib
|
||||
from collections.abc import Coroutine
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Final, Literal, Optional, cast, overload
|
||||
from typing import TYPE_CHECKING, Final, Literal, cast, overload
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
|
|
@ -70,16 +70,17 @@ from litellm.utils import (
|
|||
_PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints)
|
||||
|
||||
# Cache for ImageEditRequestUtils to avoid repeated __getattr__ calls
|
||||
_ImageEditRequestUtils_cache: Optional["ImageEditRequestUtils"] = None
|
||||
_ImageEditRequestUtils_cache: type["ImageEditRequestUtils"] | None = None
|
||||
|
||||
|
||||
def _get_ImageEditRequestUtils() -> "ImageEditRequestUtils":
|
||||
def _get_ImageEditRequestUtils() -> type["ImageEditRequestUtils"]:
|
||||
"""Get ImageEditRequestUtils, loading it lazily if needed."""
|
||||
global _ImageEditRequestUtils_cache
|
||||
if _ImageEditRequestUtils_cache is None:
|
||||
# Access via module to trigger __getattr__ if not cached
|
||||
module: Final = importlib.import_module(__name__)
|
||||
_ImageEditRequestUtils_cache = module.ImageEditRequestUtils
|
||||
from litellm.images import main as images_main
|
||||
|
||||
_ImageEditRequestUtils_cache = images_main.ImageEditRequestUtils
|
||||
assert _ImageEditRequestUtils_cache is not None # Type narrowing for type checker
|
||||
return _ImageEditRequestUtils_cache
|
||||
|
||||
|
|
|
|||
|
|
@ -166,7 +166,7 @@ class ArizeLogger(OpenTelemetry):
|
|||
primary ``otel`` callback which handles proxy-level parent spans.
|
||||
"""
|
||||
|
||||
def set_attributes(self, span: Span, kwargs, response_obj: Any | None):
|
||||
def set_attributes(self, span: Span, kwargs, response_obj: object):
|
||||
ArizeLogger.set_arize_attributes(span, kwargs, response_obj)
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -352,7 +352,7 @@ class CustomGuardrail(CustomLogger):
|
|||
self,
|
||||
violation_message: str,
|
||||
request_data: dict[str, Any],
|
||||
detection_info: dict[str, Any] | None = None,
|
||||
detection_info: Mapping[str, object] | None = None,
|
||||
original_response: object = None,
|
||||
) -> None:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Final
|
||||
|
||||
from .base import FocusDestination
|
||||
|
|
@ -20,7 +21,7 @@ class FocusDestinationFactory:
|
|||
*,
|
||||
provider: str,
|
||||
prefix: str,
|
||||
config: dict[str, Any] | None = None,
|
||||
config: Mapping[str, object] | None = None,
|
||||
) -> FocusDestination:
|
||||
"""Return a destination implementation for the requested provider."""
|
||||
provider_lower: Final = provider.lower()
|
||||
|
|
@ -39,7 +40,7 @@ class FocusDestinationFactory:
|
|||
def _resolve_config(
|
||||
*,
|
||||
provider: str,
|
||||
overrides: dict[str, object],
|
||||
overrides: Mapping[str, object],
|
||||
) -> dict[str, Any]:
|
||||
if provider == "s3":
|
||||
resolved = {
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any, Final, SupportsFloat, SupportsIndex, SupportsInt, cast
|
||||
from collections.abc import Callable, Mapping
|
||||
from typing import Final, SupportsFloat, SupportsIndex, SupportsInt, cast
|
||||
|
||||
import polars as pl
|
||||
|
||||
|
|
@ -24,7 +24,7 @@ class FocusExportEngine:
|
|||
provider: str,
|
||||
export_format: str,
|
||||
prefix: str,
|
||||
destination_config: dict[str, Any] | None = None,
|
||||
destination_config: Mapping[str, object] | None = None,
|
||||
) -> None:
|
||||
self.provider = provider
|
||||
self.export_format = export_format
|
||||
|
|
@ -77,7 +77,7 @@ class FocusExportEngine:
|
|||
return FocusParquetSerializer()
|
||||
raise NotImplementedError(f"Export format '{self.export_format}' not supported. Use 'parquet' or 'csv'.")
|
||||
|
||||
async def dry_run_export_usage_data(self, limit: int | None) -> dict[str, Any]:
|
||||
async def dry_run_export_usage_data(self, limit: int | None) -> dict[str, object]:
|
||||
data: Final = await self.database.get_usage_data(limit=limit)
|
||||
normalized: Final = self.transformer.transform(data)
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ import os
|
|||
from datetime import datetime, timedelta, timezone
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, Final, cast
|
||||
|
||||
from typing_extensions import TypedDict, Unpack
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
|
@ -25,6 +27,10 @@ FOCUS_USAGE_DATA_JOB_NAME: Final = "focus_export_usage_data"
|
|||
DEFAULT_DRY_RUN_LIMIT: Final = 500
|
||||
|
||||
|
||||
class _CustomLoggerOptions(TypedDict, total=False, extra_items=object):
|
||||
pass
|
||||
|
||||
|
||||
class FocusLogger(CustomLogger):
|
||||
"""Coordinates Focus export jobs across transformer/serializer/destination layers."""
|
||||
|
||||
|
|
@ -40,7 +46,7 @@ class FocusLogger(CustomLogger):
|
|||
interval_seconds: int | None = None,
|
||||
prefix: str | None = None,
|
||||
destination_config: dict[str, Any] | None = None,
|
||||
**kwargs: Any,
|
||||
**kwargs: Unpack[_CustomLoggerOptions],
|
||||
) -> None:
|
||||
super().__init__(**kwargs)
|
||||
self.provider = (provider or os.getenv("FOCUS_PROVIDER") or "s3").lower()
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ Usage:
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from collections.abc import AsyncIterable, Iterable
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -40,7 +41,7 @@ _mocks_initialized = False
|
|||
|
||||
# Default mock latency in seconds (simulates network round-trip)
|
||||
# Typical GCS API calls take 100-300ms for uploads, 50-150ms for GET/DELETE
|
||||
_MOCK_LATENCY_SECONDS: Final = float(__import__("os").getenv("GCS_MOCK_LATENCY_MS", "150")) / 1000.0
|
||||
_MOCK_LATENCY_SECONDS: Final = float(os.getenv("GCS_MOCK_LATENCY_MS", "150")) / 1000.0
|
||||
|
||||
|
||||
async def _mock_async_handler_get(self, url, params=None, headers=None, follow_redirects=None):
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ import os
|
|||
import traceback
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.types.utils import StandardLoggingPayload
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -27,6 +29,8 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
httpxSpecialProvider,
|
||||
)
|
||||
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class GcsPubSubLogger(CustomBatchLogger):
|
||||
def __init__(
|
||||
|
|
@ -152,7 +156,7 @@ class GcsPubSubLogger(CustomBatchLogger):
|
|||
finally:
|
||||
self.log_queue.clear()
|
||||
|
||||
async def publish_message(self, message: SpendLogsPayload | StandardLoggingPayload) -> dict[str, Any] | None:
|
||||
async def publish_message(self, message: SpendLogsPayload | StandardLoggingPayload) -> object:
|
||||
"""
|
||||
Publish message to Google Cloud Pub/Sub using REST API
|
||||
|
||||
|
|
@ -188,7 +192,7 @@ class GcsPubSubLogger(CustomBatchLogger):
|
|||
raise Exception(f"Failed to publish message: {response.text}")
|
||||
|
||||
verbose_logger.debug("Pub/Sub response: %s", response.text)
|
||||
return response.json()
|
||||
return _DECODED_JSON.validate_python(response.json())
|
||||
|
||||
except Exception as e:
|
||||
verbose_logger.error("Pub/Sub publish error: %s", str(e))
|
||||
|
|
|
|||
|
|
@ -332,7 +332,7 @@ def caller_session_id(kwargs: Mapping[str, object], trace: TraceControls) -> str
|
|||
return (None if echoes_trace_id else explicit) or trace.session_id or None
|
||||
|
||||
|
||||
def time_to_first_chunk_seconds(kwargs: Mapping[str, Any]) -> float | None:
|
||||
def time_to_first_chunk_seconds(kwargs: Mapping[str, object]) -> float | None:
|
||||
"""Seconds from the upstream request being issued (``api_call_start_time``)
|
||||
to the first streamed chunk (``completion_start_time``); ``None`` for
|
||||
non-streaming calls, where ``completion_start_time`` is backfilled with the
|
||||
|
|
@ -340,8 +340,8 @@ def time_to_first_chunk_seconds(kwargs: Mapping[str, Any]) -> float | None:
|
|||
optional_params: Final = cast(Mapping[str, object], kwargs.get("optional_params") or {})
|
||||
if not optional_params.get("stream"):
|
||||
return None
|
||||
api_call_start: Final = to_seconds(kwargs.get("api_call_start_time"))
|
||||
completion_start: Final = to_seconds(kwargs.get("completion_start_time"))
|
||||
api_call_start: Final = _epoch_seconds(kwargs.get("api_call_start_time"))
|
||||
completion_start: Final = _epoch_seconds(kwargs.get("completion_start_time"))
|
||||
if api_call_start is None or completion_start is None:
|
||||
return None
|
||||
return completion_start - api_call_start
|
||||
|
|
|
|||
|
|
@ -173,6 +173,7 @@ class _MetricKwargs(TypedDict, total=False):
|
|||
response_cost: ReadOnly[float | None]
|
||||
completion_start_time: ReadOnly[datetime | float | str | None]
|
||||
api_call_start_time: ReadOnly[datetime | float | str | None]
|
||||
end_time: ReadOnly[datetime | float | str | None]
|
||||
|
||||
|
||||
def resolve_error_type(kwargs: Mapping[str, Any]) -> str:
|
||||
|
|
@ -414,15 +415,12 @@ class GenAIMetricRecorder:
|
|||
if generation_time > 0:
|
||||
self._metrics.time_per_output_token.record(generation_time / completion_tokens, attributes=common_attrs)
|
||||
|
||||
def _record_response_duration(self, kwargs: Mapping[str, Any], end_time: datetime, common_attrs: dict) -> None:
|
||||
def _record_response_duration(self, kwargs: _MetricKwargs, end_time: datetime, common_attrs: dict) -> None:
|
||||
api_call_start_time: Final = kwargs.get("api_call_start_time")
|
||||
if api_call_start_time is None:
|
||||
return
|
||||
_end_time = kwargs.get("end_time") or end_time
|
||||
if _end_time is None:
|
||||
_end_time = datetime.now()
|
||||
api_call_start: Final = to_seconds(api_call_start_time)
|
||||
end_ts: Final = to_seconds(_end_time)
|
||||
end_ts: Final = to_seconds(kwargs.get("end_time") or end_time)
|
||||
if api_call_start is None or end_ts is None:
|
||||
return
|
||||
duration: Final = end_ts - api_call_start
|
||||
|
|
|
|||
|
|
@ -631,11 +631,14 @@ def preserve_upstream_non_openai_attributes(
|
|||
for key in candidate_keys
|
||||
)
|
||||
if all(value is None or isinstance(value, (str, int, float, bool)) for _, value in candidate_values):
|
||||
for key, value in candidate_values:
|
||||
setattr(model_response, key, value)
|
||||
_assign_attributes(model_response, candidate_values)
|
||||
return
|
||||
for key, value in original_chunk.model_dump(include=frozenset(candidate_keys)).items():
|
||||
setattr(model_response, key, value)
|
||||
_assign_attributes(model_response, original_chunk.model_dump(include=frozenset(candidate_keys)).items())
|
||||
|
||||
|
||||
def _assign_attributes(target: object, values: Iterable[tuple[str, object]]) -> None:
|
||||
for key, value in values:
|
||||
setattr(target, key, value)
|
||||
|
||||
|
||||
def safe_deep_copy(data):
|
||||
|
|
@ -761,6 +764,10 @@ def filter_exceptions_from_params(data: object, max_depth: int = 20) -> Any:
|
|||
Filtered data structure with Exception and callable objects removed, or None if the
|
||||
entire input was an Exception or callable
|
||||
"""
|
||||
return _filter_exceptions_from_params(data, max_depth)
|
||||
|
||||
|
||||
def _filter_exceptions_from_params(data: object, max_depth: int) -> object:
|
||||
if max_depth <= 0:
|
||||
return data
|
||||
|
||||
|
|
@ -782,7 +789,7 @@ def filter_exceptions_from_params(data: object, max_depth: int = 20) -> Any:
|
|||
if isinstance(v, Exception) or (callable(v) and not isinstance(v, type)):
|
||||
continue
|
||||
try:
|
||||
filtered = filter_exceptions_from_params(v, max_depth - 1)
|
||||
filtered = _filter_exceptions_from_params(v, max_depth - 1)
|
||||
if filtered is not None:
|
||||
result[k] = filtered
|
||||
except Exception:
|
||||
|
|
@ -796,7 +803,7 @@ def filter_exceptions_from_params(data: object, max_depth: int = 20) -> Any:
|
|||
if isinstance(item, Exception) or (callable(item) and not isinstance(item, type)):
|
||||
continue
|
||||
try:
|
||||
filtered = filter_exceptions_from_params(item, max_depth - 1)
|
||||
filtered = _filter_exceptions_from_params(item, max_depth - 1)
|
||||
if filtered is not None:
|
||||
result_list.append(filtered)
|
||||
except Exception:
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ def parse_cost_discount_key(key: str) -> CostDiscountKey:
|
|||
|
||||
|
||||
def _literal_length(pattern: str) -> int:
|
||||
return sum(1 for token in _GLOB_TOKEN.findall(pattern) if len(token) == 1 and token not in "*?")
|
||||
return sum(1 for match in _GLOB_TOKEN.finditer(pattern) if len(match.group()) == 1 and match.group() not in "*?")
|
||||
|
||||
|
||||
def _model_names(model: str, provider_prefix: str) -> tuple[str, ...]:
|
||||
|
|
|
|||
|
|
@ -48,5 +48,5 @@ for attempt in range(_max_retries):
|
|||
# Last attempt, re-raise the exception
|
||||
raise
|
||||
# Exponential backoff with jitter to reduce collision probability
|
||||
delay = _retry_delay * (2**attempt) + random.uniform(0, 0.1)
|
||||
delay = _retry_delay * (2.0**attempt) + random.uniform(0, 0.1)
|
||||
time.sleep(delay)
|
||||
|
|
|
|||
|
|
@ -181,7 +181,7 @@ def _delete_nested_value_custom(
|
|||
|
||||
|
||||
def delete_nested_value(
|
||||
data: dict[str, Any],
|
||||
data: dict[str, object],
|
||||
path: str,
|
||||
depth: int = 0,
|
||||
max_depth: int = 20,
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ from litellm.litellm_core_utils.fallback_generalizations import (
|
|||
|
||||
FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations"
|
||||
_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]])
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
_BUNDLED_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True))
|
||||
_CLI_ENTRYPOINT_NAMES: Final = frozenset({"lite", "litellm-proxy"})
|
||||
|
||||
|
|
@ -313,7 +314,7 @@ def _classify_fetch_error(error: httpx.HTTPError | httpx.InvalidURL, url: str) -
|
|||
|
||||
def _classify_fetch_content(content: bytes, url: str, etag: str | None) -> _FetchAttemptOutcome:
|
||||
try:
|
||||
parsed: Final[object] = json.loads(content)
|
||||
parsed: Final = _DECODED_JSON.validate_python(json.loads(content))
|
||||
except ValueError as e:
|
||||
return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}")
|
||||
if not isinstance(parsed, dict):
|
||||
|
|
@ -382,7 +383,8 @@ def _classify_fetch_response(response: httpx.Response, url: str) -> _FetchAttemp
|
|||
)
|
||||
if response.is_error:
|
||||
return ModelCostMapReloadUnavailable(reason=f"HTTP {response.status_code} from {url}")
|
||||
return _classify_fetch_content(response.content, url, response.headers.get("etag"))
|
||||
etag_values: Final = response.headers.get_list("etag")
|
||||
return _classify_fetch_content(response.content, url, ", ".join(etag_values) if etag_values else None)
|
||||
|
||||
|
||||
def _next_retry_wait(
|
||||
|
|
|
|||
|
|
@ -551,6 +551,7 @@ def _resolve_mantle_region_for_cost(
|
|||
|
||||
_PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"})
|
||||
_LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
|
||||
_REDACTED_RESULT: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
def _provider_response_id(source: object) -> str | None:
|
||||
|
|
@ -1714,7 +1715,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
verbose_logger.exception(
|
||||
"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e
|
||||
)
|
||||
original_response = redact_message_input_output_from_logging(
|
||||
redact_message_input_output_from_logging(
|
||||
model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
|
||||
result=original_response,
|
||||
)
|
||||
|
|
@ -1941,10 +1942,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
self.model_call_details.pop("response_cost_failure_debug_information", None)
|
||||
|
||||
if cache_hit is None:
|
||||
cache_hit = self.model_call_details.get("cache_hit", False)
|
||||
resolved_cache_hit: Final = self.model_call_details.get("cache_hit", False) if cache_hit is None else cache_hit
|
||||
|
||||
if cache_hit is True:
|
||||
if resolved_cache_hit is True:
|
||||
return 0.0
|
||||
|
||||
if is_unbilled_non_inference_call(
|
||||
|
|
@ -1996,7 +1996,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
model_value: Final = litellm_model_name or self.model
|
||||
cost_cache_hit_value: Final = (
|
||||
self.model_call_details.get("cache_hit", False) if cache_hit is None else cache_hit
|
||||
self.model_call_details.get("cache_hit", False) if resolved_cache_hit is None else resolved_cache_hit
|
||||
)
|
||||
cost_cache_hit: Final[bool | None] = cast( # cast-ok: callback metadata is caller-provided
|
||||
bool | None, cost_cache_hit_value
|
||||
|
|
@ -2375,8 +2375,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
def carry_failed_attempt_usage(self) -> None:
|
||||
"""Move what a failed earlier attempt stashed off the retry sharing this logging object, so the retry logs its
|
||||
own usage while the request's failure log still bills every attempt."""
|
||||
usage: Final = self.model_call_details.pop("combined_usage_object", None)
|
||||
response_cost: Final = self.model_call_details.pop("response_cost", None)
|
||||
usage: Final = self._model_call_details_view.get("combined_usage_object")
|
||||
response_cost: Final = self._model_call_details_view.get("response_cost")
|
||||
self.model_call_details.pop("combined_usage_object", None)
|
||||
self.model_call_details.pop("response_cost", None)
|
||||
if isinstance(usage, Usage):
|
||||
self.model_call_details["failed_attempts_usage"] = (
|
||||
*self.model_call_details.get("failed_attempts_usage", ()),
|
||||
|
|
@ -3170,9 +3172,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
)
|
||||
|
||||
## REDACT MESSAGES ##
|
||||
result = redact_message_input_output_from_logging(
|
||||
model_call_details=call_details,
|
||||
result=result,
|
||||
result = _REDACTED_RESULT.validate_python(
|
||||
redact_message_input_output_from_logging(
|
||||
model_call_details=call_details,
|
||||
result=result,
|
||||
)
|
||||
)
|
||||
## LOGGING HOOK ##
|
||||
hook_callbacks: Final = tuple(dict.fromkeys((*callbacks, *delivery_callbacks)))
|
||||
|
|
@ -3438,7 +3442,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
print_verbose("S3Logger Logger: Got Stream Event - Completed Stream Response")
|
||||
s3Logger.log_event(
|
||||
kwargs=call_details,
|
||||
response_obj=call_details["complete_streaming_response"],
|
||||
response_obj=call_details_view["complete_streaming_response"],
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
print_verbose=print_verbose,
|
||||
|
|
@ -3754,9 +3758,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self._get_completed_success_callbacks(asynchronous=True) if not self.stream else callbacks
|
||||
)
|
||||
|
||||
result = redact_message_input_output_from_logging(
|
||||
model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
|
||||
result=result,
|
||||
result = _REDACTED_RESULT.validate_python(
|
||||
redact_message_input_output_from_logging(
|
||||
model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
|
||||
result=result,
|
||||
)
|
||||
)
|
||||
|
||||
## LOGGING HOOK ##
|
||||
|
|
@ -4094,9 +4100,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
result: object = None # result sent to all loggers, init this to None incase it's not created
|
||||
|
||||
result = redact_message_input_output_from_logging(
|
||||
model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
|
||||
result=result,
|
||||
result = _REDACTED_RESULT.validate_python(
|
||||
redact_message_input_output_from_logging(
|
||||
model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
|
||||
result=result,
|
||||
)
|
||||
)
|
||||
self.mark_logging_complete(event_type="sync_failure")
|
||||
for callback in callbacks:
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ if TYPE_CHECKING:
|
|||
|
||||
JSON_FENCE_RE: Final = re.compile(r"```(?:json)?\s*(.*?)\s*```", re.DOTALL | re.IGNORECASE)
|
||||
_OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
def default_router_provider() -> Router | None:
|
||||
|
|
@ -40,13 +41,13 @@ def parse_json_verdict(raw: str) -> dict[str, object]: # mutable-ok: plain pars
|
|||
text = fenced.group(1).strip() # rebind-ok: progressively narrowed to the JSON payload
|
||||
parsed: object
|
||||
try:
|
||||
parsed = json.loads(text)
|
||||
parsed = _DECODED_JSON.validate_python(json.loads(text))
|
||||
except json.JSONDecodeError:
|
||||
start: Final = text.find("{")
|
||||
end: Final = text.rfind("}")
|
||||
if start == -1 or end <= start:
|
||||
raise
|
||||
parsed = json.loads(text[start : end + 1])
|
||||
parsed = _DECODED_JSON.validate_python(json.loads(text[start : end + 1]))
|
||||
if not isinstance(parsed, dict):
|
||||
raise ValueError("judge response is not a JSON object")
|
||||
return {str(k): v for k, v in parsed.items()}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Mapping
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import litellm
|
||||
|
|
@ -17,6 +17,10 @@ else:
|
|||
_generic_api_logger_cache: Final[dict[str, GenericAPILogger]] = {}
|
||||
|
||||
|
||||
def _instance_attributes(target: object) -> Mapping[str, object]:
|
||||
return vars(target)
|
||||
|
||||
|
||||
class LoggingCallbackManager:
|
||||
"""
|
||||
A centralized class that allows easy add / remove callbacks for litellm.
|
||||
|
|
@ -329,7 +333,7 @@ class LoggingCallbackManager:
|
|||
key_parts: Final = [type(custom_logger).__name__]
|
||||
|
||||
# Add only fundamental type instance variables to the key
|
||||
for attr_name, attr_value in vars(custom_logger).items():
|
||||
for attr_name, attr_value in _instance_attributes(custom_logger).items():
|
||||
if not attr_name.startswith("_"): # Skip private attributes
|
||||
if isinstance(attr_value, (str, bool, int)):
|
||||
key_parts.append(f"{attr_name}={attr_value}")
|
||||
|
|
|
|||
|
|
@ -1527,7 +1527,7 @@ def _grammar_branches(node: Mapping[str, object]) -> int:
|
|||
if properties is None:
|
||||
return 0
|
||||
optional: Final = len(_optional_property_names(node))
|
||||
return 2**optional * (len(properties) - optional + 1)
|
||||
return (1 << optional) * (len(properties) - optional + 1)
|
||||
|
||||
|
||||
def _optional_property_names(node: Mapping[str, object]) -> frozenset[str]:
|
||||
|
|
|
|||
|
|
@ -3086,10 +3086,14 @@ def anthropic_messages_pt(
|
|||
def extract_between_tags(tag: str, string: str, strip: bool = False) -> list[str]:
|
||||
ext_list = re.findall(f"<{tag}>(.+?)</{tag}>", string, re.DOTALL)
|
||||
if strip:
|
||||
ext_list = [e.strip() for e in ext_list]
|
||||
ext_list = list(_stripped(ext_list))
|
||||
return ext_list
|
||||
|
||||
|
||||
def _stripped(values: Iterable[str]) -> Iterator[str]:
|
||||
return (value.strip() for value in values)
|
||||
|
||||
|
||||
def contains_tag(tag: str, string: str) -> bool:
|
||||
return bool(re.search(f"<{tag}>(.+?)</{tag}>", string, re.DOTALL))
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import ItemsView, Mapping, Sequence
|
||||
from collections.abc import Set as AbstractSet
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Final
|
||||
|
|
@ -30,6 +30,10 @@ _DEFAULT_SENSITIVE_PATTERNS: Final = frozenset(
|
|||
)
|
||||
|
||||
|
||||
def _object_items(data: Mapping[str, object]) -> ItemsView[str, object]:
|
||||
return data.items()
|
||||
|
||||
|
||||
class SensitiveDataMasker:
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -127,7 +131,7 @@ class SensitiveDataMasker:
|
|||
return data
|
||||
|
||||
masked_data: Final[dict[str, object]] = {}
|
||||
for k, v in data.items():
|
||||
for k, v in _object_items(data):
|
||||
try:
|
||||
key_is_sensitive = self.is_sensitive_key(k, excluded_keys)
|
||||
if isinstance(v, Mapping):
|
||||
|
|
|
|||
|
|
@ -108,8 +108,7 @@ def _prepend_upstream_choice_extras(
|
|||
choice.__pydantic_fields_set__.update(upstream_extras)
|
||||
|
||||
|
||||
def _holds_only_scalars(model: BaseModel) -> bool:
|
||||
fields: Final[Iterator[tuple[str, object]]] = iter(model)
|
||||
def _holds_only_scalars(fields: Iterable[tuple[str, object]]) -> bool:
|
||||
return all(value is None or isinstance(value, (str, int, float, bool)) for _, value in fields)
|
||||
|
||||
|
||||
|
|
@ -940,8 +939,8 @@ class CustomStreamWrapper:
|
|||
or ("function_call" in completion_obj and completion_obj["function_call"] is not None)
|
||||
or (
|
||||
"tool_calls" in model_response.choices[0].delta
|
||||
and model_response.choices[0].delta["tool_calls"] is not None
|
||||
and len(model_response.choices[0].delta["tool_calls"]) > 0
|
||||
and model_response.choices[0].delta.tool_calls is not None
|
||||
and len(model_response.choices[0].delta.tool_calls) > 0
|
||||
)
|
||||
or (
|
||||
"function_call" in model_response.choices[0].delta
|
||||
|
|
|
|||
|
|
@ -132,7 +132,7 @@ class AnthropicBatchesHandler:
|
|||
max_retries: int | None,
|
||||
logging_obj: LiteLLMLoggingObj | None = None,
|
||||
litellm_params: dict | None = None, # mutable-ok: handed straight to validate_environment
|
||||
) -> LiteLLMBatch | Coroutine[Any, Any, LiteLLMBatch]:
|
||||
) -> LiteLLMBatch | Coroutine[object, object, LiteLLMBatch]:
|
||||
"""
|
||||
Retrieve a batch from Anthropic.
|
||||
|
||||
|
|
|
|||
|
|
@ -1629,14 +1629,13 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
if not line.startswith("data:"):
|
||||
return line
|
||||
try:
|
||||
data: Final[str | int | float | bool | None | Sequence[object] | Mapping[str, object]] = json.loads(
|
||||
line[len("data:") :].strip()
|
||||
)
|
||||
data: Final = _DECODED_SSE_DATA.validate_python(json.loads(line[len("data:") :].strip()))
|
||||
except json.JSONDecodeError:
|
||||
return line
|
||||
if not isinstance(data, dict):
|
||||
event: Final = as_str_mapping(data)
|
||||
if not isinstance(event, dict):
|
||||
return line
|
||||
rewritten: Final = _rewritten_event(_as_str_mapping(data), rewrite_event)
|
||||
rewritten: Final = _rewritten_event(event, rewrite_event)
|
||||
return line if rewritten is data else "data: " + json.dumps(rewritten)
|
||||
|
||||
def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None:
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ from dataclasses import dataclass
|
|||
from functools import reduce
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import Annotated, Any, Final, Literal, TypeVar, cast
|
||||
from typing import TYPE_CHECKING, Annotated, Final, Literal, TypeVar, cast
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, StrictInt, TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
|
@ -32,6 +32,9 @@ from ..constants import (
|
|||
from ..errors import AnthropicContextManagementError
|
||||
from ..placeholders import build_cleared_tool_result_content
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.types.llms.openai import ChatCompletionToolParam
|
||||
|
||||
T = TypeVar("T")
|
||||
# api.anthropic.com, measured 2026-10-19: text results of 1 to 46 bytes are never cleared or counted toward keep
|
||||
_TINY_RESULT_BYTES: Final = 46
|
||||
|
|
@ -216,7 +219,7 @@ def _trigger_met(
|
|||
current_tokens: Final = litellm.token_counter(
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=cast(Any, tools),
|
||||
tools=cast("list[ChatCompletionToolParam] | None", tools),
|
||||
)
|
||||
verbose_logger.debug("context_management polyfill: current_tokens: %s", current_tokens)
|
||||
verbose_logger.debug("context_management polyfill: threshold: %s", threshold)
|
||||
|
|
@ -366,9 +369,13 @@ def apply_clear_tool_uses_20250919(
|
|||
tokens_before: Final = (
|
||||
counted_tokens
|
||||
if counted_tokens is not None
|
||||
else litellm.token_counter(model=model, messages=messages, tools=cast(Any, tools))
|
||||
else litellm.token_counter(
|
||||
model=model, messages=messages, tools=cast("list[ChatCompletionToolParam] | None", tools)
|
||||
)
|
||||
)
|
||||
tokens_after: Final = litellm.token_counter(
|
||||
model=model, messages=final_messages, tools=cast("list[ChatCompletionToolParam] | None", tools)
|
||||
)
|
||||
tokens_after: Final = litellm.token_counter(model=model, messages=final_messages, tools=cast(Any, tools))
|
||||
cleared_input_tokens: Final = max(tokens_before - tokens_after, 0)
|
||||
|
||||
return final_messages, {
|
||||
|
|
|
|||
|
|
@ -1,12 +1,10 @@
|
|||
"""Placeholder content for cleared ``tool_result`` blocks (string or block list)."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from .constants import CLEARED_TOOL_RESULT_PLACEHOLDER
|
||||
|
||||
|
||||
def build_cleared_tool_result_content(
|
||||
original_content: Any,
|
||||
original_content: object,
|
||||
) -> str | list[dict]:
|
||||
"""Return a string or single text block list, matching ``original_content`` shape."""
|
||||
if isinstance(original_content, list):
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ attach ``iterations`` to ``usage``.
|
|||
"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Final
|
||||
from typing import Final
|
||||
|
||||
from litellm.types.llms.anthropic import (
|
||||
AppliedEdit,
|
||||
|
|
@ -19,8 +19,8 @@ from .constants import COMPACT_EDIT_TYPE
|
|||
|
||||
@dataclass
|
||||
class PolyfillResult:
|
||||
messages: list[dict[str, Any]]
|
||||
system: str | list[dict[str, Any]] | None
|
||||
messages: list[dict[str, object]]
|
||||
system: str | list[dict[str, object]] | None
|
||||
applied_edits: list[AppliedEdit] = field(default_factory=list)
|
||||
compaction_block: CompactionBlock | None = None
|
||||
iterations_usage: list[UsageIteration] | None = None
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ the LLM doesn't make a tool call, and we need to return a stream to the user.
|
|||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Final, cast
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final, cast
|
||||
|
||||
from litellm.llms.anthropic.pass_through.messages.utils import anthropic_message_delta
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import (
|
||||
|
|
@ -136,8 +136,7 @@ class FakeAnthropicMessagesStreamIterator:
|
|||
"""Convert the non-streaming response to streaming chunks"""
|
||||
chunks: Final = []
|
||||
|
||||
# Cast response to dict for easier access
|
||||
response_dict: Final = cast(dict[str, Any], self.response)
|
||||
response_dict: Final[Mapping[str, object]] = self.response
|
||||
|
||||
# 1. message_start event
|
||||
usage: Final = self.response.get("usage")
|
||||
|
|
@ -160,9 +159,8 @@ class FakeAnthropicMessagesStreamIterator:
|
|||
chunks.append(f"event: message_start\ndata: {json.dumps(message_start)}\n\n".encode())
|
||||
|
||||
# 2-4. For each content block, send start/delta/stop events
|
||||
content_blocks: Final = response_dict.get("content", [])
|
||||
for index, block in enumerate(content_blocks):
|
||||
block_dict = cast(dict[str, Any], block)
|
||||
content_blocks: Final = cast(Sequence[Mapping[str, object]], response_dict.get("content", []))
|
||||
for index, block_dict in enumerate(content_blocks):
|
||||
chunks.extend(self._create_content_block_chunks(block_dict, index))
|
||||
|
||||
# 5. message_delta event (with final usage and stop_reason)
|
||||
|
|
|
|||
|
|
@ -242,7 +242,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
@staticmethod
|
||||
def _summary_part_text(part: object) -> str:
|
||||
if isinstance(part, Mapping):
|
||||
mapping: Final = cast(Mapping[str, Any], part) # cast-ok: summary parts are untyped provider json
|
||||
mapping: Final = cast(Mapping[str, object], part) # cast-ok: summary parts are untyped provider json
|
||||
return str(mapping.get("text") or "")
|
||||
return str(getattr(part, "text", None) or "")
|
||||
|
||||
|
|
@ -471,7 +471,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
"""Convert Anthropic tool definitions to Responses API function tools."""
|
||||
result: Final[list[dict[str, object]]] = []
|
||||
for tool in tools:
|
||||
tool_dict = cast(dict[str, Any], tool)
|
||||
tool_dict = cast(Mapping[str, object], tool)
|
||||
tool_type = tool_dict.get("type", "")
|
||||
tool_name = tool_dict.get("name", "")
|
||||
# web_search tool
|
||||
|
|
@ -674,7 +674,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
output_config = anthropic_request.get("output_config")
|
||||
reasoning: Final = self.translate_thinking_to_reasoning(
|
||||
thinking,
|
||||
output_config=cast(dict[str, Any] | None, output_config),
|
||||
output_config=cast(dict[str, object] | None, output_config),
|
||||
auto_summary_level=auto_summary_level,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
from collections.abc import Coroutine
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from openai import AsyncAzureOpenAI, AzureOpenAI
|
||||
from pydantic import BaseModel
|
||||
|
|
@ -38,7 +38,7 @@ class AzureAudioTranscription(AzureChatCompletion):
|
|||
atranscription: bool = False,
|
||||
litellm_params: dict | None = None,
|
||||
custom_llm_provider: str = "azure",
|
||||
) -> TranscriptionResponse | Coroutine[Any, Any, TranscriptionResponse]:
|
||||
) -> TranscriptionResponse | Coroutine[object, object, TranscriptionResponse]:
|
||||
data: Final = {"model": model, "file": audio_file, **optional_params}
|
||||
|
||||
if atranscription is True:
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ from openai import (
|
|||
BadRequestError,
|
||||
OpenAI,
|
||||
)
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.constants import AZURE_OPERATION_POLLING_TIMEOUT, DEFAULT_MAX_RETRIES
|
||||
|
|
@ -59,6 +59,8 @@ from .image_generation import (
|
|||
)
|
||||
from .image_generation.http_utils import azure_deployment_image_generation_json_body
|
||||
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class AzureOpenAIAssistantsAPIConfig:
|
||||
"""
|
||||
|
|
@ -961,7 +963,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
|
|||
body=error_data,
|
||||
)
|
||||
|
||||
result: Final = response.json()["result"]
|
||||
result: Final = _DECODED_JSON.validate_python(response.json()["result"])
|
||||
return httpx.Response(
|
||||
status_code=200,
|
||||
headers=response.headers,
|
||||
|
|
@ -1078,7 +1080,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
|
|||
body=error_data,
|
||||
)
|
||||
|
||||
result: Final = response.json()["result"]
|
||||
result: Final = _DECODED_JSON.validate_python(response.json()["result"])
|
||||
return httpx.Response(
|
||||
status_code=200,
|
||||
headers=response.headers,
|
||||
|
|
|
|||
|
|
@ -94,7 +94,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
def get_stripped_model_name(self, model: str) -> str:
|
||||
return model.replace("responses/", "").replace("o_series/", "").replace("azure_ai/", "")
|
||||
|
||||
def _handle_reasoning_item(self, item: dict[str, Any]) -> dict[str, Any]:
|
||||
def _handle_reasoning_item(self, item: dict[str, Any]) -> Mapping[str, object]:
|
||||
"""
|
||||
Handle reasoning items to filter out the status field.
|
||||
Issue: https://github.com/BerriAI/litellm/issues/13484
|
||||
|
|
|
|||
|
|
@ -396,7 +396,7 @@ class AzureAIStudioConfig(OpenAIConfig):
|
|||
return data
|
||||
|
||||
def _drop_tool_level_extra_fields(self, request_data: dict, error_text: str) -> dict:
|
||||
fields_to_drop: Final = set(re.findall(r"tools\[\d+\]\.([\w-]+)", error_text))
|
||||
fields_to_drop: Final[Collection[str]] = set(re.findall(r"tools\[\d+\]\.([\w-]+)", error_text))
|
||||
tools: Final = request_data.get("tools")
|
||||
if fields_to_drop and isinstance(tools, list):
|
||||
for tool in tools:
|
||||
|
|
|
|||
|
|
@ -428,7 +428,7 @@ class BaseConfig(ABC):
|
|||
streaming_response: Union[Iterator[str], AsyncIterator[str], "ModelResponse"],
|
||||
sync_stream: bool,
|
||||
json_mode: bool | None = False,
|
||||
) -> Any:
|
||||
) -> object:
|
||||
pass
|
||||
|
||||
def get_model_response_iterator_for_request(
|
||||
|
|
@ -513,7 +513,7 @@ class BaseConfig(ABC):
|
|||
def apply_assembled_streaming_response_metadata(
|
||||
self,
|
||||
response: "ModelResponse",
|
||||
chunks: list[Any],
|
||||
chunks: list[object],
|
||||
) -> None:
|
||||
"""Hook for providers to merge chunk metadata into assembled streaming responses."""
|
||||
return
|
||||
|
|
|
|||
|
|
@ -121,7 +121,7 @@ class BaseTranslation(ABC):
|
|||
|
||||
@staticmethod
|
||||
def merge_user_api_key_metadata_into_request(
|
||||
request_data: dict[str, Any], # mutable-ok: proxy hooks share and mutate the request payload dict in place
|
||||
request_data: dict[str, object], # mutable-ok: proxy hooks share and mutate the request payload dict in place
|
||||
user_api_key_dict: Optional["UserAPIKeyAuth"],
|
||||
) -> None:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -141,7 +141,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
|
|||
self,
|
||||
llm_router: Router,
|
||||
model: str,
|
||||
request_data: dict[str, Any],
|
||||
request_data: dict[str, object],
|
||||
litellm_parent_otel_span: Span,
|
||||
) -> ResourceObjectType:
|
||||
"""
|
||||
|
|
@ -399,7 +399,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
|
|||
async def create_resource_for_each_model(
|
||||
self,
|
||||
llm_router: Router,
|
||||
request_data: dict[str, Any],
|
||||
request_data: dict[str, object],
|
||||
target_model_names_list: list[str],
|
||||
litellm_parent_otel_span: Span,
|
||||
) -> list[ResourceObjectType]:
|
||||
|
|
@ -550,7 +550,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
|
|||
limit: int | None = None,
|
||||
after: str | None = None,
|
||||
additional_filters: Mapping[str, object] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
) -> dict[str, object]:
|
||||
"""
|
||||
List resources created by a user.
|
||||
|
||||
|
|
|
|||
|
|
@ -152,7 +152,7 @@ class BaseVectorStoreConfig:
|
|||
api_base: str,
|
||||
litellm_logging_obj: LiteLLMLoggingObj,
|
||||
litellm_params: dict,
|
||||
extra_body: dict[str, Any] | None = None,
|
||||
extra_body: dict[str, object] | None = None,
|
||||
) -> tuple[str, dict]:
|
||||
pass
|
||||
|
||||
|
|
@ -164,7 +164,7 @@ class BaseVectorStoreConfig:
|
|||
api_base: str,
|
||||
litellm_logging_obj: LiteLLMLoggingObj,
|
||||
litellm_params: dict,
|
||||
extra_body: dict[str, Any] | None = None,
|
||||
extra_body: dict[str, object] | None = None,
|
||||
) -> tuple[str, dict]:
|
||||
"""
|
||||
Optional async version of transform_search_vector_store_request.
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Final, Optional, Union
|
|||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm._uuid import uuid
|
||||
|
|
@ -50,6 +51,8 @@ else:
|
|||
HTTPHandler = Any
|
||||
AsyncHTTPHandler = Any
|
||||
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
||||
def __init__(self, **kwargs):
|
||||
|
|
@ -313,7 +316,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
|||
return None
|
||||
|
||||
try:
|
||||
data: Final[object] = json.loads(json_str)
|
||||
data: Final = _DECODED_JSON.validate_python(json.loads(json_str))
|
||||
# Skip non-dict data (some lines contain JSON strings)
|
||||
return data if isinstance(data, dict) else None
|
||||
except json.JSONDecodeError:
|
||||
|
|
@ -481,7 +484,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
|||
if "application/json" in content_type:
|
||||
# Direct JSON response
|
||||
verbose_logger.debug("Parsing JSON response")
|
||||
response_json: Final[object] = raw_response.json()
|
||||
response_json: Final = _DECODED_JSON.validate_python(raw_response.json())
|
||||
verbose_logger.debug("Response JSON: %s", response_json)
|
||||
return self._parse_json_response(response_json)
|
||||
else:
|
||||
|
|
@ -565,7 +568,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
|||
continue
|
||||
|
||||
try:
|
||||
data_obj = json.loads(json_str)
|
||||
data_obj = _DECODED_JSON.validate_python(json.loads(json_str))
|
||||
if not isinstance(data_obj, dict):
|
||||
continue
|
||||
|
||||
|
|
@ -705,7 +708,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
|||
)
|
||||
try:
|
||||
body: Final = response.read()
|
||||
response_json: Final[object] = json.loads(body)
|
||||
response_json: Final = _DECODED_JSON.validate_python(json.loads(body))
|
||||
except (json.JSONDecodeError, Exception) as e:
|
||||
raise BedrockError(
|
||||
status_code=response.status_code,
|
||||
|
|
@ -787,7 +790,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
|||
continue
|
||||
|
||||
try:
|
||||
data_obj = json.loads(json_str)
|
||||
data_obj = _DECODED_JSON.validate_python(json.loads(json_str))
|
||||
if not isinstance(data_obj, dict):
|
||||
continue
|
||||
|
||||
|
|
@ -927,7 +930,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
|||
)
|
||||
try:
|
||||
body: Final = await response.aread()
|
||||
response_json: Final[object] = json.loads(body)
|
||||
response_json: Final = _DECODED_JSON.validate_python(json.loads(body))
|
||||
except (json.JSONDecodeError, Exception) as e:
|
||||
raise BedrockError(
|
||||
status_code=response.status_code,
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from litellm.types.utils import ModelResponse
|
|||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.llms.anthropic.chat.handler import ModelResponseIterator
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -134,7 +135,7 @@ class AmazonMantleConfig(AmazonAnthropicClaudeConfig):
|
|||
streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse,
|
||||
sync_stream: bool,
|
||||
json_mode: bool | None = False,
|
||||
) -> Any:
|
||||
) -> "ModelResponseIterator":
|
||||
return self.get_model_response_iterator_for_request(
|
||||
streaming_response=streaming_response, sync_stream=sync_stream, json_mode=json_mode, optional_params={}
|
||||
)
|
||||
|
|
@ -145,7 +146,7 @@ class AmazonMantleConfig(AmazonAnthropicClaudeConfig):
|
|||
sync_stream: bool,
|
||||
json_mode: bool | None,
|
||||
optional_params: Mapping[str, object],
|
||||
) -> object:
|
||||
) -> "ModelResponseIterator":
|
||||
from litellm.llms.anthropic.chat.handler import ModelResponseIterator
|
||||
|
||||
return ModelResponseIterator(
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import asyncio
|
||||
from collections.abc import Coroutine, Mapping
|
||||
from typing import Any, Final
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
||||
|
|
@ -139,7 +139,7 @@ class BedrockFilesHandler(BaseAWSLLM):
|
|||
optional_params: dict,
|
||||
timeout: float | httpx.Timeout,
|
||||
max_retries: int | None,
|
||||
) -> HttpxBinaryResponseContent | Coroutine[Any, Any, HttpxBinaryResponseContent]:
|
||||
) -> HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent]:
|
||||
"""
|
||||
Download file content from S3 bucket for Bedrock files.
|
||||
Supports both sync and async operations.
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ Transformation logic for Amazon Titan Image Generation.
|
|||
"""
|
||||
|
||||
import types
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
from openai.types.image import Image
|
||||
|
|
@ -35,16 +36,17 @@ class AmazonTitanImageGenerationConfig:
|
|||
width: int | None = None,
|
||||
height: int | None = None,
|
||||
) -> None:
|
||||
locals_: Final = locals().copy()
|
||||
locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy())
|
||||
for key, value in locals_.items():
|
||||
if key != "self" and value is not None:
|
||||
setattr(self.__class__, key, value)
|
||||
|
||||
@classmethod
|
||||
def get_config(cls):
|
||||
class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__)
|
||||
return {
|
||||
k: v
|
||||
for k, v in cls.__dict__.items()
|
||||
for k, v in class_attributes.items()
|
||||
if not k.startswith("__")
|
||||
and not isinstance(
|
||||
v,
|
||||
|
|
|
|||
|
|
@ -544,7 +544,7 @@ class _StreamFrame(TypedDict):
|
|||
|
||||
|
||||
def _unpack_uint32(buffer: bytes) -> int:
|
||||
return struct.unpack("!I", buffer)[0]
|
||||
return int.from_bytes(buffer, "big")
|
||||
|
||||
|
||||
def _collect_stream_delta_text_holders(delta: object) -> list[_DeltaHolder]:
|
||||
|
|
@ -1255,7 +1255,9 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
|
|||
headers_bytes = frame_raw[12 : 12 + orig_hdrs_len]
|
||||
|
||||
try:
|
||||
payload_dict = json.loads(frame_raw[12 + orig_hdrs_len : orig_total - 4])
|
||||
payload_dict = _BODY_BY_REFERENCE.validate_python(
|
||||
json.loads(frame_raw[12 + orig_hdrs_len : orig_total - 4])
|
||||
)
|
||||
for local_idx, (_, container, key) in enumerate(
|
||||
_collect_stream_delta_text_holders(payload_dict.get("delta"))
|
||||
):
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ class _BflPolledResult(TypedDict):
|
|||
|
||||
|
||||
_BFL_POLLED_RESULT_ADAPTER: Final = TypeAdapter(_BflPolledResult)
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
_BFL_REQUEST_PARAMS: Final = (
|
||||
|
|
@ -317,7 +318,7 @@ class BlackForestLabsImageEditConfig(BaseImageEditConfig):
|
|||
The response contains: {"status": "Ready", "result": {"sample": "https://..."}}
|
||||
"""
|
||||
try:
|
||||
raw_response_data: Final[object] = raw_response.json()
|
||||
raw_response_data: Final = _DECODED_JSON.validate_python(raw_response.json())
|
||||
except Exception as e:
|
||||
raise BlackForestLabsError(
|
||||
status_code=raw_response.status_code,
|
||||
|
|
|
|||
|
|
@ -72,7 +72,7 @@ class ChatGPTConfig(OpenAIConfig):
|
|||
default_headers: Final = get_chatgpt_default_headers(api_key or "", account_id, session_id)
|
||||
return {**default_headers, **validated_headers}
|
||||
|
||||
def post_stream_processing(self, stream: Any) -> Any:
|
||||
def post_stream_processing(self, stream: Any) -> ChatGPTToolCallNormalizer:
|
||||
return ChatGPTToolCallNormalizer(stream)
|
||||
|
||||
def map_openai_params(
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ Documentation: [CometAPI Documentation Link]
|
|||
"""
|
||||
|
||||
from collections.abc import AsyncIterator, Iterator
|
||||
from typing import Any, Final
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
||||
|
|
@ -138,7 +138,7 @@ class CometAPIConfig(OpenAIGPTConfig):
|
|||
streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse,
|
||||
sync_stream: bool,
|
||||
json_mode: bool | None = False,
|
||||
) -> Any:
|
||||
) -> "CometAPIChatCompletionStreamingHandler":
|
||||
"""
|
||||
Get model response iterator for streaming responses
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -2,10 +2,11 @@
|
|||
CompactifAI chat completion transformation
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
from collections.abc import Iterable, Mapping, Sequence, Sized
|
||||
from typing import TYPE_CHECKING, Any, Final, Protocol
|
||||
|
||||
import httpx
|
||||
from pydantic import InstanceOf, TypeAdapter
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
|
@ -24,6 +25,21 @@ if TYPE_CHECKING:
|
|||
else:
|
||||
LiteLLMLoggingObj = Any
|
||||
|
||||
_RESPONSE_BODY: Final = TypeAdapter(InstanceOf[dict[str, object]])
|
||||
_JSON_VALUES: Final = TypeAdapter(Iterable[object])
|
||||
_JSON_OBJECT: Final = TypeAdapter(InstanceOf[Mapping[str, object]])
|
||||
_MESSAGE: Final = TypeAdapter(InstanceOf[dict[str, object]])
|
||||
_SIZED: Final = TypeAdapter(InstanceOf[Sized])
|
||||
_TOOL_CALLS: Final = TypeAdapter(InstanceOf[Sequence[object]])
|
||||
|
||||
|
||||
class _JsonResponse(Protocol):
|
||||
def json(self) -> object: ...
|
||||
|
||||
|
||||
def _json_body(response: _JsonResponse) -> object:
|
||||
return response.json()
|
||||
|
||||
|
||||
class CompactifAIResponseFields(TypedDict, total=False):
|
||||
"""The chat completion fields of a CompactifAI response body."""
|
||||
|
|
@ -82,22 +98,23 @@ class CompactifAIChatConfig(OpenAIGPTConfig):
|
|||
)
|
||||
|
||||
## RESPONSE OBJECT
|
||||
response_json: Final = raw_response.json()
|
||||
response_json: Final = _RESPONSE_BODY.validate_python(_json_body(raw_response))
|
||||
|
||||
# Handle JSON mode if needed
|
||||
if json_mode:
|
||||
for choice in response_json["choices"]:
|
||||
message = choice.get("message")
|
||||
if message and message.get("tool_calls"):
|
||||
for choice in _JSON_VALUES.validate_python(response_json["choices"]):
|
||||
message = _JSON_OBJECT.validate_python(choice).get("message")
|
||||
message_fields = _MESSAGE.validate_python(message) if message else None
|
||||
if message_fields is not None and message_fields.get("tool_calls"):
|
||||
# Convert tool calls to content for JSON mode
|
||||
tool_calls = message.get("tool_calls", [])
|
||||
if len(tool_calls) == 1:
|
||||
message["content"] = tool_calls[0]["function"].get("arguments", "")
|
||||
message["tool_calls"] = None
|
||||
tool_calls = message_fields.get("tool_calls", [])
|
||||
if len(_SIZED.validate_python(tool_calls)) == 1:
|
||||
first_call = _JSON_OBJECT.validate_python(_TOOL_CALLS.validate_python(tool_calls)[0])
|
||||
function = _JSON_OBJECT.validate_python(first_call["function"])
|
||||
message_fields["content"] = function.get("arguments", "")
|
||||
message_fields["tool_calls"] = None
|
||||
|
||||
response_fields: Final[CompactifAIResponseFields] = response_json
|
||||
|
||||
returned_response: Final = ModelResponse(**response_fields)
|
||||
returned_response: Final = ModelResponse(**response_json)
|
||||
|
||||
# Set model name with provider prefix
|
||||
returned_response.model = f"compactifai/{model}"
|
||||
|
|
|
|||
|
|
@ -407,6 +407,27 @@ def _has_pre_call_deployment_hook(logging_obj: LiteLLMLoggingObj) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def _run_responses_deployment_hook(
|
||||
*,
|
||||
model: str,
|
||||
request_input: str | ResponseInputParam,
|
||||
custom_llm_provider: str,
|
||||
optional_params: Mapping[str, object],
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
) -> Mapping[str, object]:
|
||||
return run_async_function(
|
||||
async_pre_call_deployment_hook,
|
||||
{
|
||||
**dict(litellm_params),
|
||||
**optional_params,
|
||||
"model": model,
|
||||
"input": request_input,
|
||||
"custom_llm_provider": custom_llm_provider,
|
||||
},
|
||||
CallTypes.responses.value,
|
||||
)
|
||||
|
||||
|
||||
def _mask_presigned_request_headers(transformed_request: bytes | str | dict) -> bytes | str | dict:
|
||||
"""A pre-signed request carries its auth inside its own ``headers`` key, which
|
||||
logging treats as request body (only the top-level headers channel gets masked),
|
||||
|
|
@ -2637,16 +2658,12 @@ class BaseLLMHTTPHandler:
|
|||
litellm_params,
|
||||
)
|
||||
|
||||
modified_kwargs: Final = run_async_function(
|
||||
async_pre_call_deployment_hook,
|
||||
{
|
||||
**dict(litellm_params),
|
||||
**response_api_optional_request_params,
|
||||
"model": model,
|
||||
"input": input,
|
||||
"custom_llm_provider": custom_llm_provider,
|
||||
},
|
||||
CallTypes.responses.value,
|
||||
modified_kwargs: Final = _run_responses_deployment_hook(
|
||||
model=model,
|
||||
request_input=input,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
optional_params=response_api_optional_request_params,
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
if modified_kwargs is None:
|
||||
return (
|
||||
|
|
|
|||
|
|
@ -9,9 +9,11 @@ so the full proxy -> router -> OpenAI SDK -> httpx path is exercised.
|
|||
import json
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pre-built response templates
|
||||
|
|
@ -22,7 +24,7 @@ def _mock_id() -> str:
|
|||
return f"chatcmpl-mock-{uuid.uuid4().hex[:8]}"
|
||||
|
||||
|
||||
def _chat_completion_json(model: str) -> dict:
|
||||
def _chat_completion_json(model: object) -> dict:
|
||||
"""Return a minimal valid ChatCompletion object."""
|
||||
return {
|
||||
"id": _mock_id(),
|
||||
|
|
@ -55,6 +57,9 @@ _JSON_HEADERS: Final = {
|
|||
"content-type": "application/json",
|
||||
}
|
||||
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
_REQUEST_BODY: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
class MockOpenAITransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
|
||||
"""
|
||||
|
|
@ -64,13 +69,13 @@ class MockOpenAITransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
|
|||
"""
|
||||
|
||||
@staticmethod
|
||||
def _parse_request(request: httpx.Request) -> tuple[str, bool]:
|
||||
def _parse_request(request: httpx.Request) -> tuple[object, bool]:
|
||||
"""Extract model from the request body."""
|
||||
try:
|
||||
body: Final = json.loads(request.content)
|
||||
body: Final = _DECODED_JSON.validate_python(json.loads(request.content))
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return ("mock-model", False)
|
||||
model: Final = body.get("model", "mock-model")
|
||||
model: Final = _REQUEST_BODY.validate_python(body).get("model", "mock-model")
|
||||
return (model, False)
|
||||
|
||||
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ Translates from OpenAI's `/v1/embeddings` to Databricks' `/embeddings`
|
|||
"""
|
||||
|
||||
import types
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
|
||||
|
|
@ -16,16 +17,17 @@ class DatabricksEmbeddingConfig:
|
|||
)
|
||||
|
||||
def __init__(self, instruction: str | None = None) -> None:
|
||||
locals_: Final = locals().copy()
|
||||
locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy())
|
||||
for key, value in locals_.items():
|
||||
if key != "self" and value is not None:
|
||||
setattr(self.__class__, key, value)
|
||||
|
||||
@classmethod
|
||||
def get_config(cls):
|
||||
class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__)
|
||||
return {
|
||||
k: v
|
||||
for k, v in cls.__dict__.items()
|
||||
for k, v in class_attributes.items()
|
||||
if not k.startswith("__")
|
||||
and not isinstance(
|
||||
v,
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ import shlex
|
|||
import uuid
|
||||
from collections.abc import Awaitable, Callable, Coroutine, Iterator, Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final, TypeVar
|
||||
from typing import Final, TypeVar
|
||||
|
||||
from deepagents.backends.protocol import (
|
||||
DeleteResult,
|
||||
|
|
@ -171,7 +171,7 @@ class SandboxBackend(SandboxBackendProtocol): # pyright: ignore[reportUntypedBa
|
|||
|
||||
# -- sync bridge (deepagents only calls these outside the event loop) ---
|
||||
|
||||
def _sync(self, coro: Coroutine[Any, Any, T]) -> T:
|
||||
def _sync(self, coro: Coroutine[object, object, T]) -> T:
|
||||
try:
|
||||
running = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
|
|
|
|||
|
|
@ -11,11 +11,13 @@ from __future__ import annotations
|
|||
|
||||
import itertools
|
||||
import json
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from collections.abc import Iterable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Protocol
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.harness.options import DeepAgentsOptions
|
||||
from litellm.harness.types import (
|
||||
Capabilities,
|
||||
|
|
@ -41,6 +43,7 @@ DEEPAGENTS_DEFAULT_RECURSION_LIMIT: Final = 1000
|
|||
_MODEL_NODE: Final = "model"
|
||||
# Only these graph nodes produce new messages; middleware hooks may re-emit history.
|
||||
_EVENT_NODES: Final = frozenset({"model", "tools"})
|
||||
_TOOL_CALLS: Final = TypeAdapter(Iterable[Mapping[str, object]])
|
||||
|
||||
NORMALIZED_TO_NATIVE: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{
|
||||
|
|
@ -208,7 +211,7 @@ def update_events(
|
|||
def _message_events(message: object, skip_tools: frozenset[str]) -> tuple[Event, ...]:
|
||||
kind = getattr(message, "type", None)
|
||||
if kind == "ai":
|
||||
calls = getattr(message, "tool_calls", None) or ()
|
||||
calls: Final = _TOOL_CALLS.validate_python(getattr(message, "tool_calls", None) or ())
|
||||
return tuple(tool_call_event(call) for call in calls if call.get("name") not in skip_tools)
|
||||
if kind == "tool" and getattr(message, "name", None) not in skip_tools:
|
||||
status: Final[object] = getattr(message, "status", None)
|
||||
|
|
|
|||
|
|
@ -2,12 +2,12 @@
|
|||
Translates from OpenAI's `/v1/audio/transcriptions` to Deepgram's `/v1/listen`
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
from typing import Final
|
||||
from collections.abc import Container, Iterable, Mapping, Sequence, Sized
|
||||
from typing import Final, Protocol
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from httpx import Headers, Response
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
from pydantic import ConfigDict, InstanceOf, TypeAdapter
|
||||
|
||||
from litellm.litellm_core_utils.audio_utils.utils import process_audio_file
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
|
@ -26,6 +26,19 @@ from ..common_utils import DeepgramException
|
|||
|
||||
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
|
||||
_JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True))
|
||||
_RESPONSE_BODY: Final = TypeAdapter(InstanceOf[dict[str, object]], config=ConfigDict(hide_input_in_errors=True))
|
||||
_ITEMS: Final = TypeAdapter(InstanceOf[Sequence[object]], config=ConfigDict(hide_input_in_errors=True))
|
||||
_SIZED: Final = TypeAdapter(InstanceOf[Sized], config=ConfigDict(hide_input_in_errors=True))
|
||||
_CONTAINER: Final = TypeAdapter(InstanceOf[Container[object]], config=ConfigDict(hide_input_in_errors=True))
|
||||
_WORD_LIST: Final = TypeAdapter(InstanceOf[list[object]], config=ConfigDict(hide_input_in_errors=True))
|
||||
|
||||
|
||||
class _JsonResponse(Protocol):
|
||||
def json(self) -> object: ...
|
||||
|
||||
|
||||
def _json_body(response: _JsonResponse) -> object:
|
||||
return response.json()
|
||||
|
||||
|
||||
class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
||||
|
|
@ -81,16 +94,21 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
|||
Transforms the raw response from Deepgram to the TranscriptionResponse format
|
||||
"""
|
||||
try:
|
||||
response_json: Final = raw_response.json()
|
||||
response_json: Final = _RESPONSE_BODY.validate_python(_json_body(raw_response))
|
||||
|
||||
# Get the first alternative from the first channel
|
||||
first_channel: Final = response_json["results"]["channels"][0]
|
||||
first_alternative: Final = first_channel["alternatives"][0]
|
||||
results: Final = _JSON_OBJECT.validate_python(response_json["results"])
|
||||
first_channel: Final = _JSON_OBJECT.validate_python(_ITEMS.validate_python(results["channels"])[0])
|
||||
first_alternative: Final = _JSON_OBJECT.validate_python(
|
||||
_ITEMS.validate_python(first_channel["alternatives"])[0]
|
||||
)
|
||||
|
||||
# Detect if diarization is active by checking if words have 'speaker' field
|
||||
has_diarization = False
|
||||
if "words" in first_alternative and len(first_alternative["words"]) > 0:
|
||||
has_diarization = "speaker" in first_alternative["words"][0]
|
||||
if "words" in first_alternative and len(_SIZED.validate_python(first_alternative["words"])) > 0:
|
||||
has_diarization = "speaker" in _CONTAINER.validate_python(
|
||||
_ITEMS.validate_python(first_alternative["words"])[0]
|
||||
)
|
||||
|
||||
# Extract the transcript based on diarization mode
|
||||
if not has_diarization:
|
||||
|
|
@ -98,10 +116,10 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
|||
text = first_alternative["transcript"]
|
||||
elif "paragraphs" in first_alternative:
|
||||
# Diarization with paragraphs: use the pre-formatted diarized transcript
|
||||
text = first_alternative["paragraphs"]["transcript"]
|
||||
text = _JSON_OBJECT.validate_python(first_alternative["paragraphs"])["transcript"]
|
||||
else:
|
||||
# Diarization without paragraphs: reconstruct from words
|
||||
text = self._reconstruct_diarized_transcript(first_alternative["words"])
|
||||
text = self._reconstruct_diarized_transcript(_WORD_LIST.validate_python(first_alternative["words"]))
|
||||
|
||||
# Create TranscriptionResponse object
|
||||
response: Final = TranscriptionResponse(text=text)
|
||||
|
|
@ -110,10 +128,10 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
|||
response["task"] = "transcribe"
|
||||
|
||||
# Use detected_language if available, otherwise default to "en"
|
||||
detected_language: Final = _JSON_OBJECT.validate_python(first_channel).get("detected_language")
|
||||
detected_language: Final = first_channel.get("detected_language")
|
||||
response["language"] = detected_language if detected_language else "en"
|
||||
|
||||
response["duration"] = response_json["metadata"]["duration"]
|
||||
response["duration"] = _JSON_OBJECT.validate_python(response_json["metadata"])["duration"]
|
||||
|
||||
# Transform words to match OpenAI format
|
||||
if "words" in first_alternative:
|
||||
|
|
|
|||
|
|
@ -2,8 +2,10 @@
|
|||
DeepSeek Anthropic-compatible messages transformation config.
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from typing import Any, Final
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.llms.anthropic.pass_through.messages.transformation import (
|
||||
|
|
@ -12,6 +14,24 @@ from litellm.llms.anthropic.pass_through.messages.transformation import (
|
|||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
_TOOL_LIST: Final = TypeAdapter(list[object])
|
||||
_TOOL_FIELDS: Final = TypeAdapter(dict[object, object])
|
||||
|
||||
|
||||
def _custom_tool_without_type(tool: object) -> Mapping[object, object] | None:
|
||||
if not isinstance(tool, dict):
|
||||
return None
|
||||
fields: Final = _TOOL_FIELDS.validate_python(tool)
|
||||
if fields.get("type") == "custom":
|
||||
fields.pop("type", None)
|
||||
return fields
|
||||
return None
|
||||
|
||||
|
||||
def _sanitized_tool(tool: object) -> object:
|
||||
stripped: Final = _custom_tool_without_type(tool)
|
||||
return tool if stripped is None else stripped
|
||||
|
||||
|
||||
class DeepSeekAnthropicMessagesConfig(AnthropicMessagesConfig):
|
||||
"""
|
||||
|
|
@ -95,19 +115,11 @@ class DeepSeekAnthropicMessagesConfig(AnthropicMessagesConfig):
|
|||
return f"{base_url}/v1/messages"
|
||||
|
||||
@staticmethod
|
||||
def _sanitize_tools_for_deepseek(tools: Any) -> object:
|
||||
def _sanitize_tools_for_deepseek(tools: object) -> object:
|
||||
if not isinstance(tools, list):
|
||||
return tools
|
||||
|
||||
sanitized_tools: Final = []
|
||||
for tool in tools:
|
||||
if isinstance(tool, dict) and tool.get("type") == "custom":
|
||||
sanitized_tool = dict(tool)
|
||||
sanitized_tool.pop("type", None)
|
||||
sanitized_tools.append(sanitized_tool)
|
||||
else:
|
||||
sanitized_tools.append(tool)
|
||||
return sanitized_tools
|
||||
return [_sanitized_tool(tool) for tool in _TOOL_LIST.validate_python(tools)]
|
||||
|
||||
def transform_anthropic_messages_request(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import types
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
|
||||
|
|
@ -41,16 +42,17 @@ class PalmConfig:
|
|||
top_p: float | None = None,
|
||||
max_output_tokens: int | None = None,
|
||||
) -> None:
|
||||
locals_: Final = locals().copy()
|
||||
locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy())
|
||||
for key, value in locals_.items():
|
||||
if key != "self" and value is not None:
|
||||
setattr(self.__class__, key, value)
|
||||
|
||||
@classmethod
|
||||
def get_config(cls):
|
||||
class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__)
|
||||
return {
|
||||
k: v
|
||||
for k, v in cls.__dict__.items()
|
||||
for k, v in class_attributes.items()
|
||||
if not k.startswith("__")
|
||||
and not isinstance(
|
||||
v,
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ per-request cost at the top level of the body.
|
|||
Docs: https://www.edenai.co/docs/v3/llms/embeddings
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING, Final
|
||||
from typing import TYPE_CHECKING, Final, Protocol
|
||||
|
||||
import httpx
|
||||
|
||||
|
|
@ -24,6 +24,14 @@ if TYPE_CHECKING:
|
|||
_SUPPORTED_PARAMS: Final = ("dimensions", "encoding_format", "user")
|
||||
|
||||
|
||||
class _JsonResponse(Protocol):
|
||||
def json(self) -> object: ...
|
||||
|
||||
|
||||
def _json_body(response: _JsonResponse) -> object:
|
||||
return response.json()
|
||||
|
||||
|
||||
class EdenAIEmbeddingConfig(BaseEmbeddingConfig):
|
||||
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: inherited contract
|
||||
return list(_SUPPORTED_PARAMS)
|
||||
|
|
@ -80,12 +88,15 @@ class EdenAIEmbeddingConfig(BaseEmbeddingConfig):
|
|||
optional_params: dict[str, object], # mutable-ok: inherited contract
|
||||
litellm_params: dict[str, object], # mutable-ok: inherited contract
|
||||
) -> EmbeddingResponse:
|
||||
body: Final = raw_response.json()
|
||||
body: Final = _json_body(raw_response)
|
||||
logging_obj.post_call(original_response=body)
|
||||
cost: Final = reported_cost(body)
|
||||
response: Final[EmbeddingResponse] = convert_to_model_response_object(
|
||||
response_object=body, model_response_object=model_response, response_type="embedding"
|
||||
response_object=body if isinstance(body, dict) else None,
|
||||
model_response_object=model_response,
|
||||
response_type="embedding",
|
||||
)
|
||||
set_response_cost_in_hidden_params(response, reported_cost(body))
|
||||
set_response_cost_in_hidden_params(response, cost)
|
||||
return response
|
||||
|
||||
def get_error_class(
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ for every image model in the catalog with the real per-request cost at the top l
|
|||
Docs: https://www.edenai.co/docs/v3/llms/image-generation
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING, Final
|
||||
from typing import TYPE_CHECKING, Final, Protocol
|
||||
|
||||
import httpx
|
||||
|
||||
|
|
@ -36,6 +36,14 @@ _SUPPORTED_PARAMS: Final[tuple[OpenAIImageGenerationOptionalParams, ...]] = (
|
|||
)
|
||||
|
||||
|
||||
class _JsonResponse(Protocol):
|
||||
def json(self) -> object: ...
|
||||
|
||||
|
||||
def _json_body(response: _JsonResponse) -> object:
|
||||
return response.json()
|
||||
|
||||
|
||||
class EdenAIImageGenerationConfig(BaseImageGenerationConfig):
|
||||
def get_supported_openai_params(
|
||||
self, model: str
|
||||
|
|
@ -97,12 +105,15 @@ class EdenAIImageGenerationConfig(BaseImageGenerationConfig):
|
|||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
body: Final = raw_response.json()
|
||||
body: Final = _json_body(raw_response)
|
||||
logging_obj.post_call(original_response=body)
|
||||
cost: Final = reported_cost(body)
|
||||
response: Final[ImageResponse] = convert_to_model_response_object(
|
||||
response_object=body, model_response_object=model_response, response_type="image_generation"
|
||||
response_object=body if isinstance(body, dict) else None,
|
||||
model_response_object=model_response,
|
||||
response_type="image_generation",
|
||||
)
|
||||
set_response_cost_in_hidden_params(response, reported_cost(body))
|
||||
set_response_cost_in_hidden_params(response, cost)
|
||||
return response
|
||||
|
||||
def get_error_class(
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ Translates from OpenAI's `/v1/audio/transcriptions` to ElevenLabs's `/v1/speech-
|
|||
"""
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
from typing import Final
|
||||
from typing import Final, Protocol
|
||||
|
||||
from httpx import Headers, Response
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
|
|
@ -26,6 +26,15 @@ from ..common_utils import ElevenLabsException
|
|||
|
||||
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
|
||||
_JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True))
|
||||
_HIDDEN_PARAMS: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
class _JsonResponse(Protocol):
|
||||
def json(self) -> object: ...
|
||||
|
||||
|
||||
def _json_body(response: _JsonResponse) -> object:
|
||||
return response.json()
|
||||
|
||||
|
||||
class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
||||
|
|
@ -119,7 +128,7 @@ class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
|||
Transforms the raw response from ElevenLabs to the TranscriptionResponse format
|
||||
"""
|
||||
try:
|
||||
response_json: Final = raw_response.json()
|
||||
response_json: Final = _json_body(raw_response)
|
||||
response_object: Final = _JSON_OBJECT.validate_python(response_json)
|
||||
|
||||
# Extract the main transcript text
|
||||
|
|
@ -134,20 +143,18 @@ class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
|||
|
||||
# Map ElevenLabs words to OpenAI format
|
||||
if "words" in response_object:
|
||||
response["words"] = []
|
||||
for word_data in _JSON_OBJECTS.validate_python(response_object["words"]):
|
||||
# Only include actual words, skip spacing and audio events
|
||||
if word_data.get("type") == "word":
|
||||
response["words"].append(
|
||||
{
|
||||
"word": word_data.get("text", ""),
|
||||
"start": word_data.get("start", 0),
|
||||
"end": word_data.get("end", 0),
|
||||
}
|
||||
)
|
||||
response["words"] = [
|
||||
{
|
||||
"word": word_data.get("text", ""),
|
||||
"start": word_data.get("start", 0),
|
||||
"end": word_data.get("end", 0),
|
||||
}
|
||||
for word_data in _JSON_OBJECTS.validate_python(response_object["words"])
|
||||
if word_data.get("type") == "word"
|
||||
]
|
||||
|
||||
# Store full response in hidden params
|
||||
response.hidden_params = response_json
|
||||
response.hidden_params = _HIDDEN_PARAMS.validate_python(response_json)
|
||||
|
||||
return response
|
||||
|
||||
|
|
|
|||
|
|
@ -10,9 +10,10 @@ Proxies the Gemini v1beta Agents API:
|
|||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Final
|
||||
from typing import Any, Final, Protocol
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.base_llm.agents.transformation import BaseAgentsAPIConfig
|
||||
|
|
@ -28,6 +29,18 @@ from litellm.types.agents import (
|
|||
# create-agent body verbatim.
|
||||
_GEMINI_AGENT_BODY_KEYS: Final = ("base_agent", "instructions", "base_environment")
|
||||
|
||||
_AGENT_PAYLOAD: Final = TypeAdapter(dict[str, object])
|
||||
_AGENT_LISTING: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
class _JsonResponse(Protocol):
|
||||
def json(self) -> object: ...
|
||||
|
||||
|
||||
def _json_body(response: _JsonResponse) -> object:
|
||||
return response.json()
|
||||
|
||||
|
||||
# LiteLLM-internal keys that must never be forwarded to Gemini.
|
||||
_LITELLM_INTERNAL_KEYS: Final = frozenset(
|
||||
{
|
||||
|
|
@ -155,18 +168,19 @@ class GeminiAgentsConfig(BaseAgentsAPIConfig):
|
|||
"""
|
||||
self._raise_for_status(raw_response)
|
||||
try:
|
||||
data: dict[str, Any] = raw_response.json()
|
||||
decoded: object = _json_body(raw_response)
|
||||
except Exception:
|
||||
verbose_logger.warning(
|
||||
"GeminiAgentsConfig: non-JSON create response (status=%d).",
|
||||
raw_response.status_code,
|
||||
)
|
||||
data = {"id": name}
|
||||
decoded = {"id": name}
|
||||
data: Final = _AGENT_PAYLOAD.validate_python(decoded)
|
||||
# Gemini uses "id" as the identifier; normalise to both fields.
|
||||
data.setdefault("id", name)
|
||||
data.setdefault("name", data["id"])
|
||||
verbose_logger.debug("GeminiAgentsConfig create response: %s", data)
|
||||
return AgentCreateResponse(**data)
|
||||
return AgentCreateResponse.model_validate(data)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# LIST #
|
||||
|
|
@ -191,13 +205,16 @@ class GeminiAgentsConfig(BaseAgentsAPIConfig):
|
|||
) -> AgentListResponse:
|
||||
self._raise_for_status(raw_response)
|
||||
try:
|
||||
data = raw_response.json()
|
||||
decoded: object = _json_body(raw_response)
|
||||
except Exception:
|
||||
data = {}
|
||||
decoded = {}
|
||||
data: Final = _AGENT_LISTING.validate_python(decoded)
|
||||
verbose_logger.debug("GeminiAgentsConfig list response: %s", data)
|
||||
return AgentListResponse(
|
||||
agents=data.get("agents", []),
|
||||
next_page_token=data.get("nextPageToken"),
|
||||
return AgentListResponse.model_validate(
|
||||
{
|
||||
"agents": data.get("agents", []),
|
||||
"next_page_token": data.get("nextPageToken"),
|
||||
}
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
|
|
@ -221,13 +238,14 @@ class GeminiAgentsConfig(BaseAgentsAPIConfig):
|
|||
"""Same shape as create response — Gemini returns "id" as identifier."""
|
||||
self._raise_for_status(raw_response)
|
||||
try:
|
||||
data = raw_response.json()
|
||||
decoded: object = _json_body(raw_response)
|
||||
except Exception:
|
||||
data = {"id": name}
|
||||
decoded = {"id": name}
|
||||
data: Final = _AGENT_PAYLOAD.validate_python(decoded)
|
||||
data.setdefault("id", name)
|
||||
data.setdefault("name", data["id"])
|
||||
verbose_logger.debug("GeminiAgentsConfig get response: %s", data)
|
||||
return AgentCreateResponse(**data)
|
||||
return AgentCreateResponse.model_validate(data)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# DELETE #
|
||||
|
|
@ -284,11 +302,14 @@ class GeminiAgentsConfig(BaseAgentsAPIConfig):
|
|||
"""
|
||||
self._raise_for_status(raw_response)
|
||||
try:
|
||||
data = raw_response.json()
|
||||
decoded: object = _json_body(raw_response)
|
||||
except Exception:
|
||||
data = {}
|
||||
decoded = {}
|
||||
data: Final = _AGENT_LISTING.validate_python(decoded)
|
||||
verbose_logger.debug("GeminiAgentsConfig list_versions response for '%s': %s", name, data)
|
||||
return AgentVersionsResponse(
|
||||
agent_versions=data.get("agentVersions", []),
|
||||
next_page_token=data.get("nextPageToken"),
|
||||
return AgentVersionsResponse.model_validate(
|
||||
{
|
||||
"agent_versions": data.get("agentVersions", []),
|
||||
"next_page_token": data.get("nextPageToken"),
|
||||
}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,11 +2,11 @@ import base64
|
|||
import datetime
|
||||
import json
|
||||
import math
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Any, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
from pydantic import InstanceOf, JsonValue, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
|
||||
|
|
@ -130,6 +130,9 @@ def is_gemini_image_model(model: str) -> bool:
|
|||
|
||||
|
||||
_JSON_VALUE: Final = TypeAdapter(object)
|
||||
_MODEL_LISTING: Final = TypeAdapter(Mapping[str, object])
|
||||
_MODELS: Final = TypeAdapter(Iterable[Mapping[str, object]])
|
||||
_MODEL_NAME: Final = TypeAdapter(InstanceOf[str])
|
||||
|
||||
|
||||
def _parse_image_config_string(raw_image_config: str, model: str) -> object:
|
||||
|
|
@ -387,10 +390,10 @@ class GeminiModelInfo(BaseLLMModelInfo):
|
|||
def get_base_model(model: str) -> str | None:
|
||||
return model.replace("gemini/", "")
|
||||
|
||||
def process_model_name(self, models: list[dict[str, str]]) -> list[str]:
|
||||
def process_model_name(self, models: Iterable[Mapping[str, object]]) -> list[str]:
|
||||
litellm_model_names: Final = []
|
||||
for model in models:
|
||||
stripped_model_name = model["name"].replace("models/", "")
|
||||
stripped_model_name = _MODEL_NAME.validate_python(model["name"]).replace("models/", "")
|
||||
litellm_model_name = "gemini/" + stripped_model_name
|
||||
litellm_model_names.append(litellm_model_name)
|
||||
return litellm_model_names
|
||||
|
|
@ -414,9 +417,9 @@ class GeminiModelInfo(BaseLLMModelInfo):
|
|||
f"Failed to fetch models from Gemini. Status code: {response.status_code}, Response: {response.json()}"
|
||||
)
|
||||
|
||||
models: Final[list[dict[str, str]]] = response.json()["models"]
|
||||
listing: Final = _MODEL_LISTING.validate_python(response.json())
|
||||
|
||||
litellm_model_names: Final = self.process_model_name(models)
|
||||
litellm_model_names: Final = self.process_model_name(_MODELS.validate_python(listing["models"]))
|
||||
return litellm_model_names
|
||||
|
||||
def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException:
|
||||
|
|
|
|||
|
|
@ -3,9 +3,10 @@ Transformation for Calling Google models in their native format.
|
|||
"""
|
||||
|
||||
from copy import deepcopy
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, cast
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
|
@ -34,6 +35,16 @@ else:
|
|||
|
||||
from ..common_utils import get_api_key_from_env
|
||||
|
||||
_RESPONSE_BODY: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
class _JsonResponse(Protocol):
|
||||
def json(self) -> object: ...
|
||||
|
||||
|
||||
def _json_body(response: _JsonResponse) -> object:
|
||||
return response.json()
|
||||
|
||||
|
||||
class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
|
||||
"""
|
||||
|
|
@ -378,7 +389,7 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
|
|||
from litellm.types.google_genai.main import GenerateContentResponse
|
||||
|
||||
try:
|
||||
response = raw_response.json()
|
||||
decoded: Final = _json_body(raw_response)
|
||||
except Exception as e:
|
||||
raise self.get_error_class(
|
||||
error_message=f"Error transforming generate content response: {e}",
|
||||
|
|
@ -387,7 +398,7 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
|
|||
)
|
||||
|
||||
logging_obj.model_call_details["httpx_response"] = raw_response
|
||||
response = self.convert_citation_sources_to_citations(response)
|
||||
response: Final = self.convert_citation_sources_to_citations(_RESPONSE_BODY.validate_python(decoded))
|
||||
|
||||
return GenerateContentResponse(**response)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
import base64
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Iterable, Mapping
|
||||
from io import BufferedReader, BytesIO
|
||||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
from typing import TYPE_CHECKING, Any, Final, Protocol, cast
|
||||
|
||||
import httpx
|
||||
from httpx._types import RequestFiles
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.images.utils import ImageEditRequestUtils
|
||||
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
|
||||
|
|
@ -32,6 +33,18 @@ if TYPE_CHECKING:
|
|||
else:
|
||||
LiteLLMLoggingObj = Any
|
||||
|
||||
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
|
||||
_JSON_VALUES: Final = TypeAdapter(Iterable[object])
|
||||
_USAGE_METADATA: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
class _JsonResponse(Protocol):
|
||||
def json(self) -> object: ...
|
||||
|
||||
|
||||
def _json_body(response: _JsonResponse) -> object:
|
||||
return response.json()
|
||||
|
||||
|
||||
class GeminiImageEditConfig(BaseImageEditConfig):
|
||||
DEFAULT_BASE_URL: str = "https://generativelanguage.googleapis.com/v1beta"
|
||||
|
|
@ -125,7 +138,7 @@ class GeminiImageEditConfig(BaseImageEditConfig):
|
|||
) -> ImageResponse:
|
||||
model_response: Final = ImageResponse()
|
||||
try:
|
||||
response_json: Final = raw_response.json()
|
||||
response_json: Final = _json_body(raw_response)
|
||||
except Exception as exc:
|
||||
raise self.get_error_class(
|
||||
error_message=f"Error transforming image edit response: {exc}",
|
||||
|
|
@ -133,25 +146,29 @@ class GeminiImageEditConfig(BaseImageEditConfig):
|
|||
headers=raw_response.headers,
|
||||
)
|
||||
|
||||
candidates: Final = response_json.get("candidates", [])
|
||||
payload: Final = _JSON_OBJECT.validate_python(response_json)
|
||||
candidates: Final = payload.get("candidates", [])
|
||||
data_list: Final[list[ImageObject]] = []
|
||||
|
||||
for candidate in candidates:
|
||||
content = candidate.get("content", {})
|
||||
parts = content.get("parts", [])
|
||||
for part in parts:
|
||||
inline_data = part.get("inlineData")
|
||||
if inline_data and inline_data.get("data"):
|
||||
for candidate in _JSON_VALUES.validate_python(candidates):
|
||||
content = _JSON_OBJECT.validate_python(candidate).get("content", {})
|
||||
parts = _JSON_OBJECT.validate_python(content).get("parts", [])
|
||||
for part in _JSON_VALUES.validate_python(parts):
|
||||
inline_data = _JSON_OBJECT.validate_python(part).get("inlineData")
|
||||
image_data = _JSON_OBJECT.validate_python(inline_data).get("data") if inline_data else None
|
||||
if image_data:
|
||||
data_list.append(
|
||||
ImageObject(
|
||||
b64_json=inline_data["data"],
|
||||
b64_json=image_data,
|
||||
url=None,
|
||||
)
|
||||
)
|
||||
|
||||
model_response.data = cast(list[OpenAIImage], data_list)
|
||||
if "usageMetadata" in response_json:
|
||||
model_response.usage = transform_gemini_image_usage(response_json["usageMetadata"])
|
||||
if "usageMetadata" in payload:
|
||||
model_response.usage = transform_gemini_image_usage(
|
||||
_USAGE_METADATA.validate_python(payload["usageMetadata"])
|
||||
)
|
||||
return model_response
|
||||
|
||||
def _prepare_inline_image_parts(self, image: FileTypes | list[FileTypes]) -> list[dict[str, object]]:
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ https://github.com/caozhiyuan/copilot-api
|
|||
"""
|
||||
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
|
|
@ -16,6 +17,8 @@ from typing import (
|
|||
cast, # noqa: TID251 # narrows untyped request dicts at the session boundary
|
||||
)
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
|
||||
|
|
@ -46,6 +49,9 @@ if TYPE_CHECKING:
|
|||
else:
|
||||
LiteLLMLoggingObj = Any
|
||||
|
||||
_OBJECT_LIST: Final = TypeAdapter(list[object])
|
||||
_OBJECT_MAPPING: Final = TypeAdapter(Mapping[object, object])
|
||||
|
||||
|
||||
def github_copilot_supports_responses_api(model: str) -> bool:
|
||||
"""
|
||||
|
|
@ -390,7 +396,9 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
"""
|
||||
return self._contains_vision_content(input_param)
|
||||
|
||||
def _contains_vision_content(self, value: Any, depth: int = 0, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH) -> bool:
|
||||
def _contains_vision_content(
|
||||
self, value: object, depth: int = 0, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH
|
||||
) -> bool:
|
||||
"""
|
||||
Recursively check if a value contains vision content.
|
||||
|
||||
|
|
@ -407,22 +415,28 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
|
||||
# Check arrays
|
||||
if isinstance(value, list):
|
||||
return any(self._contains_vision_content(item, depth=depth + 1, max_depth=max_depth) for item in value)
|
||||
items: Final = _OBJECT_LIST.validate_python(value)
|
||||
return any(self._contains_vision_content(item, depth=depth + 1, max_depth=max_depth) for item in items)
|
||||
|
||||
# Only check dict/object types
|
||||
if not isinstance(value, dict):
|
||||
return False
|
||||
|
||||
fields: Final = _OBJECT_MAPPING.validate_python(value)
|
||||
|
||||
# Check if this item is an input_image
|
||||
item_type: Final = value.get("type")
|
||||
item_type: Final = fields.get("type")
|
||||
if isinstance(item_type, str) and item_type.lower() == "input_image":
|
||||
return True
|
||||
|
||||
# Check content field recursively
|
||||
if "content" in value and isinstance(value["content"], list):
|
||||
return any(
|
||||
self._contains_vision_content(item, depth=depth + 1, max_depth=max_depth) for item in value["content"]
|
||||
)
|
||||
if "content" in fields:
|
||||
content: Final = fields["content"]
|
||||
if isinstance(content, list):
|
||||
content_items: Final = _OBJECT_LIST.validate_python(content)
|
||||
return any(
|
||||
self._contains_vision_content(item, depth=depth + 1, max_depth=max_depth) for item in content_items
|
||||
)
|
||||
|
||||
return False
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,10 @@ if TYPE_CHECKING:
|
|||
def cost_per_web_search_request(usage: Usage, model_info: "ModelInfo") -> float:
|
||||
search_costs: Final = model_info.get("search_context_cost_per_query")
|
||||
cost_per_search: Final = search_costs.get("search_context_size_medium", 0.0) if search_costs else 0.0
|
||||
server_tool_use: Final = getattr(usage, "server_tool_use", None)
|
||||
try:
|
||||
server_tool_use: Final = usage.server_tool_use
|
||||
except AttributeError:
|
||||
return 0.0
|
||||
if server_tool_use is None:
|
||||
return 0.0
|
||||
searches: Final = server_tool_use.web_search_requests or 0
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ Translate from OpenAI's `/v1/audio/transcriptions` to Groq's `/v1/audio/transcri
|
|||
"""
|
||||
|
||||
import types
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import litellm
|
||||
|
|
@ -39,16 +40,17 @@ class GroqSTTConfig:
|
|||
tools: list | None = None,
|
||||
tool_choice: str | dict | None = None,
|
||||
) -> None:
|
||||
locals_: Final = locals().copy()
|
||||
locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy())
|
||||
for key, value in locals_.items():
|
||||
if key != "self" and value is not None:
|
||||
setattr(self.__class__, key, value)
|
||||
|
||||
@classmethod
|
||||
def get_config(cls):
|
||||
class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__)
|
||||
return {
|
||||
k: v
|
||||
for k, v in cls.__dict__.items()
|
||||
for k, v in class_attributes.items()
|
||||
if not k.startswith("__")
|
||||
and not isinstance(
|
||||
v,
|
||||
|
|
|
|||
|
|
@ -93,7 +93,7 @@ class HuggingFaceRerankConfig(BaseRerankConfig):
|
|||
model: str,
|
||||
drop_params: bool,
|
||||
query: str,
|
||||
documents: list[str | dict[str, Any]],
|
||||
documents: list[str | dict[str, object]],
|
||||
custom_llm_provider: str | None = None,
|
||||
top_n: int | None = None,
|
||||
rank_fields: list[str] | None = None,
|
||||
|
|
|
|||
|
|
@ -9,10 +9,13 @@ import uuid
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class LangGraphSSEStreamIterator:
|
||||
"""
|
||||
|
|
@ -60,7 +63,7 @@ class LangGraphSSEStreamIterator:
|
|||
return None
|
||||
|
||||
try:
|
||||
data: Final = json.loads(json_str)
|
||||
data: Final = _DECODED_JSON.validate_python(json.loads(json_str))
|
||||
return self._process_data(data)
|
||||
except json.JSONDecodeError:
|
||||
verbose_logger.debug("Skipping non-JSON SSE line: %s", line[:100])
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ Translate from OpenAI's `/v1/chat/completions` to Lemonade's `/v1/chat/completio
|
|||
"""
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
from typing import TYPE_CHECKING, Any, Final, Protocol
|
||||
from typing import TYPE_CHECKING, Final, Protocol
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
|
|
@ -26,6 +26,7 @@ if TYPE_CHECKING:
|
|||
_JSON_DICT: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True))
|
||||
_JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True))
|
||||
_STR: Final = TypeAdapter(str)
|
||||
_OPAQUE_VALUE: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class _JsonResponse(Protocol):
|
||||
|
|
@ -122,7 +123,7 @@ class LemonadeChatConfig(OpenAILikeChatConfig):
|
|||
return ["lemonade/" + _STR.validate_python(model["id"]) for model in model_list]
|
||||
|
||||
@staticmethod
|
||||
def _get_positive_int(value: Any) -> int | None:
|
||||
def _get_positive_int(value: object) -> int | None:
|
||||
if isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, int) and value > 0:
|
||||
|
|
@ -162,7 +163,7 @@ class LemonadeChatConfig(OpenAILikeChatConfig):
|
|||
provider_specific_entry.get("context_window"),
|
||||
provider_specific_entry.get("max_context_window"),
|
||||
):
|
||||
parsed = self._get_positive_int(value)
|
||||
parsed = self._get_positive_int(_OPAQUE_VALUE.validate_python(value))
|
||||
if parsed is not None:
|
||||
return parsed
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@ Supports Docker, Podman, and Kubernetes backends.
|
|||
|
||||
import base64
|
||||
import os
|
||||
from typing import Any, Final, Protocol, TypedDict
|
||||
from collections.abc import Sequence
|
||||
from typing import Final, Protocol, TypedDict
|
||||
|
||||
from typing_extensions import ReadOnly
|
||||
|
||||
|
|
@ -41,6 +42,13 @@ class _GeneratedFile(TypedDict):
|
|||
mime_type: ReadOnly[str]
|
||||
|
||||
|
||||
class _SandboxExecution(TypedDict):
|
||||
success: ReadOnly[bool]
|
||||
output: ReadOnly[str]
|
||||
error: ReadOnly[str]
|
||||
files: ReadOnly[Sequence[_GeneratedFile]]
|
||||
|
||||
|
||||
class SkillsSandboxExecutor:
|
||||
"""
|
||||
Executes skill code in llm-sandbox Docker container.
|
||||
|
|
@ -76,7 +84,7 @@ class SkillsSandboxExecutor:
|
|||
code: str,
|
||||
skill_files: dict[str, bytes],
|
||||
requirements: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
) -> _SandboxExecution:
|
||||
"""
|
||||
Execute code with skill files in sandbox.
|
||||
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ Docs - https://lmstudio.ai/docs/basics/server
|
|||
"""
|
||||
|
||||
import types
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
|
||||
|
|
@ -18,16 +19,17 @@ class LmStudioEmbeddingConfig:
|
|||
def __init__(
|
||||
self,
|
||||
) -> None:
|
||||
locals_: Final = locals().copy()
|
||||
locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy())
|
||||
for key, value in locals_.items():
|
||||
if key != "self" and value is not None:
|
||||
setattr(self.__class__, key, value)
|
||||
|
||||
@classmethod
|
||||
def get_config(cls):
|
||||
class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__)
|
||||
return {
|
||||
k: v
|
||||
for k, v in cls.__dict__.items()
|
||||
for k, v in class_attributes.items()
|
||||
if not k.startswith("__")
|
||||
and not isinstance(
|
||||
v,
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ API calling is done using the OpenAI SDK with an api_base
|
|||
"""
|
||||
|
||||
import types
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
|
||||
|
|
@ -32,16 +33,17 @@ class NvidiaNimEmbeddingConfig:
|
|||
input_type: str | None = None,
|
||||
truncate: str | None = None,
|
||||
) -> None:
|
||||
locals_: Final = locals().copy()
|
||||
locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy())
|
||||
for key, value in locals_.items():
|
||||
if key != "self" and value is not None:
|
||||
setattr(self.__class__, key, value)
|
||||
|
||||
@classmethod
|
||||
def get_config(cls):
|
||||
class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__)
|
||||
return {
|
||||
k: v
|
||||
for k, v in cls.__dict__.items()
|
||||
for k, v in class_attributes.items()
|
||||
if not k.startswith("__")
|
||||
and not isinstance(
|
||||
v,
|
||||
|
|
|
|||
|
|
@ -188,6 +188,7 @@ def _normalize_tool_choice(selected_params: dict) -> None:
|
|||
_TOOL_CHOICE_MODES: Final = frozenset({"auto", "none", "required", "any"})
|
||||
_TOOLS_ADAPTER: Final = TypeAdapter(list[dict[str, object]])
|
||||
_JSON_OBJECT_ADAPTER: Final = TypeAdapter(dict[str, object])
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
def _as_json_object(value: object) -> Mapping[str, object] | None:
|
||||
|
|
@ -700,7 +701,7 @@ class OCIChatConfig(BaseConfig):
|
|||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
response_json: Final = raw_response.json()
|
||||
response_json: Final = _DECODED_JSON.validate_python(raw_response.json())
|
||||
|
||||
if not isinstance(response_json, dict):
|
||||
raise OCIError(
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
|||
from litellm.types.utils import ModelResponse, ModelResponseStream
|
||||
|
||||
_PROVIDER_FIELDS: Final = TypeAdapter(dict[str, object])
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class OllamaResponseIdNormalizer:
|
||||
|
|
@ -151,7 +152,7 @@ class OllamaModelInfo(BaseLLMModelInfo):
|
|||
try:
|
||||
resp: Final = httpx.get(f"{base}/api/tags", headers=headers)
|
||||
resp.raise_for_status()
|
||||
data: Final = resp.json()
|
||||
data: Final = _DECODED_JSON.validate_python(resp.json())
|
||||
# Expecting a dict with a 'models' list
|
||||
models_list = []
|
||||
if isinstance(data, dict) and "models" in data and isinstance(data["models"], list):
|
||||
|
|
|
|||
|
|
@ -23,13 +23,14 @@ from openai.types.chat.chat_completion import Choice
|
|||
from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice
|
||||
from openai.types.chat.chat_completion_chunk import ChoiceDelta
|
||||
from openai.types.completion_usage import CompletionUsage
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from aiohttp import ClientSession
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.token_counter import token_counter
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
|
|
@ -134,6 +135,16 @@ class OpenAIError(BaseLLMException):
|
|||
)
|
||||
|
||||
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
def decoded_json_object(value: object) -> Mapping[str, object]:
|
||||
mapping: Final = as_str_mapping(value)
|
||||
if mapping is not None:
|
||||
return mapping
|
||||
raise AttributeError(f"{type(value).__name__!r} object has no attribute 'get'", name="get", obj=value)
|
||||
|
||||
|
||||
####### Error Handling Utils for OpenAI API #######################
|
||||
###################################################################
|
||||
def drop_params_from_unprocessable_entity_error(
|
||||
|
|
@ -152,7 +163,7 @@ def drop_params_from_unprocessable_entity_error(
|
|||
"""
|
||||
invalid_params: Final[list[str]] = []
|
||||
if isinstance(e, httpx.HTTPStatusError):
|
||||
error_json: Final = e.response.json()
|
||||
error_json: Final = decoded_json_object(_DECODED_JSON.validate_python(e.response.json()))
|
||||
error_message: Final = error_json.get("error", {})
|
||||
error_body = error_message
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from collections.abc import Mapping, Sequence
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -18,6 +19,8 @@ from litellm.llms.openai.responses.count_tokens.transformation import (
|
|||
OpenAICountTokensConfig,
|
||||
)
|
||||
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class OpenAICountTokensHandler(OpenAICountTokensConfig):
|
||||
"""
|
||||
|
|
@ -34,7 +37,7 @@ class OpenAICountTokensHandler(OpenAICountTokensConfig):
|
|||
tools: Sequence[Mapping[str, object]] | None = None,
|
||||
instructions: str | None = None,
|
||||
text: Mapping[str, object] | None = None,
|
||||
) -> dict[str, object]:
|
||||
) -> object:
|
||||
"""
|
||||
Handle a token counting request to OpenAI's Responses API.
|
||||
|
||||
|
|
@ -84,7 +87,7 @@ class OpenAICountTokensHandler(OpenAICountTokensConfig):
|
|||
message=error_text,
|
||||
)
|
||||
|
||||
openai_response: Final = response.json()
|
||||
openai_response: Final = _DECODED_JSON.validate_python(response.json())
|
||||
verbose_logger.debug("OpenAI response: %s", openai_response)
|
||||
return openai_response
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ from litellm.llms.anthropic.pass_through.responses_adapters.transformation impor
|
|||
LiteLLMAnthropicToResponsesAPIAdapter,
|
||||
)
|
||||
from litellm.llms.base_llm.base_utils import BaseTokenCounter
|
||||
from litellm.llms.openai.common_utils import OpenAIError, get_openai_credentials
|
||||
from litellm.llms.openai.common_utils import OpenAIError, decoded_json_object, get_openai_credentials
|
||||
from litellm.llms.openai.responses.count_tokens.handler import (
|
||||
OpenAICountTokensHandler,
|
||||
)
|
||||
|
|
@ -197,7 +197,7 @@ class OpenAITokenCounter(BaseTokenCounter):
|
|||
if result is not None:
|
||||
return TokenCountResponse.model_validate(
|
||||
{
|
||||
"total_tokens": result.get("input_tokens", 0),
|
||||
"total_tokens": decoded_json_object(result).get("input_tokens", 0),
|
||||
"request_model": request_model,
|
||||
"model_used": model_to_use,
|
||||
"tokenizer_type": "openai_api",
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from typing import Any, Final
|
||||
from typing import Final
|
||||
|
||||
import litellm
|
||||
from litellm.llms.anthropic.common_utils import normalize_cache_control_in_anthropic_payload
|
||||
|
|
@ -40,7 +40,7 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig):
|
|||
self,
|
||||
headers: dict[str, str],
|
||||
model: str,
|
||||
messages: list[Any],
|
||||
messages: list[object],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
api_key: str | None = None,
|
||||
|
|
@ -150,7 +150,7 @@ class JSONProviderAnthropicMessagesConfig(OpenAILikeAnthropicMessagesConfig):
|
|||
self,
|
||||
headers: dict[str, str],
|
||||
model: str,
|
||||
messages: list[Any],
|
||||
messages: list[object],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
api_key: str | None = None,
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from litellm.constants import (
|
|||
OPEN_SANDBOX_POLL_INTERVAL,
|
||||
OPEN_SANDBOX_READY_TIMEOUT,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
|
||||
from litellm.llms.base_llm.sandbox.transformation import (
|
||||
SANDBOX_MAX_OUTPUT_BYTES,
|
||||
|
|
@ -43,6 +44,13 @@ _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_i
|
|||
_JSON_VALUE: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
def _decoded_json_object(value: object) -> Mapping[str, object]:
|
||||
mapping: Final = as_str_mapping(value)
|
||||
if mapping is not None:
|
||||
return mapping
|
||||
raise AttributeError(f"{type(value).__name__!r} object has no attribute 'get'", name="get", obj=value)
|
||||
|
||||
|
||||
class OpenSandboxSandboxConfig(BaseSandboxConfig):
|
||||
def _http(self, client: AsyncHTTPHandler | None) -> AsyncHTTPHandler:
|
||||
if client is not None:
|
||||
|
|
@ -320,7 +328,7 @@ class OpenSandboxSandboxConfig(BaseSandboxConfig):
|
|||
headers=headers,
|
||||
params={"use_server_proxy": use_server_proxy},
|
||||
)
|
||||
data: Final = response.json()
|
||||
data: Final = _decoded_json_object(_JSON_VALUE.validate_python(response.json()))
|
||||
endpoint: Final = data.get("endpoint")
|
||||
if not endpoint:
|
||||
raise ValueError(f"OpenSandbox did not return an execd endpoint for {sandbox_id}")
|
||||
|
|
@ -491,7 +499,7 @@ class OpenSandboxSandboxConfig(BaseSandboxConfig):
|
|||
if not data:
|
||||
return None
|
||||
try:
|
||||
parsed: Final = json.loads(data)
|
||||
parsed: Final = _JSON_VALUE.validate_python(json.loads(data))
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
if not isinstance(parsed, dict):
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ Ref: https://docs.perplexity.ai/api-reference/responses-post
|
|||
from typing import Any, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
|
@ -21,6 +22,8 @@ from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIResponse
|
|||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
|
||||
def get_supported_openai_params(self, model: str) -> list:
|
||||
|
|
@ -103,7 +106,7 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
|
|||
) -> ResponsesAPIResponse:
|
||||
"""Check for Perplexity's status:'failed' on HTTP 200 before delegating to base."""
|
||||
try:
|
||||
raw_response_json = raw_response.json()
|
||||
raw_response_json = _DECODED_JSON.validate_python(raw_response.json())
|
||||
except Exception:
|
||||
raw_response_json = None
|
||||
|
||||
|
|
|
|||
|
|
@ -40,6 +40,11 @@ class _RunwayMLTask(TypedDict, total=False):
|
|||
failureCode: ReadOnly[str]
|
||||
|
||||
|
||||
def _parse_runway_task(response: httpx.Response) -> _RunwayMLTask:
|
||||
task: Final[_RunwayMLTask] = response.json()
|
||||
return task
|
||||
|
||||
|
||||
class RunwayMLImageGenerationConfig(BaseImageGenerationConfig):
|
||||
"""
|
||||
Configuration for RunwayML image generation models.
|
||||
|
|
@ -238,7 +243,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig):
|
|||
response = client.get(url=task_url, headers=headers)
|
||||
response.raise_for_status()
|
||||
|
||||
response_data: _RunwayMLTask = response.json()
|
||||
response_data: _RunwayMLTask = _parse_runway_task(response)
|
||||
|
||||
# Check task status
|
||||
status = self._check_task_status(response_data=response_data)
|
||||
|
|
@ -287,7 +292,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig):
|
|||
response = await client.get(url=task_url, headers=headers)
|
||||
response.raise_for_status()
|
||||
|
||||
response_data: _RunwayMLTask = response.json()
|
||||
response_data: _RunwayMLTask = _parse_runway_task(response)
|
||||
|
||||
# Check task status
|
||||
status = self._check_task_status(response_data=response_data)
|
||||
|
|
@ -333,7 +338,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig):
|
|||
}
|
||||
"""
|
||||
try:
|
||||
response_data: _RunwayMLTask = raw_response.json()
|
||||
response_data: _RunwayMLTask = _parse_runway_task(raw_response)
|
||||
except Exception as e:
|
||||
raise self.get_error_class(
|
||||
error_message=f"Error transforming image generation response: {e}",
|
||||
|
|
@ -363,7 +368,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig):
|
|||
)
|
||||
|
||||
# Update response_data with polled result
|
||||
response_data = raw_response.json()
|
||||
response_data = _parse_runway_task(raw_response)
|
||||
|
||||
verbose_logger.debug("RunwayML polling complete, transforming to OpenAI format")
|
||||
|
||||
|
|
@ -393,7 +398,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig):
|
|||
We need to poll the task until it completes (status SUCCEEDED) using async polling.
|
||||
"""
|
||||
try:
|
||||
response_data: _RunwayMLTask = raw_response.json()
|
||||
response_data: _RunwayMLTask = _parse_runway_task(raw_response)
|
||||
except Exception as e:
|
||||
raise self.get_error_class(
|
||||
error_message=f"Error transforming image generation response: {e}",
|
||||
|
|
@ -423,7 +428,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig):
|
|||
)
|
||||
|
||||
# Update response_data with polled result
|
||||
response_data = raw_response.json()
|
||||
response_data = _parse_runway_task(raw_response)
|
||||
|
||||
verbose_logger.debug("RunwayML polling complete (async), transforming to OpenAI format")
|
||||
|
||||
|
|
|
|||
|
|
@ -40,6 +40,11 @@ class _RunwayTtsTaskResponse(TypedDict, total=False):
|
|||
failureCode: ReadOnly[str]
|
||||
|
||||
|
||||
def _parse_runway_tts_task(response: httpx.Response) -> _RunwayTtsTaskResponse:
|
||||
task: Final[_RunwayTtsTaskResponse] = response.json()
|
||||
return task
|
||||
|
||||
|
||||
class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
|
||||
"""
|
||||
Configuration for RunwayML Text-to-Speech
|
||||
|
|
@ -323,7 +328,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
|
|||
response = client.get(url=task_url, headers=headers)
|
||||
response.raise_for_status()
|
||||
|
||||
response_data: _RunwayTtsTaskResponse = response.json()
|
||||
response_data: _RunwayTtsTaskResponse = _parse_runway_tts_task(response)
|
||||
|
||||
# Check task status
|
||||
status = self._check_task_status(response_data=response_data)
|
||||
|
|
@ -371,7 +376,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
|
|||
response = await client.get(url=task_url, headers=headers)
|
||||
response.raise_for_status()
|
||||
|
||||
response_data: _RunwayTtsTaskResponse = response.json()
|
||||
response_data: _RunwayTtsTaskResponse = _parse_runway_tts_task(response)
|
||||
|
||||
# Check task status
|
||||
status = self._check_task_status(response_data=response_data)
|
||||
|
|
@ -462,7 +467,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
|
|||
from litellm.types.llms.openai import HttpxBinaryResponseContent
|
||||
|
||||
try:
|
||||
response_data: Final[_RunwayTtsTaskResponse] = raw_response.json()
|
||||
response_data: Final[_RunwayTtsTaskResponse] = _parse_runway_tts_task(raw_response)
|
||||
except Exception as e:
|
||||
raise self.get_error_class(
|
||||
error_message=f"Error parsing RunwayML TTS response: {e}",
|
||||
|
|
@ -492,7 +497,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
|
|||
)
|
||||
|
||||
# Get the completed task data
|
||||
task_data: Final[_RunwayTtsTaskResponse] = polled_response.json()
|
||||
task_data: Final[_RunwayTtsTaskResponse] = _parse_runway_tts_task(polled_response)
|
||||
|
||||
verbose_logger.debug("RunwayML TTS polling complete, downloading audio")
|
||||
|
||||
|
|
@ -531,7 +536,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
|
|||
from litellm.types.llms.openai import HttpxBinaryResponseContent
|
||||
|
||||
try:
|
||||
response_data: Final[_RunwayTtsTaskResponse] = raw_response.json()
|
||||
response_data: Final[_RunwayTtsTaskResponse] = _parse_runway_tts_task(raw_response)
|
||||
except Exception as e:
|
||||
raise self.get_error_class(
|
||||
error_message=f"Error parsing RunwayML TTS response: {e}",
|
||||
|
|
@ -561,7 +566,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
|
|||
)
|
||||
|
||||
# Get the completed task data
|
||||
task_data: Final[_RunwayTtsTaskResponse] = polled_response.json()
|
||||
task_data: Final[_RunwayTtsTaskResponse] = _parse_runway_tts_task(polled_response)
|
||||
|
||||
verbose_logger.debug("RunwayML TTS polling complete (async), downloading audio")
|
||||
|
||||
|
|
|
|||
|
|
@ -119,7 +119,7 @@ class RunwayMLVideoConfig(BaseVideoConfig):
|
|||
|
||||
@staticmethod
|
||||
def _parse_task_response(raw_response: httpx.Response) -> _RunwayTaskResponse:
|
||||
return raw_response.json()
|
||||
return _parse_runway_task_response(raw_response)
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import builtins
|
||||
import codecs
|
||||
import functools
|
||||
import json
|
||||
|
|
@ -34,6 +35,14 @@ class _SageMakerMessagesChunkData(TypedDict, total=False):
|
|||
choices: ReadOnly[Required[list[Mapping[str, object]]]]
|
||||
|
||||
|
||||
class _SageMakerPayloadPart(TypedDict, total=False):
|
||||
bytes: ReadOnly[builtins.bytes]
|
||||
|
||||
|
||||
class _SageMakerStreamEvent(TypedDict, total=False):
|
||||
chunk: ReadOnly[_SageMakerPayloadPart]
|
||||
|
||||
|
||||
class _SageMakerChunkDataRoot(RootModel[_SageMakerChunkData]):
|
||||
model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
|
||||
|
||||
|
|
@ -345,7 +354,7 @@ class AWSEventStreamDecoder:
|
|||
),
|
||||
)
|
||||
response_dict: Final = event.to_response_dict()
|
||||
parsed_response: Final = self.parser.parse(response_dict, response_stream_shape)
|
||||
parsed_response: Final[_SageMakerStreamEvent] = self.parser.parse(response_dict, response_stream_shape)
|
||||
body: Final = response_dict.get("body")
|
||||
|
||||
if response_dict["status_code"] != 200:
|
||||
|
|
|
|||
|
|
@ -10,7 +10,8 @@ be of type Object`.
|
|||
Reference: https://docs.cohere.com/v2/reference/embed
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
|
@ -25,6 +26,7 @@ from litellm.llms.bedrock.embed.cohere_transformation import (
|
|||
BedrockCohereEmbeddingConfig,
|
||||
)
|
||||
from litellm.llms.cohere.embed.v1_transformation import CohereEmbeddingConfig
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import EmbeddingResponse
|
||||
|
||||
from ..common_utils import SagemakerError
|
||||
|
|
@ -123,7 +125,7 @@ class SagemakerCohereEmbeddingConfig(BaseEmbeddingConfig):
|
|||
self,
|
||||
headers: dict,
|
||||
model: str,
|
||||
messages: list[Any],
|
||||
messages: Sequence[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
api_key: str | None = None,
|
||||
|
|
|
|||
|
|
@ -4,7 +4,8 @@ Translate from OpenAI's `/v1/embeddings` to Sagemaker's `/invoke`
|
|||
In the Huggingface TGI format.
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
|
@ -15,6 +16,7 @@ from httpx._models import Headers, Response
|
|||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig
|
||||
from litellm.llms.voyage.embedding.transformation import VoyageEmbeddingConfig
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import EmbeddingResponse, Usage
|
||||
|
||||
from ..common_utils import SagemakerError
|
||||
|
|
@ -158,7 +160,7 @@ class SagemakerEmbeddingConfig(BaseEmbeddingConfig):
|
|||
self,
|
||||
headers: dict,
|
||||
model: str,
|
||||
messages: list[Any],
|
||||
messages: Sequence[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
api_key: str | None = None,
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ if TYPE_CHECKING:
|
|||
else:
|
||||
LiteLLMLoggingObj = Any
|
||||
|
||||
from ..credentials import get_token_creator
|
||||
from ..credentials import AICoreConfiguration, AICoreDeploymentList, get_token_creator
|
||||
from .handler import (
|
||||
AsyncSAPStreamIterator,
|
||||
GenAIHubOrchestrationError,
|
||||
|
|
@ -185,11 +185,13 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
|
|||
# Keep a short, tight client lifecycle here to avoid fd leaks
|
||||
client: Final = litellm.module_level_client
|
||||
# with httpx.Client(timeout=30) as client:
|
||||
deployments: Final = client.get(f"{self.base_url}/lm/deployments", headers=self.headers).json()
|
||||
deployments: Final[AICoreDeploymentList] = client.get(
|
||||
f"{self.base_url}/lm/deployments", headers=self.headers
|
||||
).json()
|
||||
valid: Final[list[tuple[str, str]]] = []
|
||||
for dep in deployments.get("resources", []):
|
||||
if dep.get("scenarioId") == "orchestration":
|
||||
cfg = client.get(
|
||||
cfg: AICoreConfiguration = client.get(
|
||||
f"{self.base_url}/lm/configurations/{dep['configurationId']}",
|
||||
headers=self.headers,
|
||||
).json()
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue