scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-12 08:42:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 207

refactor(typing): clear 1141 basedpyright Any errors across 182 files (#50711)

Browse source 
* refactor(typing): clear 340 basedpyright Any errors (wave 1)

* refactor(typing): clear 497 basedpyright Any errors (wave 2)

* refactor(typing): clear 74 basedpyright Any errors (wave 3)

* refactor(typing): clear 143 basedpyright Any errors (wave 4)

* refactor(typing): clear 106 basedpyright Any errors (wave 5)

* fix(typing): keep router fence visibility and py310-safe Self import

Bind the Protocol-narrowed routers in pass_through_endpoints and rag
ingestion to router-named locals so router_surface_check still sees the
aadapter_completion and aocr uses, and import Self from typing_extensions
in prisma_client for check_py310_typing_imports.

* perf(core_helpers): fast-path dicts and lists in the checked JSON helpers

as_str_mapping now tries the concrete dict check before the Mapping ABC
check, and iter_items returns lists before the runtime-checkable Protocol
check. Both return the same object as before, and roughly halve the
overhead CodSpeed flagged on the A2A text-extraction benchmarks.

* refactor(realtime): keep the base TypeError for a non-mapping translation expires_after

* fix(proxy): restore the require_attributes import the main merge dropped

* test: cover the retyped realtime, vector store, agent, stream, video and provider response paths

* fix(realtime): name the public request types in non-mapping TypeErrors

keyword_arguments wraps expires_after and transcription_session so a non-mapping value raises the same TypeError text as before the typing pass. Tests narrow Greptile's pegasus case with isinstance and freeze its model.

---------

Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
quietjules 2026-11-11 14:11:47 -08:00 • committed by GitHub
parent 5e74515c29
commit 89149b4544
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
194 changed files with 4116 additions and 1332 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py
View file

@ -123,7 +123,7 @@ class _PROXY_LiteLLMManagedVectorStores(
"""
# Use the router to create the vector store
response: Final = await llm_router.avector_store_create(model=model, **request_data)
return cast(VectorStoreCreateResponse, response)
return response
# ============================================================================
# VECTOR STORE CRUD OPERATIONS

51
litellm/assistants/main.py
View file

@ -5,14 +5,16 @@ import contextvars
import os
from collections.abc import Coroutine, Iterable, Mapping, Sequence
from functools import partial
from typing import Any, Final, Literal
from typing import Any, Final, Literal, Protocol
import httpx
from openai import AsyncOpenAI, OpenAI
from openai.types.beta.assistant import Assistant
from openai.types.beta.assistant_deleted import AssistantDeleted
from pydantic import TypeAdapter
import litellm
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.types.router import GenericLiteLLMParams
from litellm.utils import (
exception_type,
@ -31,6 +33,21 @@ from .utils import get_optional_params_add_message
####### ENVIRONMENT VARIABLES ###################
openai_assistants_api: Final = OpenAIAssistantsAPI()
azure_assistants_api: Final = AzureAssistantsAPI()
_EXTRA_BODY: Final = TypeAdapter(object)
class _PopsKeys(Protocol):
def pop(self, key: str, default: None, /) -> object: ...
def _pop_azure_ad_token(extra_body: object) -> object:
return require_attributes(extra_body, _PopsKeys, "pop").pop("azure_ad_token", None)
def _popped_azure_ad_token(extra_body: object) -> str | None:
token: Final = _pop_azure_ad_token(extra_body)
return token if isinstance(token, str) else None
### ASSISTANTS ###
@ -146,10 +163,10 @@ def get_assistants(
or get_secret("AZURE_API_KEY")
)
extra_body: Final = optional_params.get("extra_body", {})
extra_body: Final = _EXTRA_BODY.validate_python(optional_params.get("extra_body", {}))
azure_ad_token: str | None = None
if extra_body is not None:
azure_ad_token = extra_body.pop("azure_ad_token", None)
azure_ad_token = _popped_azure_ad_token(extra_body)
else:
azure_ad_token = get_secret("AZURE_AD_TOKEN")
@ -331,10 +348,10 @@ def create_assistants(
or get_secret("AZURE_API_KEY")
)
extra_body: Final = optional_params.get("extra_body", {})
extra_body: Final = _EXTRA_BODY.validate_python(optional_params.get("extra_body", {}))
azure_ad_token: str | None = None
if extra_body is not None:
azure_ad_token = extra_body.pop("azure_ad_token", None)
azure_ad_token = _popped_azure_ad_token(extra_body)
else:
azure_ad_token = get_secret("AZURE_AD_TOKEN")
@ -481,10 +498,10 @@ def delete_assistant(
or get_secret("AZURE_API_KEY")
)
extra_body: Final = optional_params.get("extra_body", {})
extra_body: Final = _EXTRA_BODY.validate_python(optional_params.get("extra_body", {}))
azure_ad_token: str | None = None
if extra_body is not None:
azure_ad_token = extra_body.pop("azure_ad_token", None)
azure_ad_token = _popped_azure_ad_token(extra_body)
else:
azure_ad_token = get_secret("AZURE_AD_TOKEN")
@ -657,10 +674,10 @@ def create_thread(
api_version: str | None = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION")
extra_body: Final = optional_params.get("extra_body", {})
extra_body: Final = _EXTRA_BODY.validate_python(optional_params.get("extra_body", {}))
azure_ad_token: str | None = None
if extra_body is not None:
azure_ad_token = extra_body.pop("azure_ad_token", None)
azure_ad_token = _popped_azure_ad_token(extra_body)
else:
azure_ad_token = get_secret("AZURE_AD_TOKEN")
@ -804,10 +821,10 @@ def get_thread(
or get_secret("AZURE_API_KEY")
)
extra_body: Final = optional_params.get("extra_body", {})
extra_body: Final = _EXTRA_BODY.validate_python(optional_params.get("extra_body", {}))
azure_ad_token: str | None = None
if extra_body is not None:
azure_ad_token = extra_body.pop("azure_ad_token", None)
azure_ad_token = _popped_azure_ad_token(extra_body)
else:
azure_ad_token = get_secret("AZURE_AD_TOKEN")
@ -982,10 +999,10 @@ def add_message(
or get_secret("AZURE_API_KEY")
)
extra_body: Final = optional_params.get("extra_body", {})
extra_body: Final = _EXTRA_BODY.validate_python(optional_params.get("extra_body", {}))
azure_ad_token: str | None = None
if extra_body is not None:
azure_ad_token = extra_body.pop("azure_ad_token", None)
azure_ad_token = _popped_azure_ad_token(extra_body)
else:
azure_ad_token = get_secret("AZURE_AD_TOKEN")
@ -1134,10 +1151,10 @@ def get_messages(
or get_secret("AZURE_API_KEY")
)
extra_body: Final = optional_params.get("extra_body", {})
extra_body: Final = _EXTRA_BODY.validate_python(optional_params.get("extra_body", {}))
azure_ad_token: str | None = None
if extra_body is not None:
azure_ad_token = extra_body.pop("azure_ad_token", None)
azure_ad_token = _popped_azure_ad_token(extra_body)
else:
azure_ad_token = get_secret("AZURE_AD_TOKEN")
@ -1333,10 +1350,10 @@ def run_thread(
or get_secret("AZURE_API_KEY")
)
extra_body: Final = optional_params.get("extra_body", {})
extra_body: Final = _EXTRA_BODY.validate_python(optional_params.get("extra_body", {}))
azure_ad_token = None
if extra_body is not None:
azure_ad_token = extra_body.pop("azure_ad_token", None)
azure_ad_token = _pop_azure_ad_token(extra_body)
else:
azure_ad_token = get_secret("AZURE_AD_TOKEN")

9
litellm/batches/batch_utils.py
View file

@ -12,6 +12,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils import batch_utils as core_batch_utils
from litellm.litellm_core_utils.cloud_storage_security import stored_s3_bucket_kwargs
from litellm.litellm_core_utils.core_helpers import decoded_json_object
from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details
from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo
@ -849,14 +850,14 @@ def _get_batch_job_usage_from_response_body(
return usage
def _get_anthropic_result_from_batch_results_line(batch_results_line: Mapping[str, Any]) -> Mapping[str, Any]:
def _get_anthropic_result_from_batch_results_line(batch_results_line: Mapping[str, object]) -> Mapping[str, object]:
"""
Get the ``result`` object from a line of an Anthropic message batch results JSONL file.
Anthropic batch results lines look like:
``{"custom_id": ..., "result": {"type": "succeeded", "message": {..., "usage": {...}}}}``
"""
return batch_results_line.get("result", None) or {}
return decoded_json_object(batch_results_line.get("result", None) or {})
def _get_response_from_batch_job_output_file(
@ -866,7 +867,9 @@ def _get_response_from_batch_job_output_file(
Get the response from the batch job output file
"""
if custom_llm_provider == "anthropic":
return _get_anthropic_result_from_batch_results_line(batch_job_output_file).get("message", None) or {}
return decoded_json_object(
_get_anthropic_result_from_batch_results_line(batch_job_output_file).get("message", None) or {}
)
if custom_llm_provider == "bedrock":
return batch_job_output_file.get("modelOutput", None) or {}
_response: Final = cast( # cast-ok: batch output response comes from provider JSON

13
litellm/batches/main.py
View file

@ -15,13 +15,15 @@ import contextvars
import os
from collections.abc import Coroutine
from functools import partial
from typing import Any, Final, Literal, cast
from typing import Any, Final, Literal, Protocol, cast
import httpx
from openai.types.batch import BatchRequestCounts
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.batches.handler import AnthropicBatchesHandler
from litellm.llms.azure.batches.handler import AzureBatchesAPI
@ -60,10 +62,15 @@ azure_batches_instance: Final = AzureBatchesAPI()
vertex_ai_batches_instance: Final = VertexAIBatchPrediction(gcs_bucket_name="")
anthropic_batches_instance: Final = AnthropicBatchesHandler()
xai_batches_instance: Final = XAIBatchesHandler()
_EXTRA_BODY: Final = TypeAdapter(object)
base_llm_http_handler = BaseLLMHTTPHandler()
#################################################
class _PopsKeys(Protocol):
def pop(self, key: str, default: None, /) -> object: ...
def _resolve_timeout(
optional_params: GenericLiteLLMParams,
kwargs: dict[str, Any],
@ -485,9 +492,9 @@ def _handle_retrieve_batch_providers_without_provider_config(
or get_secret_str("AZURE_API_KEY")
)
extra_body: Final = optional_params.get("extra_body", {})
extra_body: Final = _EXTRA_BODY.validate_python(optional_params.get("extra_body", {}))
if extra_body is not None:
extra_body.pop("azure_ad_token", None)
require_attributes(extra_body, _PopsKeys, "pop").pop("azure_ad_token", None)
else:
get_secret_str("AZURE_AD_TOKEN")

28
litellm/caching/caching_handler.py
View file

@ -19,7 +19,7 @@ import datetime
import inspect
import time
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Generator, Mapping
from typing import TYPE_CHECKING, Any, Final, Optional, TypeVar
from typing import TYPE_CHECKING, Any, Final, Optional, Protocol, TypeVar
from pydantic import ConfigDict, SkipValidation, TypeAdapter, ValidationError
from typing_extensions import TypedDict
@ -72,6 +72,8 @@ from litellm.litellm_core_utils.core_helpers import (
as_object_list,
as_str_mapping,
get_parent_otel_span_from_kwargs,
json_contains,
require_attributes,
)
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
@ -246,6 +248,15 @@ def _stream_replay_requested(kwargs: Mapping[str, object]) -> bool:
return converted_stream_requested(kwargs) and not kwargs.get("_agentic_loop_depth")
class _CacheControls(Protocol):
def get(self, key: str, default: bool, /) -> object: ...
def _cache_reads_disabled(kwargs: Mapping[str, object]) -> bool:
cache_controls: Final = require_attributes(kwargs.get("cache", {}), _CacheControls, "get")
return cache_controls.get("no-cache", False) is True
def _should_defer_streaming_cache_hit_callbacks(*, cached_result: object) -> bool:
"""
When the cache hit is replayed as a stream, do not run success callbacks at cache-hit time.
@ -403,7 +414,7 @@ class LLMCachingHandler:
if (
(kwargs.get("caching", None) is None and litellm.cache is not None) or kwargs.get("caching", False) is True
) and (
kwargs.get("cache", {}).get("no-cache", False) is not True
not _cache_reads_disabled(kwargs)
): # allow users to control returning cached responses from the completion function
args = args or ()
final_embedding_cached_response: EmbeddingResponse | None = None
@ -521,7 +532,7 @@ class LLMCachingHandler:
kwargs: dict[str, Any],
args: tuple[object, ...] | None = None,
) -> CachingHandlerResponse:
cached_result: Any | None = None
cached_result: object | None = None
# Check if caching should be performed BEFORE doing expensive kwargs copy
if _is_response_cache_excluded(model=model, kwargs=kwargs):
@ -550,7 +561,7 @@ class LLMCachingHandler:
verbose_logger.debug("LiteLLM Cache: cached response has no output, treating it as a miss")
return CachingHandlerResponse(cached_result=None)
if cached_result is not None:
if "detail" in cached_result:
if json_contains(cached_result, "detail"):
# implies an error occurred
pass
else:
@ -1460,6 +1471,7 @@ class LLMCachingHandler:
Returns:
None
"""
request_kwargs: Final = dict[str, object](kwargs)
litellm_params: Final = {
"logger_fn": kwargs.get("logger_fn", None),
"acompletion": is_async,
@ -1473,7 +1485,7 @@ class LLMCachingHandler:
if litellm.cache is not None:
litellm_params["preset_cache_key"] = (
self.preset_cache_key or litellm.cache.get_preset_cache_key_from_kwargs(**kwargs)
self.preset_cache_key or litellm.cache.get_preset_cache_key_from_kwargs(**request_kwargs)
)
else:
litellm_params["preset_cache_key"] = None
@ -1483,11 +1495,11 @@ class LLMCachingHandler:
user=kwargs.get("user", None),
optional_params={},
litellm_params=litellm_params,
input=(kwargs.get("messages", "") if not is_embedding else kwargs.get("input", "")),
api_key=kwargs.get("api_key", None),
input=(request_kwargs.get("messages", "") if not is_embedding else request_kwargs.get("input", "")),
api_key=request_kwargs.get("api_key", None),
original_response=str(cached_result),
additional_args=None,
stream=kwargs.get("stream", False),
stream=request_kwargs.get("stream", False),
custom_llm_provider=custom_llm_provider,
)

8
litellm/caching/qdrant_semantic_cache.py
View file

@ -41,6 +41,7 @@ from .base_cache import BaseCache
_WAIT_FOR_INDEXING: Final = MappingProxyType({"wait": "true"})
_EMBEDDING_VECTOR: Final = TypeAdapter(object)
_CACHED_RESPONSE: Final = TypeAdapter(object)
if TYPE_CHECKING:
from litellm.router import Router
@ -175,14 +176,13 @@ class QdrantSemanticCache(BaseCache):
else:
raise Exception("Error while creating new collection")
def _get_cache_logic(self, cached_response: Any):
def _get_cache_logic(self, cached_response: Any) -> object:
if cached_response is None:
return cached_response
try:
cached_response = json.loads(cached_response) # Convert string to dictionary
return _CACHED_RESPONSE.validate_python(json.loads(cached_response))
except Exception:
cached_response = ast.literal_eval(cached_response)
return cached_response
return _CACHED_RESPONSE.validate_python(ast.literal_eval(cached_response))
def _get_qdrant_cache_key_filter(self, key: str) -> dict:
return {

9
litellm/google_genai/adapters/handler.py
View file

@ -2,6 +2,7 @@ from collections.abc import AsyncIterator, Coroutine, Mapping
from typing import Final, cast
from openai.types.chat import ChatCompletionStreamOptionsParam
from typing_extensions import TypedDict
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -15,6 +16,10 @@ from .transformation import GoogleGenAIAdapter
GOOGLE_GENAI_ADAPTER: Final = GoogleGenAIAdapter()
class _TranslatedCompletionKwargs(TypedDict, total=False, extra_items=object):
pass
def _mark_failure_logged_by_inner_completion(adapter_kwargs: Mapping[str, object]) -> None:
outer_logging_obj: Final = adapter_kwargs.get("litellm_logging_obj")
if isinstance(outer_logging_obj, LiteLLMLoggingObj):
@ -55,7 +60,7 @@ class GenerateContentToCompletionHandler:
**(extra_kwargs or {}),
)
completion_kwargs: Final = dict(completion_request)
completion_kwargs: Final = dict[str, object](completion_request)
# Forward extra_kwargs that should be passed to completion call
if extra_kwargs is not None:
@ -76,7 +81,7 @@ class GenerateContentToCompletionHandler:
if stream_options is not None:
completion_kwargs["stream_options"] = stream_options
return GenerateContentCompletionKwargs(**completion_kwargs)
return GenerateContentCompletionKwargs(**_TranslatedCompletionKwargs(**completion_kwargs))
@staticmethod
async def async_generate_content_handler(

13
litellm/google_genai/main.py
View file

@ -40,6 +40,7 @@ else:
# Initialize any necessary instances or variables here
base_llm_http_handler = BaseLLMHTTPHandler()
_GENERATE_CONTENT_RESULT: Final = TypeAdapter(object)
_GENERATE_CONTENT_PAYLOAD: Final = TypeAdapter(object)
#################################################
@ -368,8 +369,8 @@ def generate_content(
response: Final = _GENERATE_CONTENT_RESULT.validate_python(
base_llm_http_handler.generate_content_handler(
model=setup_result.model,
contents=contents,
tools=tools,
contents=_GENERATE_CONTENT_PAYLOAD.validate_python(contents),
tools=_GENERATE_CONTENT_PAYLOAD.validate_python(tools),
generate_content_provider_config=setup_result.generate_content_provider_config,
generate_content_config_dict=setup_result.generate_content_config_dict,
custom_llm_provider=setup_result.custom_llm_provider,
@ -465,10 +466,10 @@ async def agenerate_content_stream(
return _GENERATE_CONTENT_RESULT.validate_python(
await base_llm_http_handler.generate_content_handler(
model=setup_result.model,
contents=contents,
contents=_GENERATE_CONTENT_PAYLOAD.validate_python(contents),
generate_content_provider_config=setup_result.generate_content_provider_config,
generate_content_config_dict=setup_result.generate_content_config_dict,
tools=tools,
tools=_GENERATE_CONTENT_PAYLOAD.validate_python(tools),
custom_llm_provider=setup_result.custom_llm_provider,
litellm_params=setup_result.litellm_params,
logging_obj=setup_result.litellm_logging_obj,
@ -556,10 +557,10 @@ def generate_content_stream(
return _GENERATE_CONTENT_RESULT.validate_python(
base_llm_http_handler.generate_content_handler(
model=setup_result.model,
contents=contents,
contents=_GENERATE_CONTENT_PAYLOAD.validate_python(contents),
generate_content_provider_config=setup_result.generate_content_provider_config,
generate_content_config_dict=setup_result.generate_content_config_dict,
tools=tools,
tools=_GENERATE_CONTENT_PAYLOAD.validate_python(tools),
custom_llm_provider=setup_result.custom_llm_provider,
litellm_params=setup_result.litellm_params,
logging_obj=setup_result.litellm_logging_obj,

6
litellm/harness/endpoint.py
View file

@ -37,7 +37,7 @@ from litellm.constants import (
from litellm.harness.context import GatewayTarget
from litellm.harness.errors import HarnessError, HarnessInstallFailed
from litellm.harness.types import Harness, Usage
from litellm.litellm_core_utils.core_helpers import has_attributes
from litellm.litellm_core_utils.core_helpers import decoded_json_object, has_attributes
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
from litellm.types.llms.custom_http import httpxSpecialProvider
@ -722,7 +722,7 @@ class ModelEndpoint:
return await litellm.anthropic.messages.acreate(**kwargs)
if route == ROUTE_CHAT:
if not kwargs.get("stream"):
return await litellm.acompletion(**kwargs)
return await litellm.acompletion(**decoded_json_object(kwargs))
stream_options = kwargs.get("stream_options") or {} # mutable-ok: empty default for a JSON field
return await litellm.acompletion(
**{
@ -731,7 +731,7 @@ class ModelEndpoint:
STREAM_USAGE_INJECTED_FIELD: "include_usage" not in stream_options,
}
)
return await litellm.aresponses(**kwargs)
return await litellm.aresponses(**decoded_json_object(kwargs))
async def _call_sdk(self, route: str, body: Mapping[str, object]) -> Response:
kwargs = self._sdk_kwargs(body)

11
litellm/images/main.py
View file

@ -6,6 +6,7 @@ from functools import partial
from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload
from pydantic import TypeAdapter, ValidationError
from typing_extensions import TypedDict
if TYPE_CHECKING:
from litellm.images.utils import ImageEditRequestUtils
@ -698,7 +699,7 @@ async def aimage_variation(*args, **kwargs) -> ImageResponse:
init_response = await loop.run_in_executor(None, func_with_context)
if isinstance(init_response, dict) or isinstance(init_response, ImageResponse): ## CACHING SCENARIO
if isinstance(init_response, dict):
init_response = ImageResponse(**init_response)
init_response = _image_response_from_fields(init_response)
response = init_response
elif asyncio.iscoroutine(init_response):
response = await init_response
@ -717,6 +718,14 @@ async def aimage_variation(*args, **kwargs) -> ImageResponse:
)
class _ImageResponseFields(TypedDict, total=False, extra_items=object):
pass
def _image_response_from_fields(fields: Mapping[str, object]) -> ImageResponse:
return ImageResponse(**_ImageResponseFields(**fields))
@client
def image_variation(
image: FileTypes,

7
litellm/integrations/clickhouse/clickhouse_spend_logger.py
View file

@ -13,6 +13,7 @@ from types import MappingProxyType
from typing import Any, Final, Protocol
from pydantic import JsonValue, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
@ -57,6 +58,10 @@ class _SupportsItems(Protocol):
def items(self) -> Iterable[tuple[object, object]]: ...
class _SpendLogKwargs(TypedDict, total=False, extra_items=object):
standard_logging_object: ReadOnly[StandardLoggingPayload | None]
def _request_metadata_fields(value: object) -> Iterator[tuple[str, JsonValue]]:
if value is None:
return
@ -243,7 +248,7 @@ class ClickHouseSpendLogger(ClickHouseBatchLogger):
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time) -> None:
self._log(kwargs)
def _log(self, kwargs: Mapping[str, Any]) -> None:
def _log(self, kwargs: _SpendLogKwargs) -> None:
try:
payload = kwargs.get("standard_logging_object")
if payload is None or _is_trace_ingest(payload):

40
litellm/integrations/cloudzero/cz_resource_names.py
View file

@ -17,10 +17,16 @@
"""CloudZero Resource Names (CZRN) generation and validation for LiteLLM resources."""
import re
from collections.abc import Mapping
from enum import Enum
from typing import Any, Final, cast
from typing import Final, Protocol, cast
import litellm
from litellm.litellm_core_utils.core_helpers import require_attributes
class _SupportsLower(Protocol):
def lower(self) -> str: ...
class CZEntityType(str, Enum):
@ -35,7 +41,7 @@ class CZRNGenerator:
def __init__(self):
"""Initialize CZRN generator."""
def create_from_litellm_data(self, row: dict[str, Any]) -> str:
def create_from_litellm_data(self, row: Mapping[str, object]) -> str:
"""Create a CZRN from LiteLLM daily spend data.
CZRN format: czrn:<service-type>:<provider>:<region>:<owner-account-id>:<resource-type>:<cloud-local-id>
@ -49,19 +55,21 @@ class CZRNGenerator:
- cloud-local-id: model
"""
service_type: Final = "litellm"
provider: Final = self._normalize_provider(row.get("custom_llm_provider", "unknown"))
provider: Final = self._normalize_provider(
require_attributes(row.get("custom_llm_provider", "unknown"), _SupportsLower, "lower")
)
region: Final = "cross-region"
# Use the actual entity_id (team_id or user_id) as the owner account
team_id: Final = row.get("team_id", "unknown")
owner_account_id: Final = self._normalize_component(team_id)
owner_account_id: Final = self._normalize_owner(team_id)
resource_type: Final = "llm-usage"
# Create a unique identifier with just the model (entity info already in owner_account_id)
model: Final = row.get("model", "unknown")
cloud_local_id: Final = model
cloud_local_id: Final = f"{model}"
return self.create_from_components(
service_type=service_type,
@ -112,7 +120,7 @@ class CZRNGenerator:
return cast(tuple[str, str, str, str, str, str], match.groups())
def _normalize_provider(self, provider: str) -> str:
def _normalize_provider(self, provider: _SupportsLower) -> str:
"""Normalize provider names to standard CZRN format."""
# Map common provider names to CZRN standards
provider_map: Final = {
@ -141,16 +149,14 @@ class CZRNGenerator:
return "unknown"
# Convert to lowercase unless uppercase is allowed
if not allow_uppercase:
component = component.lower()
return self._clean_component(component if allow_uppercase else component.lower())
# Replace invalid characters with hyphens
component = re.sub(r"[^a-zA-Z0-9-]", "-", component)
def _normalize_owner(self, owner: object) -> str:
if not owner:
return "unknown"
return self._clean_component(require_attributes(owner, _SupportsLower, "lower").lower())
# Remove consecutive hyphens
component = re.sub(r"-+", "-", component)
# Remove leading/trailing hyphens
component = component.strip("-")
return component or "unknown"
def _clean_component(self, component: str) -> str:
hyphenated: Final = re.sub(r"[^a-zA-Z0-9-]", "-", component)
collapsed: Final = re.sub(r"-+", "-", hyphenated)
return collapsed.strip("-") or "unknown"

2
litellm/integrations/custom_logger.py
View file

@ -759,7 +759,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
response: object,
plan: AgenticLoopPlan,
kwargs: dict,
) -> Any:
) -> object:
"""
Post-process the response returned by the agentic-loop follow-up call.

16
litellm/integrations/dotprompt/prompt_manager.py
View file

@ -5,14 +5,14 @@ Based on Google's GenAI Kit dotprompt implementation: https://google.github.io/d
import re
from collections.abc import Mapping
from pathlib import Path
from typing import Any, Final
from typing import Any, Final, Protocol
import yaml
from jinja2 import DictLoader, select_autoescape
from jinja2.sandbox import ImmutableSandboxedEnvironment
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.core_helpers import as_str_mapping, require_attributes
class _PromptFileJson(TypedDict):
@ -27,6 +27,14 @@ class PromptJsonEntry(TypedDict):
metadata: ReadOnly[dict[str, object]]
class _SupportsGet(Protocol):
def get(self, key: str, default: object = None, /) -> object: ...
def _metadata_section(metadata: Mapping[str, object], name: str) -> _SupportsGet:
return require_attributes(metadata.get(name, {}), _SupportsGet, "get")
def strip_version_suffix(prompt_id: str) -> str | None:
base, separator, version = prompt_id.rpartition(".v")
if separator and base and version.isdigit():
@ -51,8 +59,8 @@ class PromptTemplate:
restricted_keys: Final = ["model", "input", "output"]
self.model = self.metadata.get("model")
self.input_schema = self.metadata.get("input", {}).get("schema", {})
self.output_format = self.metadata.get("output", {}).get("format")
self.output_schema = self.metadata.get("output", {}).get("schema", {})
self.output_format = _metadata_section(self.metadata, "output").get("format")
self.output_schema = _metadata_section(self.metadata, "output").get("schema", {})
self.optional_params = {}
for key in self.metadata:
if key not in restricted_keys:

11
litellm/integrations/focus/destinations/gcs_destination.py
View file

@ -2,8 +2,9 @@
from __future__ import annotations
from collections.abc import Mapping
from datetime import timezone
from typing import Any, Final
from typing import Final
from litellm._logging import verbose_logger
from litellm.integrations.gcs_bucket.gcs_bucket_base import GCSBucketBase
@ -21,14 +22,14 @@ class FocusGCSDestination(GCSBucketBase, FocusDestination):
self,
*,
prefix: str,
config: dict[str, Any] | None = None,
config: Mapping[str, str] | None = None,
) -> None:
config = config or {}
bucket_name: Final = config.get("bucket_name")
resolved_config: Final[Mapping[str, str]] = config or {}
bucket_name: Final = resolved_config.get("bucket_name")
if not bucket_name:
raise ValueError("bucket_name must be provided for GCS destination")
super().__init__(bucket_name=bucket_name)
service_account_json: Final = config.get("service_account_json")
service_account_json: Final = resolved_config.get("service_account_json")
if service_account_json is not None:
self.path_service_account_json = service_account_json
self.prefix = prefix.rstrip("/")

77
litellm/integrations/galileo.py
View file

@ -6,7 +6,7 @@ import re
import uuid
from collections.abc import Callable, Mapping, Sequence
from datetime import datetime, timezone, tzinfo
from typing import Any, Final, Protocol, cast
from typing import Final, Protocol, cast
import httpx
from pydantic import ConfigDict, Field, TypeAdapter
@ -109,9 +109,58 @@ class LLMResponse(LiteLLMBaseModel):
user_metadata: dict[str, object] | None = None
class _GalileoRecord(TypedDict, total=False):
latency_ms: ReadOnly[int]
status_code: ReadOnly[int]
input_text: ReadOnly[str]
output_text: ReadOnly[str]
node_type: ReadOnly[str]
model: ReadOnly[str]
num_input_tokens: ReadOnly[int]
num_output_tokens: ReadOnly[int]
num_total_tokens: ReadOnly[int]
cost: ReadOnly[float | None]
output_logprobs: ReadOnly[Mapping[str, object] | None]
created_at: ReadOnly[str]
tags: ReadOnly[Sequence[str] | None]
user_metadata: ReadOnly[Mapping[str, object] | None]
messages: ReadOnly[Sequence[object]]
class _GalileoLoggingKwargs(TypedDict, total=False, extra_items=object):
standard_logging_object: ReadOnly[GalileoStandardLoggingFields | None]
def _record_messages(messages: object) -> object:
return messages.get("messages") if isinstance(messages, dict) else messages
def _galileo_record(response: LLMResponse, messages: object) -> _GalileoRecord:
record: Final[_GalileoRecord] = {
"latency_ms": response.latency_ms,
"status_code": response.status_code,
"input_text": response.input_text,
"output_text": response.output_text,
"node_type": response.node_type,
"model": response.model,
"num_input_tokens": response.num_input_tokens,
"num_output_tokens": response.num_output_tokens,
"num_total_tokens": response.num_total_tokens,
"cost": response.cost,
"output_logprobs": response.output_logprobs,
"created_at": response.created_at,
"tags": response.tags,
"user_metadata": response.user_metadata,
}
record_messages: Final = _record_messages(messages)
if isinstance(record_messages, list) and record_messages:
return {**record, "messages": record_messages}
return record
class GalileoObserve(CustomLogger):
def __init__(self) -> None:
self.in_memory_records: list[Mapping[str, object]] = []
self.in_memory_records: list[_GalileoRecord] = []
self.batch_size = 1
self.api_key = os.getenv("GALILEO_API_KEY")
self.project_id = os.getenv("GALILEO_PROJECT_ID")
@ -278,7 +327,7 @@ class GalileoObserve(CustomLogger):
return created_at
@staticmethod
def _token_metrics_from_record(record: Mapping[str, Any]) -> dict[str, object]:
def _token_metrics_from_record(record: _GalileoRecord) -> dict[str, object]:
num_input_tokens: Final = int(record.get("num_input_tokens") or 0)
num_output_tokens: Final = int(record.get("num_output_tokens") or 0)
num_total_tokens = int(record.get("num_total_tokens") or 0)
@ -296,7 +345,7 @@ class GalileoObserve(CustomLogger):
@staticmethod
def _record_to_v2_span(
record: Mapping[str, Any],
record: _GalileoRecord,
*,
trace_id: str,
span_id: str,
@ -322,12 +371,13 @@ class GalileoObserve(CustomLogger):
**GalileoObserve._token_metrics_from_record(record),
},
}
if record.get("tags"):
span["tags"] = record["tags"]
tags: Final = record.get("tags")
if tags:
span["tags"] = tags
return span
@staticmethod
def _record_to_v2_trace(record: Mapping[str, Any]) -> dict[str, object]:
def _record_to_v2_trace(record: _GalileoRecord) -> dict[str, object]:
trace_id: Final = str(uuid.uuid4())
span_id: Final = str(uuid.uuid4())
created_at: Final = GalileoObserve._normalize_created_at(record.get("created_at", ""))
@ -347,7 +397,7 @@ class GalileoObserve(CustomLogger):
"spans": [GalileoObserve._record_to_v2_span(record, trace_id=trace_id, span_id=span_id)],
}
def _build_traces_payload(self, records: Sequence[Mapping[str, object]]) -> dict[str, object]:
def _build_traces_payload(self, records: Sequence[_GalileoRecord]) -> dict[str, object]:
payload: Final[dict[str, object]] = {
"traces": [self._record_to_v2_trace(record) for record in records],
"logging_method": "api_direct",
@ -668,7 +718,7 @@ class GalileoObserve(CustomLogger):
async def async_log_success_event(
self,
kwargs: Mapping[str, object],
kwargs: _GalileoLoggingKwargs,
response_obj: object,
start_time: object,
end_time: object,
@ -686,7 +736,7 @@ class GalileoObserve(CustomLogger):
async def _async_log_success_event_impl(
self,
kwargs: Mapping[str, Any],
kwargs: _GalileoLoggingKwargs,
response_obj: object,
start_time: object,
end_time: object,
@ -752,12 +802,7 @@ class GalileoObserve(CustomLogger):
created_at=GalileoObserve._format_created_at(start_ts),
)
request_dict: Final = request_record.model_dump()
if isinstance(messages, dict):
messages = messages.get("messages")
if isinstance(messages, list) and messages:
request_dict["messages"] = messages
self.in_memory_records.append(request_dict)
self.in_memory_records.append(_galileo_record(request_record, messages))
verbose_logger.debug("Galileo Logger: queued record, in_memory=%d", len(self.in_memory_records))
# Bound the buffer so persistent flush failures cannot grow it

23
litellm/integrations/gitlab/gitlab_prompt_manager.py
View file

@ -3,13 +3,14 @@ GitLab prompt manager with configurable prompts folder.
"""
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final, TypeVar
from typing import TYPE_CHECKING, Any, Final, Protocol, TypeVar
from jinja2 import DictLoader, select_autoescape
from jinja2.sandbox import ImmutableSandboxedEnvironment
from typing_extensions import ReadOnly, TypedDict
from litellm.integrations.custom_prompt_management import CustomPromptManagement
from litellm.litellm_core_utils.core_helpers import require_attributes
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -40,6 +41,18 @@ class GitLabCachedPrompt(TypedDict):
optional_params: ReadOnly[Mapping[str, object]]
class _SupportsGet(Protocol):
def get(self, key: str, default: object = None, /) -> object: ...
def _input_schema(metadata: Mapping[str, object]) -> object:
return require_attributes(metadata.get("input", {}), _SupportsGet, "get").get("schema", {})
def _optional_params(metadata: Mapping[str, object]) -> Mapping[str, object]:
return {k: v for k, v in metadata.items() if k not in ["model", "input", "content"]}
def encode_prompt_id(raw_id: str) -> str:
"""Convert GitLab path IDs like 'invoice/extract' → 'gitlab::invoice::extract'"""
if raw_id.startswith(GITLAB_PREFIX):
@ -68,8 +81,8 @@ class GitLabPromptTemplate:
self.model = model or metadata.get("model")
self.temperature = metadata.get("temperature")
self.max_tokens = metadata.get("max_tokens")
self.input_schema = metadata.get("input", {}).get("schema", {})
self.optional_params = {k: v for k, v in metadata.items() if k not in ["model", "input", "content"]}
self.input_schema = _input_schema(metadata)
self.optional_params = _optional_params(metadata)
def __repr__(self):
return f"GitLabPromptTemplate(id='{self.template_id}', model='{self.model}')"
@ -84,7 +97,7 @@ class GitLabTemplateManager:
def __init__(
self,
gitlab_config: dict[str, Any],
gitlab_config: Mapping[str, Any],
prompt_id: str | None = None,
ref: str | None = None,
gitlab_client: GitLabClient | None = None,
@ -297,7 +310,7 @@ class GitLabPromptManager(CustomPromptManagement):
def __init__(
self,
gitlab_config: dict[str, object],
gitlab_config: Mapping[str, object],
prompt_id: str | None = None,
ref: str | None = None, # tag/branch/SHA override
gitlab_client: GitLabClient | None = None,

7
litellm/integrations/langfuse/langfuse.py
View file

@ -22,6 +22,7 @@ from litellm.integrations.langfuse.langfuse_mock_client import (
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
filter_exceptions_from_params,
json_contains,
reconstruct_model_name,
safe_deep_copy,
)
@ -1311,11 +1312,11 @@ def _end_grounding_span(*, tracing: "LangfuseTracing", parent: "LangfuseObservat
).end()
def log_requester_metadata(clean_metadata: Mapping[str, Any]):
def log_requester_metadata(clean_metadata: Mapping[str, object]):
returned_metadata: Final = {}
requester_metadata: Final = clean_metadata.get("requester_metadata") or {}
requester_metadata: Final[object] = clean_metadata.get("requester_metadata") or {}
for k, v in clean_metadata.items():
if k not in requester_metadata:
if not json_contains(requester_metadata, k):
returned_metadata[k] = v
returned_metadata.update({"requester_metadata": requester_metadata})

19
litellm/integrations/opik/opik_payload_builder/extractors.py
View file

@ -2,15 +2,28 @@
import json
from collections.abc import Mapping
from typing import Any, Final
from typing import Any, Final, Protocol
from pydantic import TypeAdapter
from litellm import _logging
from litellm.litellm_core_utils.core_helpers import require_attributes
_DECODED_JSON: Final = TypeAdapter(object)
class _SpanIdentifiers(Protocol):
@property
def trace_id(self) -> str | None: ...
@property
def id(self) -> str | None: ...
def _trace_and_span_ids(span: _SpanIdentifiers) -> tuple[str | None, str | None]:
return span.trace_id, span.id
def normalize_provider_name(provider: str | None) -> str | None:
"""
Normalize LiteLLM provider names to standardized string names.
@ -77,7 +90,7 @@ def extract_opik_metadata(
def extract_span_identifiers(
current_span_data: Any,
current_span_data: object,
) -> tuple[str | None, str | None]:
"""
Extract trace_id and parent_span_id from current_span_data.
@ -95,7 +108,7 @@ def extract_span_identifiers(
return (current_span_data.get("trace_id"), current_span_data.get("id"))
try:
return current_span_data.trace_id, current_span_data.id
return _trace_and_span_ids(require_attributes(current_span_data, _SpanIdentifiers, "trace_id", "id"))
except AttributeError:
_logging.verbose_logger.warning("Unexpected current_span_data format: %s", type(current_span_data))
return None, None

12
litellm/interactions/litellm_responses_transformation/handler.py
View file

@ -2,8 +2,8 @@
Handler for transforming interactions API requests to litellm.responses requests.
"""
from collections.abc import AsyncIterator, Callable, Coroutine, Iterator
from typing import Any, Final
from collections.abc import AsyncIterator, Callable, Coroutine, Iterator, Mapping
from typing import Final
import litellm
from litellm.interactions.litellm_responses_transformation.streaming_iterator import (
@ -72,6 +72,7 @@ class LiteLLMResponsesInteractionsHandler:
model=model,
input=input,
optional_params=optional_params,
custom_llm_provider=custom_llm_provider,
**kwargs,
)
@ -108,10 +109,11 @@ class LiteLLMResponsesInteractionsHandler:
async def async_interactions_api_handler(
self,
responses_request: dict[str, Any],
responses_request: Mapping[str, object],
model: str,
input: InteractionInput | None,
optional_params: InteractionsAPIOptionalRequestParams,
custom_llm_provider: str | None = None,
**kwargs,
) -> InteractionsAPIResponse | AsyncIterator[InteractionsAPIStreamingResponse]:
"""Async handler for interactions API requests."""
@ -128,7 +130,7 @@ class LiteLLMResponsesInteractionsHandler:
litellm_custom_stream_wrapper=responses_response,
request_input=input,
optional_params=optional_params,
custom_llm_provider=responses_request.get("custom_llm_provider"),
custom_llm_provider=custom_llm_provider,
litellm_metadata=kwargs.get("litellm_metadata", {}),
)
@ -139,5 +141,5 @@ class LiteLLMResponsesInteractionsHandler:
return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response(
responses_response=responses_api_response,
model=model,
custom_llm_provider=responses_request.get("custom_llm_provider"),
custom_llm_provider=custom_llm_provider,
)

15
litellm/litellm_core_utils/chat_completion_agentic_loop.py
View file

@ -99,6 +99,11 @@ def _post_hook_overridden(callback: CustomLogger) -> bool:
_STREAM_OPTIONS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
_HOOK_RESPONSE: Final = TypeAdapter(object)
def _plan_response_override(plan: AgenticLoopPlan) -> object | None:
return plan.response_override
def _stashed_stream_options(kwargs: Mapping[str, object]) -> Mapping[str, object] | None:
@ -396,8 +401,10 @@ async def _execute_chat_completion_agentic_plan(
)
if _post_hook_overridden(callback):
try:
response_followup = await callback.async_post_agentic_loop_response_hook(
response=response_followup, plan=plan, kwargs=kwargs
response_followup = _HOOK_RESPONSE.validate_python(
await callback.async_post_agentic_loop_response_hook(
response=response_followup, plan=plan, kwargs=kwargs
)
)
except Exception as e:
_call_id = getattr(logging_obj, "litellm_call_id", "unknown")
@ -646,8 +653,8 @@ async def run_chat_completion_agentic_hooks(
kwargs=hook_kwargs,
)
if plan.response_override is not None:
return plan.response_override
if (response_override := _plan_response_override(plan)) is not None:
return response_override
if plan.terminate:
verbose_logger.debug(
"Agentic chat loop terminated by callback=%s reason=%s", type(callback).__name__, plan.stop_reason

36
litellm/litellm_core_utils/core_helpers.py
View file

@ -102,7 +102,7 @@ _LAX_BOOL: Final = TypeAdapter(bool)
def as_str_mapping(value: object) -> Mapping[str, object] | None:
return value if isinstance(value, Mapping) else None
return value if isinstance(value, (dict, Mapping)) else None
def as_object_list(value: object) -> Sequence[object] | None:
@ -180,7 +180,24 @@ def require_attributes(value: object, shape: type[_Shape], *names: str) -> _Shap
raise AttributeError(f"'{type(value).__name__}' object has no attribute '{missing}'", name=missing, obj=value)
class KeywordArguments(Protocol):
def keys(self) -> Iterable[str]: ...
def __getitem__(self, key: str, /) -> object: ...
def keyword_arguments(value: object, target: type[object]) -> KeywordArguments:
if has_attributes(value, KeywordArguments, "keys"):
return value
raise TypeError(
f"{target.__module__}.{target.__qualname__}() argument after ** must be a mapping, not {type(value).__name__}"
)
def iter_items(value: object) -> Iterable[object]:
listed: Final = as_object_list(value)
if listed is not None:
return listed
if isinstance(value, _SupportsIter):
return value
if isinstance(value, _SupportsGetItem):
@ -746,17 +763,22 @@ def process_response_headers(
return {**additional_headers, **({"request-id": request_id} if request_id is not None else {})}
def preserve_upstream_non_openai_attributes(
model_response: "ModelResponseStream", original_chunk: "ModelResponseStream"
):
def preserve_upstream_non_openai_attributes(model_response: "ModelResponseStream", original_chunk: object):
"""
Preserve non-OpenAI attributes from the original chunk.
"""
# Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings
expected_keys: Final = frozenset(type(model_response).model_fields) | frozenset({"usage"})
chunk_model: Final = original_chunk if isinstance(original_chunk, BaseModel) else None
if chunk_model is None:
raise AttributeError(
f"type object '{type(original_chunk).__name__}' has no attribute 'model_fields'",
name="model_fields",
obj=type(original_chunk),
)
candidate_keys: Final = tuple(
key
for key in (*type(original_chunk).model_fields, *(original_chunk.__pydantic_extra__ or {}))
for key in (*type(chunk_model).model_fields, *(chunk_model.__pydantic_extra__ or {}))
if key not in expected_keys
)
if not candidate_keys:
@ -766,7 +788,7 @@ def preserve_upstream_non_openai_attributes(
key,
cast( # cast-ok: Pydantic extras are scalar-checked before assignment
object,
getattr(original_chunk, key),
getattr(chunk_model, key),
),
)
for key in candidate_keys
@ -774,7 +796,7 @@ def preserve_upstream_non_openai_attributes(
if all(value is None or isinstance(value, (str, int, float, bool)) for _, value in candidate_values):
_assign_attributes(model_response, candidate_values)
return
_assign_attributes(model_response, original_chunk.model_dump(include=frozenset(candidate_keys)).items())
_assign_attributes(model_response, chunk_model.model_dump(include=set(candidate_keys)).items())
def _assign_attributes(target: BaseModel, values: Iterable[tuple[str, object]]) -> None:

34
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -17,7 +17,7 @@ from litellm.litellm_core_utils.bug_report import (
build_bug_report,
should_report_bug,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, json_contains, subscript
from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string
from litellm.llms.bedrock.common_utils import bedrock_model_reaches_converse_by_default
from litellm.llms.bigmodel.common_utils import raise_bigmodel_business_error
@ -1859,26 +1859,32 @@ def _map_together_ai_exception(
extra_information: str,
) -> None:
try:
error_response = json.loads(error_str)
error_response = _DECODED_ERROR_BODY.validate_python(json.loads(error_str))
except Exception:
error_response = {"error": error_str}
if "error" in error_response and "`inputs` tokens + `max_new_tokens` must be <=" in error_response["error"]:
if json_contains(error_response, "error") and json_contains(
subscript(error_response, "error"), "`inputs` tokens + `max_new_tokens` must be <="
):
raise ContextWindowExceededError(
message=f"TogetherAIException - {error_response['error']}",
message=f"TogetherAIException - {subscript(error_response, 'error')}",
model=model,
llm_provider="together_ai",
response=getattr(original_exception, "response", None),
)
elif "error" in error_response and "invalid private key" in error_response["error"]:
elif json_contains(error_response, "error") and json_contains(
subscript(error_response, "error"), "invalid private key"
):
raise AuthenticationError(
message=f"TogetherAIException - {error_response['error']}",
message=f"TogetherAIException - {subscript(error_response, 'error')}",
llm_provider="together_ai",
model=model,
response=getattr(original_exception, "response", None),
)
elif "error" in error_response and "INVALID_ARGUMENT" in error_response["error"]:
elif json_contains(error_response, "error") and json_contains(
subscript(error_response, "error"), "INVALID_ARGUMENT"
):
raise BadRequestError(
message=f"TogetherAIException - {error_response['error']}",
message=f"TogetherAIException - {subscript(error_response, 'error')}",
model=model,
llm_provider="together_ai",
response=getattr(original_exception, "response", None),
@ -1890,13 +1896,13 @@ def _map_together_ai_exception(
llm_provider="together_ai",
)
elif (
"error" in error_response
and "API key doesn't match expected format." in error_response["error"]
or "error_type" in error_response
and error_response["error_type"] == "validation"
json_contains(error_response, "error")
and json_contains(subscript(error_response, "error"), "API key doesn't match expected format.")
or json_contains(error_response, "error_type")
and subscript(error_response, "error_type") == "validation"
):
raise BadRequestError(
message=f"TogetherAIException - {error_response['error']}",
message=f"TogetherAIException - {subscript(error_response, 'error')}",
model=model,
llm_provider="together_ai",
response=getattr(original_exception, "response", None),
@ -1910,7 +1916,7 @@ def _map_together_ai_exception(
)
elif original_exception.status_code == 422:
raise BadRequestError(
message=f"TogetherAIException - {error_response['error']}",
message=f"TogetherAIException - {subscript(error_response, 'error')}",
model=model,
llm_provider="together_ai",
response=getattr(original_exception, "response", None),

18
litellm/litellm_core_utils/json_validation_rule.py
View file

@ -1,9 +1,11 @@
import json
from typing import Any, Final
from collections.abc import Mapping
from typing import Final
from pydantic import TypeAdapter
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import is_str_keyed_dict
_DECODED_RESPONSE: Final = TypeAdapter(object)
@ -76,7 +78,7 @@ def normalize_json_schema_types(
return schema
def normalize_tool_schema(tool: dict[str, Any]) -> dict[str, Any]:
def normalize_tool_schema(tool: Mapping[str, object]) -> Mapping[str, object]:
"""
Normalize a tool's parameter schema to use standard JSON Schema lowercase types.
@ -92,10 +94,14 @@ def normalize_tool_schema(tool: dict[str, Any]) -> dict[str, Any]:
normalized_tool: Final = tool.copy()
# Normalize function parameters if present
if "function" in tool and isinstance(tool["function"], dict):
normalized_tool["function"] = tool["function"].copy()
if "parameters" in tool["function"]:
normalized_tool["function"]["parameters"] = normalize_json_schema_types(tool["function"]["parameters"])
if "function" not in tool:
return normalized_tool
function: Final = tool["function"]
if is_str_keyed_dict(function):
normalized_function: Final = function.copy()
normalized_tool["function"] = normalized_function
if "parameters" in function:
normalized_function["parameters"] = normalize_json_schema_types(function["parameters"])
return normalized_tool

152
litellm/litellm_core_utils/litellm_logging.py
View file

@ -25,6 +25,7 @@ from httpx import Response, Timeout
from openai import AsyncStream as OpenAIAsyncStream
from openai import Stream as OpenAIStream
from pydantic import BaseModel, JsonValue, TypeAdapter
from typing_extensions import TypedDict
import litellm
from litellm._internal_context import in_mirrored_call, outside_mirrored_call, post_response_phase
@ -91,8 +92,11 @@ from litellm.litellm_core_utils.classifier_logging import (
)
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
attribute_or_none,
decoded_json_object,
finite_cost,
get_provider_response_headers_from_hidden_params,
has_attributes,
is_expected_client_error,
native_messages_reported_cost,
provider_reported_usage_cost,
@ -103,6 +107,7 @@ from litellm.litellm_core_utils.core_helpers import (
require_attributes,
set_response_cost_in_hidden_params,
single_log_line,
subscript,
)
from litellm.litellm_core_utils.error_normalization import normalize_error
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
@ -669,6 +674,10 @@ class UsageObservation(Enum):
ESTIMATED = "estimated"
class _UsageFields(TypedDict, total=False, extra_items=object):
pass
class _A2AUsageSlot(Protocol):
usage: Mapping[str, object] | None
@ -685,6 +694,31 @@ class _ParamsLookup(Protocol):
def get(self, key: str, default: object, /) -> object: ...
class _HiddenParamsSlot(Protocol):
get: Callable[[str], object]
__setitem__: Callable[[str, object], None]
class _HiddenParamsCopier(Protocol):
copy: Callable[[], object]
class _MetadataDefaults(Protocol):
setdefault: Callable[[str, Mapping[str, object]], object]
class _MetadataSlot(Protocol):
__setitem__: Callable[[str, object], None]
class _MetadataUpdate(Protocol):
update: Callable[[Mapping[str, object]], None]
class _ExceptionHeaders(Protocol):
headers: object
def _callback_control_params(litellm_params: object) -> dict[str, object]:
mapping: Final = as_str_mapping(litellm_params)
if isinstance(mapping, dict):
@ -1682,8 +1716,10 @@ class Logging(LiteLLMLoggingBaseClass):
)
# log raw request to provider (like LangFuse) -- if opted in.
if self.log_raw_request_response is True or litellm.log_raw_request_response is True:
_litellm_params: Final = self.model_call_details.get("litellm_params", {})
_metadata: Final = _litellm_params.get("metadata", {}) or {}
_litellm_params: Final = require_attributes(
self._model_call_details_view.get("litellm_params", dict[str, object]()), _ParamsLookup, "get"
)
_metadata: Final = _litellm_params.get("metadata", dict[str, object]()) or dict[str, object]()
try:
self.model_call_details["raw_request_typed_dict"] = RawRequestTypedDict(
raw_request_api_base=self._get_masked_api_base(str(additional_args.get("api_base") or "")),
@ -1697,6 +1733,8 @@ class Logging(LiteLLMLoggingBaseClass):
error=None,
)
if should_redact_message_logging(self.model_call_details):
if not has_attributes(_metadata, _MetadataSlot, "__setitem__"):
raise TypeError(f"'{type(_metadata).__name__}' object does not support item assignment")
_metadata["raw_request"] = REDACTED_BY_LITELLM
else:
curl_command: Final = self._get_request_curl_command(
@ -1705,15 +1743,21 @@ class Logging(LiteLLMLoggingBaseClass):
additional_args=additional_args,
data=additional_args.get("complete_input_dict", {}),
)
_metadata["raw_request"] = redact_string(str(curl_command))
redacted_curl_command: Final = redact_string(str(curl_command))
if not has_attributes(_metadata, _MetadataSlot, "__setitem__"):
raise TypeError(f"'{type(_metadata).__name__}' object does not support item assignment")
_metadata["raw_request"] = redacted_curl_command
except Exception as e:
self.model_call_details["raw_request_typed_dict"] = RawRequestTypedDict(
error=str(e),
)
_metadata["raw_request"] = redact_string(
unlogged_raw_request: Final = redact_string(
f"Unable to Log \
raw request: {e}"
)
if not has_attributes(_metadata, _MetadataSlot, "__setitem__"):
raise TypeError(f"'{type(_metadata).__name__}' object does not support item assignment")
_metadata["raw_request"] = unlogged_raw_request
if getattr(self, "logger_fn", None) and callable(self.logger_fn):
try:
self.logger_fn(
@ -2344,7 +2388,7 @@ class Logging(LiteLLMLoggingBaseClass):
def _record_zero_cost_diagnostic(
self,
result: object,
response_cost: float | None,
response_cost: object,
*,
calculation_failed: bool = False,
litellm_model_name: str | None = None,
@ -2375,7 +2419,7 @@ class Logging(LiteLLMLoggingBaseClass):
def _zero_cost_finding(
self,
result: object,
response_cost: float | None,
response_cost: object,
*,
calculation_failed: bool,
litellm_model_name: str | None,
@ -3079,19 +3123,21 @@ class Logging(LiteLLMLoggingBaseClass):
elif self.relays_llm_passthrough and isinstance(result, Response):
from litellm.utils import ProviderConfigManager
provider_config: Final = ProviderConfigManager.get_provider_passthrough_config(
provider=self.model_call_details.get("custom_llm_provider", ""),
model=self.model,
)
if provider_config is not None:
logging_result = provider_config.logging_non_streaming_response(
passthrough_provider: Final = self._model_call_details_view.get("custom_llm_provider", "")
if isinstance(passthrough_provider, str):
provider_config: Final = ProviderConfigManager.get_provider_passthrough_config(
provider=passthrough_provider,
model=self.model,
custom_llm_provider=self.model_call_details.get("custom_llm_provider", ""),
httpx_response=result,
request_data=self.model_call_details.get("request_data", {}),
logging_obj=self,
endpoint=self.model_call_details.get("endpoint", ""),
)
if provider_config is not None:
logging_result = provider_config.logging_non_streaming_response(
model=self.model,
custom_llm_provider=passthrough_provider,
httpx_response=result,
request_data=self.model_call_details.get("request_data", {}),
logging_obj=self,
endpoint=self.model_call_details.get("endpoint", ""),
)
return logging_result
def _surface_response_headers_from_result(self, logging_result: object) -> None:
@ -3138,19 +3184,25 @@ class Logging(LiteLLMLoggingBaseClass):
"""
if logging_result is None:
return
hidden_params: Final = getattr(logging_result, HIDDEN_PARAMS_ATTR, None)
hidden_params: Final = attribute_or_none(logging_result, HIDDEN_PARAMS_ATTR)
if not hidden_params:
return
if self.model_call_details.get("litellm_params") is None:
return
metadata_hidden_params: Final = hidden_params.copy()
metadata_hidden_params: Final = require_attributes(
require_attributes(hidden_params, _HiddenParamsCopier, "copy").copy(), _HiddenParamsSlot, "get"
)
response_cost: Final[object] = self.model_call_details.get("response_cost")
if metadata_hidden_params.get("response_cost") is None and response_cost is not None:
metadata_hidden_params["response_cost"] = response_cost
litellm_params: Final = self.model_call_details["litellm_params"]
metadata: Final = litellm_params.get("metadata") or {}
litellm_params: Final = require_attributes(
self._model_call_details_view["litellm_params"], _HiddenParamsSlot, "get"
)
metadata: Final = litellm_params.get("metadata") or dict[str, object]()
litellm_params["metadata"] = metadata
if not has_attributes(metadata, _MetadataSlot, "__setitem__"):
raise TypeError(f"'{type(metadata).__name__}' object does not support item assignment")
metadata["hidden_params"] = metadata_hidden_params
def _process_hidden_params_and_response_cost(
@ -3166,7 +3218,9 @@ class Logging(LiteLLMLoggingBaseClass):
hidden_params: Final = getattr(logging_result, HIDDEN_PARAMS_ATTR, {})
if hidden_params:
if self.model_call_details.get("litellm_params") is not None:
self.model_call_details["litellm_params"].setdefault("metadata", {})
require_attributes(
self._model_call_details_view["litellm_params"], _MetadataDefaults, "setdefault"
).setdefault("metadata", {})
if self.model_call_details["litellm_params"]["metadata"] is None:
self.model_call_details["litellm_params"]["metadata"] = {}
self.model_call_details["litellm_params"]["metadata"]["hidden_params"] = getattr(
@ -3179,8 +3233,9 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["response_cost"] = 0.0
self._add_failed_attempts_spend(get_usage_object(completion_response=logging_result))
elif "response_cost" in hidden_params:
self.model_call_details["response_cost"] = hidden_params["response_cost"]
self._record_zero_cost_diagnostic(logging_result, hidden_params["response_cost"])
hidden_response_cost: Final = subscript(hidden_params, "response_cost")
self.model_call_details["response_cost"] = hidden_response_cost
self._record_zero_cost_diagnostic(logging_result, hidden_response_cost)
self._add_failed_attempts_spend(get_usage_object(completion_response=logging_result))
elif (existing_cost := self.model_call_details.get("response_cost")) is not None and (
existing_cost != 0 or self.call_type == CallTypes.pass_through.value # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped
@ -4659,10 +4714,23 @@ class Logging(LiteLLMLoggingBaseClass):
elif self.model_call_details.get("combined_usage_object") is None:
self.model_call_details["response_cost"] = 0
if hasattr(exception, "headers") and isinstance(exception.headers, dict):
raised_exception: Final[object] = exception
exception_headers: Final = (
as_str_mapping(raised_exception.headers)
if has_attributes(raised_exception, _ExceptionHeaders, "headers")
else None
)
if isinstance(exception_headers, dict):
self.model_call_details.setdefault("litellm_params", {})
metadata: Final = self.model_call_details["litellm_params"].get("metadata", {}) or {}
metadata.update(exception.headers)
metadata: Final = require_attributes(
require_attributes(self._model_call_details_view["litellm_params"], _ParamsLookup, "get").get(
"metadata", dict[str, object]()
)
or dict[str, object](),
_MetadataUpdate,
"update",
)
metadata.update(exception_headers)
## STANDARDIZED LOGGING PAYLOAD
@ -6412,7 +6480,7 @@ def build_custom_logger_compatible_class(
return callback
# Get global BitBucket config
gitlab_config: Final = getattr(litellm, "global_gitlab_config", None)
gitlab_config: Final = litellm.global_gitlab_config if hasattr(litellm, "global_gitlab_config") else None
if gitlab_config is None:
raise ValueError("Gitlab configuration not found. Please set litellm.global_gitlab_config first.")
@ -7198,11 +7266,15 @@ class StandardLoggingPayloadSetup:
if isinstance(reported, Usage):
return reported
normalized: Final = StandardLoggingPayloadSetup.get_usage_as_dict(response_obj)
return Usage(**normalized) if normalized else Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0)
return (
Usage(**_UsageFields(**decoded_json_object(normalized)))
if normalized
else Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0)
)
@staticmethod
def get_usage_as_dict(
response_obj: dict[str, Any] | None,
response_obj: Mapping[str, object] | None,
combined_usage_object: Usage | None = None,
) -> dict[str, Any]:
"""
@ -7226,19 +7298,23 @@ class StandardLoggingPayloadSetup:
reported: Final = response_obj.get("usage_info")
if isinstance(reported, BaseModel):
return reported.model_dump(exclude_none=True)
reported_fields: Final = as_str_mapping(reported)
return (
{key: value for key, value in reported.items() if value is not None}
if isinstance(reported, dict)
{key: value for key, value in reported_fields.items() if value is not None}
if isinstance(reported_fields, dict)
else dict(get_rerank_usage(_RESPONSE_MAPPING.validate_python(response_obj)) or _empty)
)
if isinstance(_raw, ResponseAPIUsage):
return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(_raw).model_dump()
if isinstance(_raw, dict):
if ResponseAPILoggingUtils.is_response_api_usage(_raw):
return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(_raw).model_dump()
if InteractionsUsageObjectTransformation.is_interactions_usage_object(_raw):
return InteractionsUsageObjectTransformation.transform_interactions_usage_object(_raw).model_dump()
return _raw
raw_fields: Final = as_str_mapping(_raw)
if isinstance(raw_fields, dict):
if ResponseAPILoggingUtils.is_response_api_usage(raw_fields):
return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(raw_fields).model_dump()
if InteractionsUsageObjectTransformation.is_interactions_usage_object(raw_fields):
return InteractionsUsageObjectTransformation.transform_interactions_usage_object(
raw_fields
).model_dump()
return raw_fields
if isinstance(_raw, Usage):
return _raw.model_dump()
return _empty

2
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -1429,7 +1429,7 @@ def get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str |
residency: Final = data_residency.lower()
if residency not in _VALID_DATA_RESIDENCIES:
return 1.0
multiplier: Final = model_info.get(f"regional_processing_uplift_multiplier_{residency}")
multiplier: Final[object] = model_info.get(f"regional_processing_uplift_multiplier_{residency}")
if multiplier is None:
return 1.0
if not isinstance(multiplier, (str, Buffer, SupportsFloat, SupportsIndex)):

19
litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py
View file

@ -18,6 +18,7 @@ from litellm.constants import (
STREAM_FINISH_CHOICE_FIELDS,
VLLM_CHOICE_FIELDS,
)
from litellm.litellm_core_utils.core_helpers import iter_items, subscript
from litellm.litellm_core_utils.hidden_params import get_hidden_params
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
extract_reasoning_content,
@ -68,6 +69,14 @@ _MODEL_RESPONSE_FIELDS: Final[frozenset] = frozenset(ModelResponse.model_fields.
_ANNOTATION_ENTRIES: Final = TypeAdapter(tuple[object, ...])
class _ImageResponseFields(TypedDict, total=False, extra_items=object):
pass
def _image_response_from_fields(fields: Mapping[str, object]) -> ImageResponse:
return ImageResponse(**_ImageResponseFields(**fields))
def _is_provider_specific_annotation(annotation: object) -> bool:
match annotation:
case {"type": "url_citation"}:
@ -639,7 +648,7 @@ class LiteLLMResponseObjectHandler:
model_response_dict: Final = model_response_object.model_dump()
model_response_dict.update(response_object)
model_response_object = ImageResponse(**model_response_dict)
model_response_object = _image_response_from_fields(model_response_dict)
return model_response_object
@staticmethod
@ -683,12 +692,12 @@ class LiteLLMResponseObjectHandler:
choices_list: Final[list[TextChoices]] = []
# Convert each choice to TextChoices
for choice in response["choices"]:
for choice in iter_items(subscript(response, "choices")):
text_choices = TextChoices()
text_choices["text"] = choice["message"]["content"]
text_choices["index"] = choice["index"]
text_choices["text"] = subscript(subscript(choice, "message"), "content")
text_choices["index"] = subscript(choice, "index")
text_choices["logprobs"] = transformed_logprobs
text_choices["finish_reason"] = choice["finish_reason"]
text_choices["finish_reason"] = subscript(choice, "finish_reason")
choices_list.append(text_choices)
text_completion_response["choices"] = choices_list

34
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -13,7 +13,7 @@ from types import MappingProxyType
from typing import Annotated, Any, Final, Protocol, TypeAlias, TypedDict, TypeVar, cast, overload
from jinja2.sandbox import ImmutableSandboxedEnvironment
from pydantic import BaseModel, BeforeValidator, ConfigDict, JsonValue, TypeAdapter, ValidationError, model_validator
from pydantic import BaseModel, BeforeValidator, ConfigDict, TypeAdapter, ValidationError, model_validator
from pydantic_core import ErrorDetails, PydanticCustomError
import litellm
@ -22,7 +22,12 @@ import litellm.types.llms
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.constants import REDACTED_BY_LITELLM
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, require_attributes
from litellm.litellm_core_utils.core_helpers import (
as_object_list,
as_str_mapping,
attribute_or_none,
require_attributes,
)
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
anthropic_system_messages,
find_anthropic_server_tool_result,
@ -1455,22 +1460,25 @@ def convert_to_gemini_tool_call_invoke(
raise Exception(f"Unable to convert openai tool calls={message} to gemini tool calls. Received error={e}")
def _json_children(node: JsonValue) -> Iterable[JsonValue]:
if isinstance(node, dict):
return node.values()
if isinstance(node, list):
return (child for child in node if not isinstance(child, list))
def _json_children(node: object) -> Iterable[object]:
mapping: Final = as_str_mapping(node)
if mapping is not None:
return mapping.values()
items: Final = as_object_list(node)
if items is not None:
return (child for child in items if not isinstance(child, list))
return ()
def _next_json_level(level: tuple[JsonValue, ...], _: None) -> tuple[JsonValue, ...]:
def _next_json_level(level: tuple[object, ...], _: None) -> tuple[object, ...]:
return tuple(itertools.chain.from_iterable(_json_children(node) for node in level))
def _is_part_reference(node: JsonValue) -> bool:
if not isinstance(node, dict):
def _is_part_reference(node: object) -> bool:
mapping: Final = as_str_mapping(node)
if mapping is None:
return False
ref: Final[JsonValue] = node.get("$ref")
ref: Final = mapping.get("$ref")
return isinstance(ref, str) and ref != ""
@ -1482,7 +1490,7 @@ def _may_spell_ref_key(raw_json: str) -> bool:
return '"$ref"' in raw_json or _REF_CHAR_ESCAPE.search(raw_json) is not None
def _contains_part_reference(payload: JsonValue) -> bool:
def _contains_part_reference(payload: object) -> bool:
"""Gemini reads a {"$ref": "<non-empty string>"} object in a function_response as a pointer to one of its parts,
unless it sits under a list nested directly in a list
@ -1739,7 +1747,7 @@ def convert_to_gemini_tool_call_result(
try:
if content_str.strip().startswith("{") or content_str.strip().startswith("["):
# Try to parse as JSON (for Computer Use structured responses)
parsed: Final[JsonValue] = json.loads(content_str)
parsed: Final = as_str_mapping(_DECODED_JSON.validate_python(json.loads(content_str)))
if isinstance(parsed, dict) and not (_may_spell_ref_key(content_str) and _contains_part_reference(parsed)):
response_data = parsed # Use the parsed JSON directly
else:

8
litellm/litellm_core_utils/prompt_templates/huggingface_template_handler.py
View file

@ -1,6 +1,6 @@
import json
from datetime import datetime
from typing import Any, Final, Literal
from typing import Final, Literal
from pydantic import TypeAdapter
from typing_extensions import NotRequired, ReadOnly, TypedDict
@ -28,6 +28,10 @@ class _ChatTemplateFileResult(TypedDict):
chat_template: NotRequired[ReadOnly[str]]
class _AddedToken(TypedDict, total=False, extra_items=object):
content: ReadOnly[str]
def strftime_now(fmt: str) -> str:
"""
Custom function for templates that need current date/time formatting (e.g., gpt-oss)
@ -159,7 +163,7 @@ async def aget_chat_template_file(hf_model_name: str) -> _ChatTemplateFileResult
_aget_chat_template_file = aget_chat_template_file
def extract_token_value(token_value: None | str | dict[str, Any]) -> str:
def extract_token_value(token_value: None | str | _AddedToken) -> str:
"""
Extract token string from various formats (string, dict, etc.)

34
litellm/litellm_core_utils/realtime_streaming.py
View file

@ -26,7 +26,7 @@ from litellm.constants import (
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
)
from litellm.litellm_core_utils.audio_utils.utils import normalized_audio_duration_seconds
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object, json_contains
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.llms.base_llm.realtime.transcription_protocol import RealtimeTranscriptionProtocolError
from litellm.llms.base_llm.realtime.transformation import (
@ -220,9 +220,13 @@ def _unscreened_audio_error_event(client_event_id: object) -> Mapping[str, objec
}
def _decode_json(payload: str) -> object:
return _DECODED_JSON.validate_python(json.loads(payload))
def _decode_json_object(payload: str) -> Mapping[str, object]:
"""Decode a realtime frame into its top-level field mapping."""
return json.loads(payload)
return decoded_json_object(_decode_json(payload))
_GUARDRAIL_TURN_DETECTION_GATE: Final[Mapping[str, object]] = MappingProxyType(
@ -511,7 +515,7 @@ class RealTimeStreaming:
self._held_client_content: tuple[str, ...] = ()
self._held_client_content_bytes: int = 0
self._held_client_content_deadline: float | None = None
self._replay_scope: Final = client_websocket_scope(websocket)
self._replay_scope: Final = client_websocket_scope(self.websocket)
self.backend_ws = backend_ws
self.logging_obj = logging_obj
self._logging_worker = logging_worker
@ -533,7 +537,7 @@ class RealTimeStreaming:
self._translation_usage_finalized = False
# Detect whether the client is explicitly opting into the beta protocol.
self._client_wants_beta = self.detect_beta_header(websocket)
self._client_wants_beta = self.detect_beta_header(self.websocket)
self._backend_uses_beta_protocol = not translation_session and (
self._client_wants_beta if backend_uses_beta_protocol is None else backend_uses_beta_protocol
)
@ -674,7 +678,7 @@ class RealTimeStreaming:
if not self._should_store_message(message_obj):
return
try:
event_type: Final = message_obj.get("type", "")
event_type: Final = decoded_json_object(message_obj).get("type", "")
if event_type in self._SESSION_EVENT_TYPES:
typed_obj: OpenAIRealtimeEvents = OpenAIRealtimeStreamSessionEvents(**message_obj)
else:
@ -1020,7 +1024,7 @@ class RealTimeStreaming:
sent = True
continue
try:
msg_obj = _decode_json_object(msg)
msg_obj = as_str_mapping(_decode_json(msg))
except (json.JSONDecodeError, TypeError):
msg_obj = None
if isinstance(msg_obj, dict) and self.provider_config.is_setup_message(msg_obj):
@ -1162,13 +1166,15 @@ class RealTimeStreaming:
return _session_update_frame(self._held_session_update or _NO_SESSION_FIELDS)
async def _hold_session_update(self, message: str) -> None:
client_event: Final = _decode_json_object(message)
client_event: Final = _decode_json(message)
if self._held_session_update_bytes + len(message.encode("utf-8")) > RealTimeStreaming._MAX_BUFFERED_BYTES:
await self.websocket.send_text(
json.dumps(_HELD_SESSION_UPDATE_TOO_LARGE.error_event(client_event.get("event_id")))
json.dumps(
_HELD_SESSION_UPDATE_TOO_LARGE.error_event(decoded_json_object(client_event).get("event_id"))
)
)
return
session: Final = client_event.get("session")
session: Final = decoded_json_object(client_event).get("session")
update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS
held: Final = _merge_session_fields(self._held_session_update or _NO_SESSION_FIELDS, update)
self._held_session_update = held
@ -1182,7 +1188,7 @@ class RealTimeStreaming:
_session_update_frame(session), self.model, None
)
return next(
(frame for frame in preview if isinstance(frame, str) and "setup" in _decode_json_object(frame)), None
(frame for frame in preview if isinstance(frame, str) and json_contains(_decode_json(frame), "setup")), None
)
def _leaves_applied_setup_unchanged(self, update: Mapping[str, object]) -> bool:
@ -1194,7 +1200,7 @@ class RealTimeStreaming:
return (
applied_setup is not None
and merged_setup is not None
and _decode_json_object(applied_setup) == _decode_json_object(merged_setup)
and _decode_json(applied_setup) == _decode_json(merged_setup)
)
async def _acknowledge_session_setup(self, session: Mapping[str, object]) -> None:
@ -1522,8 +1528,8 @@ class RealTimeStreaming:
``return_new_content_delta_events`` modality lookup, ...).
"""
try:
message_obj: Final = _decode_json_object(transformed_message)
if "setup" in message_obj:
message_obj: Final = _decode_json(transformed_message)
if json_contains(message_obj, "setup"):
self.session_configuration_request = transformed_message
except (json.JSONDecodeError, TypeError):
return
@ -1852,7 +1858,7 @@ class RealTimeStreaming:
def _parse_backend_event(raw_response: str) -> dict[str, object] | None:
"""Parse a backend frame once. Returns None for non-JSON or non-object frames."""
try:
event: Final = _decode_json_object(raw_response)
event: Final = as_str_mapping(_decode_json(raw_response))
except (json.JSONDecodeError, TypeError):
return None
return event if isinstance(event, dict) else None

2
litellm/litellm_core_utils/redact_messages.py
View file

@ -351,7 +351,7 @@ def _redacted_web_search_action(action: object) -> object:
def _redacted_web_search_call(item: object) -> object:
if isinstance(item, BaseModel):
action: Final = getattr(item, "action", None)
action: Final = attribute_or_none(item, "action")
return item if action is None else item.model_copy(update={"action": _redacted_web_search_action(action)})
mapping: Final = _as_mapping(item)
if mapping is None or mapping.get("action") is None:

20
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Final, Optional, Protocol, TypeAlias, TypedDic
from pydantic import BaseModel, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, Required
from typing_extensions import TypedDict as ExtraItemsTypedDict
import litellm
from litellm._logging import verbose_logger
@ -19,6 +20,7 @@ from litellm.litellm_core_utils.core_helpers import (
chunk_usage,
finite_cost,
provider_reported_usage_cost,
require_attributes,
stamp_litellm_bill_on_usage,
)
from litellm.litellm_core_utils.hidden_params import (
@ -82,6 +84,18 @@ if TYPE_CHECKING:
)
class _UsageFields(ExtraItemsTypedDict, total=False, extra_items=object):
pass
def _usage_fields(fields: Mapping[str, object]) -> _UsageFields:
return _UsageFields(**fields)
class _SupportsModelDump(Protocol):
def model_dump(self) -> Mapping[str, object]: ...
class _ThinkingBlockFragment(TypedDict, total=False):
type: str | None
data: str | None
@ -946,12 +960,12 @@ class ChunkProcessor:
usage_chunk = chunk.hidden_params.get("usage", None)
if isinstance(usage_chunk, dict):
return Usage(**usage_chunk)
return Usage(**_usage_fields(usage_chunk))
if isinstance(usage_chunk, UnreportedUsage):
return None
if usage_chunk is None or isinstance(usage_chunk, Usage):
return usage_chunk
return Usage(**usage_chunk.model_dump())
return Usage(**_usage_fields(require_attributes(usage_chunk, _SupportsModelDump, "model_dump").model_dump()))
def _calculate_usage_per_chunk(
self,
@ -1275,7 +1289,7 @@ class ChunkProcessor:
if value is not None
}
returned_usage = Usage(**returned_usage.model_dump(), **provider_pricing_fields)
returned_usage = Usage(**_usage_fields(returned_usage.model_dump()), **provider_pricing_fields)
return returned_usage

234
litellm/litellm_core_utils/streaming_handler.py
View file

@ -27,11 +27,17 @@ from litellm.constants import STREAM_CHUNK_CHOICE_FIELDS, STREAM_FINISH_CHOICE_F
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.core_helpers import (
UPSTREAM_USAGE_COST,
as_str_mapping,
attribute_or_none,
chunk_usage,
decoded_json_object,
has_attributes,
json_contains,
provider_reported_usage_cost,
require_attributes,
stamp_litellm_bill_on_usage,
subscript,
without,
)
from litellm.litellm_core_utils.hidden_params import (
HIDDEN_PARAMS_ATTR,
@ -195,6 +201,10 @@ def _response_choice_index(response_obj: Mapping[str, object]) -> int:
return _upstream_choice_index(response_obj.get("original_chunk"))
def _delta_choice_index(completion_fields: Mapping[str, object], response_fields: Mapping[str, object]) -> object:
return completion_fields.get("index", response_fields.get("choice_index"))
def _non_null_upstream_choice_fields(upstream_chunk: object, fields: Sequence[str]) -> Mapping[str, object]:
first_choice: Final = _upstream_first_choice(upstream_chunk)
upstream_extras: Final = first_choice.model_extra if first_choice is not None else None
@ -231,13 +241,21 @@ def _holds_only_scalars(fields: Iterable[tuple[str, object]]) -> bool:
return all(value is None or isinstance(value, (str, int, float, bool)) for _, value in fields)
class _KeywordFields(TypedDict, total=False, extra_items=object):
pass
def _keyword_fields(fields: Mapping[str, object]) -> _KeywordFields:
return _KeywordFields(**fields)
def _delta_from_upstream(upstream_delta: "BaseModel | _UpstreamDeltaLike | None") -> Delta | None:
if upstream_delta is None:
return Delta()
if not isinstance(upstream_delta, BaseModel):
return None
try:
return Delta(**dict(iter(upstream_delta)))
return Delta(**_keyword_fields(dict(iter(upstream_delta))))
except Exception:
return None
@ -275,7 +293,7 @@ def print_verbose(print_statement: object):
@dataclass(frozen=True, slots=True)
class _ProviderChunkParsed:
response_obj: dict[str, object]
response_obj: Mapping[str, object]
@dataclass(frozen=True, slots=True)
@ -286,19 +304,8 @@ class _ProviderChunkEarlyReturn:
_ProviderChunkResult = _ProviderChunkParsed | _ProviderChunkEarlyReturn
class _PredibaseStreamData(TypedDict):
token: NotRequired[Mapping[str, str]]
details: Mapping[str, str]
generated_text: str | None
error: str | None
class _NlpCloudStreamData(TypedDict):
generated_text: str
class _AlephAlphaStreamData(TypedDict):
completions: Sequence[Mapping[str, str]]
class _Replaceable(Protocol):
def replace(self, old: str, new: str, /) -> object: ...
class _DeltaDumpDict(TypedDict):
@ -365,6 +372,32 @@ class _VertexChunkLike(Protocol):
candidates: Sequence[_VertexCandidateLike]
class _TokenTotalsLike(Protocol):
prompt_tokens: int
completion_tokens: int
total_tokens: int
class _ParamsLookupLike(Protocol):
def get(self, key: str, default: object, /) -> object: ...
class _IdentifiedChunkLike(Protocol):
id: str
class _Decodable(Protocol):
def decode(self, encoding: str, /) -> str: ...
class _VllmRequestOutputLike(Protocol):
outputs: object
class _VllmCompletionOutputLike(Protocol):
text: object
class _ParsedChunkHiddenParams(LiteLLMBaseModel):
provider_specific_fields: Mapping[str, object] | None = None
upstream_provider: str | None = None
@ -783,28 +816,33 @@ class CustomStreamWrapper:
self.holding_chunk = ""
return hold, curr_chunk
def handle_predibase_chunk(self, chunk):
def handle_predibase_chunk(self, chunk: object):
try:
if not isinstance(chunk, str):
chunk = chunk.decode("utf-8") # DO NOT REMOVE this: This is required for HF inference API + Streaming
chunk = require_attributes(chunk, _Decodable, "decode").decode(
"utf-8"
) # DO NOT REMOVE this: This is required for HF inference API + Streaming
text = ""
is_finished = False
finish_reason = ""
print_verbose(f"chunk: {chunk}")
if chunk.startswith("data:"):
data_json: Final[_PredibaseStreamData] = json.loads(chunk[5:])
data_json: Final = _DECODED_JSON.validate_python(json.loads(chunk[5:]))
print_verbose(f"data json: {data_json}")
if "token" in data_json and "text" in data_json["token"]:
text = data_json["token"]["text"]
if data_json.get("details", False) and data_json["details"].get("finish_reason", False):
if json_contains(data_json, "token") and json_contains(subscript(data_json, "token"), "text"):
text = subscript(subscript(data_json, "token"), "text")
data_lookup: Final = require_attributes(data_json, _ParamsLookupLike, "get")
if data_lookup.get("details", False) and require_attributes(
subscript(data_json, "details"), _ParamsLookupLike, "get"
).get("finish_reason", False):
is_finished = True
finish_reason = data_json["details"]["finish_reason"]
elif data_json.get("generated_text", False): # if full generated text exists, then stream is complete
finish_reason = subscript(subscript(data_json, "details"), "finish_reason")
elif data_lookup.get("generated_text", False): # if full generated text exists, then stream is complete
text = "" # don't return the final bos token
is_finished = True
finish_reason = "stop"
elif data_json.get("error", False):
raise Exception(data_json.get("error"))
elif data_lookup.get("error", False):
raise Exception(data_lookup.get("error", None))
return {
"text": text,
"is_finished": is_finished,
@ -828,11 +866,11 @@ class CustomStreamWrapper:
if self.model and "dolphin" in self.model:
chunk = self.process_chunk(chunk=chunk)
else:
data_json: Final[_NlpCloudStreamData] = json.loads(chunk)
chunk = data_json["generated_text"]
data_json: Final = _DECODED_JSON.validate_python(json.loads(chunk))
chunk = subscript(data_json, "generated_text")
text = chunk
if "[DONE]" in text:
text = text.replace("[DONE]", "")
if json_contains(text, "[DONE]"):
text = require_attributes(text, _Replaceable, "replace").replace("[DONE]", "")
is_finished = True
finish_reason = "stop"
return {
@ -845,9 +883,9 @@ class CustomStreamWrapper:
def handle_aleph_alpha_chunk(self, chunk):
chunk = chunk.decode("utf-8")
data_json: Final[_AlephAlphaStreamData] = json.loads(chunk)
data_json: Final = _DECODED_JSON.validate_python(json.loads(chunk))
try:
text: Final = data_json["completions"][0]["completion"]
text: Final = subscript(subscript(subscript(data_json, "completions"), 0), "completion")
is_finished: Final = True
finish_reason: Final = "stop"
return {
@ -1014,20 +1052,17 @@ class CustomStreamWrapper:
def model_response_creator(
self,
chunk: dict | None = None,
chunk: dict[str, object] | None = None,
hidden_params: Mapping[str, object] | None = None,
delta: Delta | None = None,
) -> ModelResponseStream:
_model: Final = self._served_openrouter_model() or self._cached_model_name
_logging_obj_llm_provider: Final = self._cached_logging_llm_provider
if chunk is None:
args: dict[str, Any] = {"model": _model}
else:
if chunk is not None:
chunk.pop("model", None)
args = {"model": _model}
if chunk:
args.update({k: v for k, v in chunk.items() if k != "stream"})
chunk_fields: Final[Mapping[str, object]] = {k: v for k, v in chunk.items() if k != "stream"} if chunk else {}
args: Final = _keyword_fields({"model": _model, **chunk_fields})
model_response: Final = (
ModelResponseStream(**args)
@ -1108,7 +1143,8 @@ class CustomStreamWrapper:
provider_specific_fields: Final = getattr(original_chunk, "provider_specific_fields", None)
if provider_specific_fields is not None:
model_response.provider_specific_fields = provider_specific_fields
_apply_provider_fields(model_response, provider_specific_fields)
if model_response.provider_specific_fields is not None:
_apply_provider_fields(model_response, model_response.provider_specific_fields)
return model_response
def is_chunk_non_empty(
@ -1169,10 +1205,8 @@ class CustomStreamWrapper:
model_response.choices[0].delta["role"] = "assistant"
self.sent_first_chunk = True
elif self.sent_first_chunk is True and hasattr(model_response.choices[0].delta, "role"):
_initial_delta: Final = model_response.choices[0].delta.model_dump()
_initial_delta.pop("role", None)
set_extra_attribute(model_response.choices[0], "delta", Delta(**_initial_delta))
_initial_delta: Final = without(model_response.choices[0].delta.model_dump(), "role")
set_extra_attribute(model_response.choices[0], "delta", Delta(**_keyword_fields(_initial_delta)))
return model_response
def _prebuilt_plain_delta(self, chunk: object) -> Delta | None:
@ -1275,7 +1309,7 @@ class CustomStreamWrapper:
self,
completion_obj: dict[str, Any],
model_response: ModelResponseStream,
response_obj: dict[str, Any],
response_obj: Mapping[str, Any],
plain_delta_created: bool = False,
):
from litellm.litellm_core_utils.core_helpers import (
@ -1300,19 +1334,15 @@ class CustomStreamWrapper:
if hold is False:
## check if openai/azure chunk
original_chunk: Final = response_obj.get("original_chunk", None)
original_chunk: Final[object] = response_obj.get("original_chunk", None)
if original_chunk:
upstream_chunk: Final[_UpstreamChunkLike] = original_chunk
upstream_chunk: Final = require_attributes(original_chunk, _UpstreamChunkLike, "choices")
if len(upstream_chunk.choices) > 0:
original_chunk_for_reuse: Final[object] = cast( # cast-ok: runtime type-check before access
object,
original_chunk,
)
upstream_choice: Final = (
original_chunk_for_reuse.choices[0]
original_chunk.choices[0]
if plain_delta_created
and isinstance(original_chunk_for_reuse, (ChatCompletionChunk, ModelResponseStream))
and len(original_chunk_for_reuse.choices) == 1
and isinstance(original_chunk, (ChatCompletionChunk, ModelResponseStream))
and len(original_chunk.choices) == 1
else None
)
upstream_delta: Final = cast( # cast-ok: [LIT006] runtime delta may be absent
@ -1343,13 +1373,13 @@ class CustomStreamWrapper:
for choice in upstream_chunk.choices:
try:
if isinstance(choice, BaseModel):
choice_json = choice.model_dump()
choice_json.pop(
"finish_reason", None
) # for mistral etc. which return a value in their last chunk (not-openai compatible).
choice_json.pop("logprobs", None)
choices.append(
StreamingChoices(**choice_json, logprobs=getattr(choice, "logprobs", None))
StreamingChoices(
**_keyword_fields(
without(choice.model_dump(), "finish_reason", "logprobs")
), # for mistral etc. which return a value in their last chunk (not-openai compatible).
logprobs=getattr(choice, "logprobs", None),
)
)
except Exception:
choices.append(StreamingChoices())
@ -1377,10 +1407,10 @@ class CustomStreamWrapper:
self.sent_first_chunk = True
if response_obj.get("provider_specific_fields") is not None:
completion_obj["provider_specific_fields"] = response_obj["provider_specific_fields"]
model_response.choices[0].delta = Delta(**completion_obj)
_index: Final[int | None] = completion_obj.get("index", response_obj.get("choice_index"))
model_response.choices[0].delta = Delta(**_keyword_fields(completion_obj))
_index: Final = _delta_choice_index(completion_obj, response_obj)
if _index is not None:
model_response.choices[0].index = _index
set_extra_attribute(model_response.choices[0], "index", _index)
_apply_choice_fields(model_response.choices[0], chunk_choice_fields)
if sibling_finish_reason is not None:
@ -1436,7 +1466,7 @@ class CustomStreamWrapper:
# Preserve custom attributes from original chunk (applies to both
# empty and non-empty delta final chunks).
_original_chunk: Final = response_obj.get("original_chunk", None)
_original_chunk: Final[object] = response_obj.get("original_chunk", None)
if _original_chunk is not None:
preserve_upstream_non_openai_attributes(
model_response=model_response,
@ -1504,7 +1534,7 @@ class CustomStreamWrapper:
def _dispatch_provider_chunk(
self,
chunk: Any,
chunk: object,
model_response: ModelResponseStream,
completion_obj: dict[str, object],
) -> _ProviderChunkResult:
@ -1532,19 +1562,23 @@ class CustomStreamWrapper:
chunk.choices[0].finish_reason = None
return _ProviderChunkEarlyReturn(chunk)
chunk_fields: Final = as_str_mapping(chunk)
generic_chunk: Final = chunk_fields if isinstance(chunk_fields, dict) else None
if (
isinstance(chunk, dict)
and generic_chunk_has_all_required_fields(chunk=chunk) # check if chunk is a generic streaming chunk
# check if chunk is a generic streaming chunk
generic_chunk is not None and generic_chunk_has_all_required_fields(chunk=generic_chunk)
) or (self.custom_llm_provider and self.custom_llm_provider in litellm._custom_providers):
if self.received_finish_reason is not None:
_chunk_has_content: Final = isinstance(chunk, dict) and (
bool(chunk.get("text", ""))
or chunk.get("tool_use") is not None
_chunk_has_content: Final = generic_chunk is not None and (
bool(generic_chunk.get("text", ""))
or generic_chunk.get("tool_use") is not None
# Usage-only final chunks are valid and needed to surface
# finish_reason/usage to downstream translators.
or chunk.get("usage") is not None
or generic_chunk.get("usage") is not None
)
if not _chunk_has_content and (not isinstance(chunk, dict) or "provider_specific_fields" not in chunk):
if not _chunk_has_content and (
generic_chunk is None or "provider_specific_fields" not in generic_chunk
):
raise StopIteration
anthropic_response_obj: Final[GChunk] = cast(GChunk, chunk)
completion_obj["content"] = anthropic_response_obj["text"]
@ -1582,7 +1616,12 @@ class CustomStreamWrapper:
if response_obj["is_finished"]:
self.received_finish_reason = response_obj["finish_reason"]
elif self.custom_llm_provider and self.custom_llm_provider == "vllm":
completion_obj["content"] = chunk[0].outputs[0].text
vllm_output: Final = require_attributes(
subscript(require_attributes(subscript(chunk, 0), _VllmRequestOutputLike, "outputs").outputs, 0),
_VllmCompletionOutputLike,
"text",
)
completion_obj["content"] = vllm_output.text
elif (
self.custom_llm_provider and self.custom_llm_provider == "aleph_alpha"
): # aleph alpha doesn't provide streaming
@ -1689,7 +1728,13 @@ class CustomStreamWrapper:
if response_obj["is_finished"] and response_obj["sibling_finish_reason"] is None:
self.received_finish_reason = response_obj["finish_reason"]
if response_obj["usage"] is not None:
_text_completion_usage: Final[Usage] = response_obj["usage"]
_text_completion_usage: Final = require_attributes(
_DECODED_JSON.validate_python(response_obj["usage"]),
_TokenTotalsLike,
"prompt_tokens",
"completion_tokens",
"total_tokens",
)
setattr(
model_response,
"usage",
@ -1711,7 +1756,13 @@ class CustomStreamWrapper:
if response_obj["is_finished"]:
self.received_finish_reason = response_obj["finish_reason"]
if "usage" in response_obj is not None:
_codestral_usage: Final[Usage] = response_obj["usage"]
_codestral_usage: Final = require_attributes(
_DECODED_JSON.validate_python(response_obj["usage"]),
_TokenTotalsLike,
"prompt_tokens",
"completion_tokens",
"total_tokens",
)
setattr(
model_response,
"usage",
@ -1813,14 +1864,14 @@ class CustomStreamWrapper:
)
return _ProviderChunkParsed(response_obj)
def chunk_creator(self, chunk: Any):
def chunk_creator(self, chunk: object):
if isinstance(chunk, ModelResponseStream) and chunk.provider_specific_fields:
self.logging_obj.record_native_bedrock_guardrail(chunk.provider_specific_fields)
if self.custom_llm_provider != "cached_response" and isinstance(
chunk, (ChatCompletionChunk, ModelResponseStream)
):
self._ollama_response_ids.observe(chunk)
if hasattr(chunk, "id"):
if has_attributes(chunk, _IdentifiedChunkLike, "id"):
self.response_id = chunk.id
if self._ollama_response_ids.response_id is not None:
self.response_id = self._ollama_response_ids.response_id
@ -1840,10 +1891,10 @@ class CustomStreamWrapper:
),
delta=prebuilt_delta,
)
response_obj: dict[str, Any] = {}
response_obj: Mapping[str, object] = {}
try:
# return this for all models
completion_obj: Final[dict[str, Any]] = {"content": ""}
completion_obj: Final[dict[str, object]] = {"content": ""}
dispatch_result: Final = self._dispatch_provider_chunk(
chunk=chunk,
model_response=model_response,
@ -1855,24 +1906,23 @@ class CustomStreamWrapper:
model_response.model = self._served_openrouter_model() or self.model
## FUNCTION CALL PARSING
original_chunk: Final[_UpstreamChunkLike | None] = (
response_obj.get("original_chunk") if response_obj is not None else None
)
original_chunk: Final = response_obj.get("original_chunk") if response_obj is not None else None
if (
original_chunk is not None
): # function / tool calling branch - only set for openai/azure compatible endpoints
# enter this branch when no content has been passed in response
if hasattr(original_chunk, "id"):
if has_attributes(original_chunk, _IdentifiedChunkLike, "id"):
model_response = self.set_model_id(original_chunk.id, model_response)
if stored_attribute(original_chunk, "provider_specific_fields") is not None:
model_response = self.copy_model_response_level_provider_specific_fields(
original_chunk, model_response
)
if original_chunk.choices and len(original_chunk.choices) > 0:
delta: Final = original_chunk.choices[0].delta
upstream_chunk: Final = require_attributes(original_chunk, _UpstreamChunkLike, "choices")
if upstream_chunk.choices and len(upstream_chunk.choices) > 0:
delta: Final = upstream_chunk.choices[0].delta
if delta is not None and (delta.function_call is not None or delta.tool_calls is not None):
try:
model_response.system_fingerprint = original_chunk.system_fingerprint
model_response.system_fingerprint = upstream_chunk.system_fingerprint
## AZURE - check if arguments is not None
if delta.function_call is not None:
if delta.function_call.arguments is None:
@ -1922,7 +1972,7 @@ class CustomStreamWrapper:
if "tool_calls" in completion_obj and _entry_len(completion_obj, "tool_calls") > 0:
if self.is_function_call is True: # user passed in 'functions' param
completion_obj["function_call"] = completion_obj["tool_calls"][0]["function"]
completion_obj["function_call"] = subscript(subscript(completion_obj["tool_calls"], 0), "function")
completion_obj["tool_calls"] = None
self.tool_call = True
@ -1931,7 +1981,7 @@ class CustomStreamWrapper:
if isinstance(chunk_usage, Usage):
model_response.usage = chunk_usage
elif isinstance(chunk_usage, BaseModel):
model_response.usage = Usage(**chunk_usage.model_dump())
model_response.usage = Usage(**_keyword_fields(chunk_usage.model_dump()))
## RETURN ARG
if isinstance(chunk, ModelResponseStream) and chunk.provider_specific_fields:
@ -2126,7 +2176,11 @@ class CustomStreamWrapper:
else:
asyncio.run(self.logging_obj.async_success_handler(processed_chunk, None, None, cache_hit))
## SYNC LOGGING — only for sync SDK entrypoints; async proxy paths export via async_success_handler
litellm_params: Final[Mapping[str, object]] = self.logging_obj.model_call_details.get("litellm_params", {})
litellm_params: Final = require_attributes(
_DECODED_JSON.validate_python(self.logging_obj.model_call_details.get("litellm_params", {})),
_ParamsLookupLike,
"get",
)
if self.logging_obj.is_sync_litellm_request(litellm_params):
self.logging_obj.success_handler(processed_chunk, None, None, cache_hit)
@ -3015,9 +3069,9 @@ class CustomStreamWrapper:
_strip_sse_data_from_chunk = strip_sse_data_from_chunk
def _usage_from_payload(payload: Mapping[str, Any]) -> Usage:
def _usage_from_payload(payload: Mapping[str, object]) -> Usage:
"""Build ``Usage`` from a wire-shaped usage dict, forwarding every key."""
return litellm.Usage(**payload)
return litellm.Usage(**_keyword_fields(payload))
def _cache_token_count(details: PromptTokensDetailsWrapper | None, keys: tuple[str, ...]) -> int:

30
litellm/llms/a2a/common_utils.py
View file

@ -2,11 +2,12 @@
Common utilities for A2A (Agent-to-Agent) Protocol
"""
from collections.abc import Awaitable, Callable, Mapping
from typing import Any, Final
from collections.abc import Awaitable, Callable, Mapping, Sequence
from typing import Final
from pydantic import BaseModel
from litellm.litellm_core_utils.core_helpers import decoded_json_object, iter_items
from litellm.litellm_core_utils.prompt_templates.common_utils import (
convert_content_list_to_str,
)
@ -67,7 +68,15 @@ def convert_messages_to_prompt(messages: list[AllMessageValues]) -> str:
return "\n".join(conversation_parts)
def extract_text_from_a2a_message(message: dict[str, Any], depth: int = 0, max_depth: int = 10) -> str:
def _join_with_spaces(texts: Sequence[object]) -> str:
strings: Final = [text for text in texts if isinstance(text, str)]
if len(strings) == len(texts):
return " ".join(strings)
first_bad: Final = next(position for position, text in enumerate(texts) if not isinstance(text, str))
raise TypeError(f"sequence item {first_bad}: expected str instance, {type(texts[first_bad]).__name__} found")
def extract_text_from_a2a_message(message: object, depth: int = 0, max_depth: int = 10) -> str:
"""
Extract text content from A2A message parts.
@ -82,19 +91,20 @@ def extract_text_from_a2a_message(message: dict[str, Any], depth: int = 0, max_d
if message is None or depth >= max_depth:
return ""
parts: Final = message.get("parts", [])
text_parts: Final[list[str]] = []
parts: Final = iter_items(decoded_json_object(message).get("parts", []))
text_parts: Final[list[object]] = []
for part in parts:
if part.get("kind") == "text":
text_parts.append(part.get("text", ""))
part_fields = decoded_json_object(part)
if part_fields.get("kind") == "text":
text_parts.append(part_fields.get("text", ""))
# Handle nested parts if they exist
elif "parts" in part:
nested_text = extract_text_from_a2a_message(part, depth + 1, max_depth)
elif "parts" in part_fields:
nested_text = extract_text_from_a2a_message(part_fields, depth + 1, max_depth)
if nested_text:
text_parts.append(nested_text)
return " ".join(text_parts)
return _join_with_spaces(text_parts)
def extract_text_from_a2a_response(response_dict: Mapping[str, object], max_depth: int = 10) -> str:

15
litellm/llms/aiohttp_openai/chat/transformation.py
View file

@ -11,7 +11,8 @@ from collections.abc import Iterable, Mapping
from typing import TYPE_CHECKING, Any, Final
from aiohttp import ClientResponse
from pydantic import ConfigDict, TypeAdapter
from pydantic import ConfigDict, JsonValue, TypeAdapter
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig
@ -31,6 +32,16 @@ _JSON_OBJECTS: Final = TypeAdapter(
)
class _ChatCompletionBody(TypedDict, extra_items=object):
id: ReadOnly[str]
created: ReadOnly[int]
model: ReadOnly[str]
object: ReadOnly[str]
choices: ReadOnly[JsonValue]
system_fingerprint: ReadOnly[NotRequired[str | None]]
service_tier: ReadOnly[NotRequired[str | None]]
class AiohttpOpenAIChatConfig(OpenAILikeChatConfig):
def get_complete_url(
self,
@ -73,7 +84,7 @@ class AiohttpOpenAIChatConfig(OpenAILikeChatConfig):
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:
_json_response: Final = await raw_response.json()
_json_response: Final[_ChatCompletionBody] = await raw_response.json()
model_response.id = _json_response.get("id")
model_response.choices = [
Choices.model_validate(choice) for choice in _JSON_OBJECTS.validate_python(_json_response.get("choices"))

80
litellm/llms/anthropic/chat/transformation.py
View file

@ -6,7 +6,7 @@ from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from functools import partial
from itertools import chain
from types import MappingProxyType
from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Final, NoReturn, cast
from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Final, NoReturn, Protocol, cast, runtime_checkable
import httpx
from openai.types.responses.response_function_web_search import ResponseFunctionWebSearch
@ -31,6 +31,7 @@ from litellm.constants import (
)
from litellm.litellm_core_utils.asyncify import is_event_loop_running
from litellm.litellm_core_utils.core_helpers import (
as_object_list,
as_str_mapping,
coerce_token_limit,
drops_unsupported_params,
@ -450,6 +451,46 @@ def _enum_conflicts_with_declared_type(schema: Mapping[str, object]) -> bool:
return check is not None and not all(check(value) for value in enum_values)
def _as_json_object(value: object) -> Mapping[str, object] | None:
return value if isinstance(value, dict) else None
def _filtered_output_subschema(value: object) -> object:
subschema: Final = _as_json_object(value)
return value if subschema is None else AnthropicConfig.filter_anthropic_output_schema(subschema)
def _filtered_output_subschemas(value: object) -> object:
subschemas: Final = _as_json_object(value)
if subschemas is None:
return value
return {name: _filtered_output_subschema(subschema) for name, subschema in subschemas.items()}
def _filtered_output_branches(branches: Sequence[object]) -> Sequence[object]:
return [_filtered_output_subschema(branch) for branch in branches]
@runtime_checkable
class _SupportsAdd(Protocol):
def __add__(self, other: object, /) -> object: ...
def _concatenated(left: object, right: str | Sequence[object]) -> object:
if isinstance(left, _SupportsAdd):
return left + right
raise TypeError(f"unsupported operand type(s) for +: '{type(left).__name__}' and '{type(right).__name__}'")
def _extended_any_of(result: Mapping[str, object], branches: Sequence[object]) -> object:
filtered: Final = _filtered_output_branches(branches)
return _concatenated(result["anyOf"], filtered) if "anyOf" in result else filtered
def _described_with_note(description: object, note: str) -> object:
return _concatenated(_concatenated(description, " "), note) if description else note
# Single, internal-only key on ``litellm_params`` used to thread the per-
# request reverse map (sanitized -> original) from request build to response
# parsing. ``litellm_params`` is never serialized to a provider; ``optional_
@ -927,7 +968,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
return params
@staticmethod
def filter_anthropic_output_schema(schema: dict[str, Any]) -> dict[str, Any]:
def filter_anthropic_output_schema(schema: Mapping[str, object]) -> Mapping[str, object]:
"""
Filter out unsupported fields from JSON schema for Anthropic's output_format API.
@ -1019,16 +1060,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
note_value = json.dumps(value) if isinstance(value, (dict, list)) else value
constraint_descriptions.append(label.format(note_value))
result: Final[dict[str, Any]] = {}
result: Final[dict[str, object]] = {}
# Update description with removed constraint info
if constraint_descriptions:
existing_desc: Final = schema.get("description", "")
constraint_note: Final = "Note: " + ", ".join(constraint_descriptions) + "."
if existing_desc:
result["description"] = existing_desc + " " + constraint_note
else:
result["description"] = constraint_note
result["description"] = _described_with_note(schema.get("description"), constraint_note)
drops_conflicting_type: Final = _enum_conflicts_with_declared_type(schema)
@ -1041,26 +1078,21 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# Already handled above
continue
if key in ("properties", "$defs", "definitions") and isinstance(value, dict):
nested_schemas: dict[str, Any] = value # mutable-ok: filter_anthropic_output_schema takes a dict
result[key] = {k: AnthropicConfig.filter_anthropic_output_schema(v) for k, v in nested_schemas.items()}
elif key == "items" and isinstance(value, dict):
nested_schema: dict[str, Any] = value # mutable-ok: filter_anthropic_output_schema takes a dict
result[key] = AnthropicConfig.filter_anthropic_output_schema(nested_schema)
elif key == "anyOf" and isinstance(value, list):
result["anyOf"] = result.get("anyOf", []) + [
AnthropicConfig.filter_anthropic_output_schema(item) for item in value
]
elif key == "allOf" and isinstance(value, list):
result[key] = [AnthropicConfig.filter_anthropic_output_schema(item) for item in value]
branches = as_object_list(value)
if key in ("properties", "$defs", "definitions"):
result[key] = _filtered_output_subschemas(value)
elif key == "items":
result[key] = _filtered_output_subschema(value)
elif key == "anyOf" and branches is not None:
result["anyOf"] = _extended_any_of(result, branches)
elif key == "allOf" and branches is not None:
result[key] = _filtered_output_branches(branches)
elif key == "$ref" and isinstance(value, str):
result[key] = schema_fragment_ref(value)
elif key == "oneOf" and isinstance(value, list):
elif key == "oneOf" and branches is not None:
# Anthropic rejects oneOf ("Schema type 'oneOf' is not supported");
# the Anthropic SDK rewrites it to anyOf, so do the same.
result["anyOf"] = result.get("anyOf", []) + [
AnthropicConfig.filter_anthropic_output_schema(item) for item in value
]
result["anyOf"] = _extended_any_of(result, branches)
else:
result[key] = value

6
litellm/llms/anthropic/common_utils.py
View file

@ -2570,7 +2570,7 @@ def changed_server_tool_call_arguments(
def write_anthropic_hosted_web_call_arguments(holder: Message | Delta, arguments_by_tool_id: Mapping[str, str]) -> None:
fields: Final = holder.provider_specific_fields
fields: Final[Mapping[str, object]] = holder.provider_specific_fields or {}
if not arguments_by_tool_id or not fields or fields.get("web_search_calls") is None:
return
holder.provider_specific_fields = { # rebind-ok: the hosted copy of a rewritten call must not keep its old input
@ -2598,7 +2598,7 @@ def _jsonable_hosted_web_call(item: object, arguments_by_tool_id: Mapping[str, s
call: Final = _with_hosted_web_call_arguments(item, arguments_by_tool_id)
if not isinstance(call, ResponseFunctionWebSearch):
return call
return _WEB_SEARCH_CALL_ADAPTER.dump_python(call, mode="json", exclude_none=True)
return _DECODED_JSON.validate_python(_WEB_SEARCH_CALL_ADAPTER.dump_python(call, mode="json", exclude_none=True))
def _with_hosted_web_call_arguments(item: object, arguments_by_tool_id: Mapping[str, str]) -> object:
@ -2613,7 +2613,7 @@ def _with_hosted_web_call_arguments(item: object, arguments_by_tool_id: Mapping[
def without_anthropic_server_tool_calls(holder: Message, tool_ids: frozenset[str]) -> None:
fields: Final = holder.provider_specific_fields or {}
fields: Final[Mapping[str, object]] = holder.provider_specific_fields or {}
holder.provider_specific_fields = { # rebind-ok: a denied call must leave every copy of the message it was on
**fields,
"server_tool_calls": [

22
litellm/llms/anthropic/messages/adapters/transformation.py
View file

@ -4,12 +4,17 @@ import json
from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast
from pydantic import JsonValue, TypeAdapter
import litellm
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items, normalize_drop_params
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
attribute_items,
normalize_drop_params,
require_attributes,
)
from litellm.litellm_core_utils.hidden_params import get_hidden_params
from litellm.llms.anthropic.messages.convertible_blocks import tool_result_text_part
from litellm.llms.anthropic.messages.utils import (
@ -254,6 +259,11 @@ if TYPE_CHECKING:
ToolResultContent: TypeAlias = str | list[ToolMessageContentPart]
class _CarriesUsage(Protocol):
@property
def usage(self) -> Usage: ...
def target_supports_mid_conversation_system(model: str | None, custom_llm_provider: str | None) -> bool:
if not model:
return False
@ -681,7 +691,7 @@ class LiteLLMAnthropicMessagesAdapter:
## ASSISTANT MESSAGE ##
assistant_message_str: str | None = None
assistant_content_list: list[dict[str, Any]] = [] # For content blocks with cache_control
assistant_content_list: list[ChatCompletionTextObject] = [] # For content blocks with cache_control
has_cache_control_in_text = False
tool_calls: list[ChatCompletionAssistantToolCall] = []
thinking_blocks: list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] = []
@ -694,7 +704,7 @@ class LiteLLMAnthropicMessagesAdapter:
assistant_message_str = str(content)
elif isinstance(content, dict):
if content.get("type") == "text":
text_block: dict[str, object] = {
text_block: ChatCompletionTextObject = {
"type": "text",
"text": content.get("text", ""),
}
@ -753,7 +763,7 @@ class LiteLLMAnthropicMessagesAdapter:
):
# Use list format if any text block has cache_control, otherwise use string
if has_cache_control_in_text and len(assistant_content_list) > 0:
assistant_content: Any = assistant_content_list
assistant_content: str | Sequence[ChatCompletionTextObject] | None = assistant_content_list
elif len(assistant_content_list) > 0 and not has_cache_control_in_text:
# Concatenate text blocks into string when no cache_control
assistant_content = "".join(block.get("text", "") for block in assistant_content_list)
@ -1795,7 +1805,7 @@ class LiteLLMAnthropicMessagesAdapter:
else translated_finish_reason
)
# extract usage
usage: Final[Usage] = getattr(response, "usage")
usage: Final = require_attributes(response, _CarriesUsage, "usage").usage
message_usage: Final = self.translate_openai_usage_to_anthropic_usage(usage)
polyfill_iterations: Final = polyfill_result.iterations_usage if polyfill_result is not None else None
anthropic_usage: Final[AnthropicUsage] = (

7
litellm/llms/anthropic/messages/native/mcp_handler.py
View file

@ -46,6 +46,10 @@ def _get_stop_reason(response: AnthropicMessagesResponse) -> str | None:
return stop_reason if isinstance(stop_reason, str) else None
def _requests_stream(kwargs: Mapping[str, object]) -> bool:
return bool(kwargs.get("stream", False))
def _build_tool_result_message(tool_results: Sequence[Mapping[str, object]]) -> AnthropicMessagesUserMessageParam:
"""Turn executed tool results into the user message Anthropic expects."""
return AnthropicMessagesUserMessageParam(
@ -120,7 +124,8 @@ async def anthropic_messages_with_mcp(
should_auto_execute: Final = LiteLLM_Proxy_MCP_Handler.should_auto_execute_tools(
mcp_tools_with_litellm_proxy=mcp_references
)
stream: Final = bool(kwargs.pop("stream", False))
stream: Final = _requests_stream(kwargs)
kwargs.pop("stream", None)
base_call_args: Final[Mapping[str, object]] = {
"max_tokens": max_tokens,

9
litellm/llms/anthropic/messages/native/streaming_iterator.py
View file

@ -28,6 +28,7 @@ from litellm.llms.anthropic.messages.utils import anthropic_sse_frame
from litellm.proxy.pass_through_endpoints.success_handler import (
PassThroughEndpointLogging,
)
from litellm.responses.mcp.parked_stream_dispatch import DeferredStreamRelease
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType
from litellm.types.utils import GenericStreamingChunk, ModelResponseStream
@ -647,8 +648,12 @@ class BaseAnthropicMessagesStreamingIterator:
disconnect (GeneratorExit / CancelledError) re-raises past it. Without
this dispatch that window loses the spend row entirely.
"""
deferred_cb: Final = getattr(self.litellm_logging_obj, "_on_deferred_stream_complete", None)
deferred_args: Final = getattr(self.litellm_logging_obj, "_deferred_stream_complete_args", None)
deferred_cb: Final[DeferredStreamRelease | None] = getattr(
self.litellm_logging_obj, "_on_deferred_stream_complete", None
)
deferred_args: Final[tuple[object, ...] | None] = getattr(
self.litellm_logging_obj, "_deferred_stream_complete_args", None
)
if deferred_cb is None or deferred_args is None:
return
self.litellm_logging_obj._on_deferred_stream_complete = None

64
litellm/llms/azure_ai/agents/handler.py
View file

@ -29,6 +29,7 @@ import httpx
from typing_extensions import ReadOnly
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import decoded_json_object, subscript
from litellm.litellm_core_utils.hidden_params import set_hidden_param
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
from litellm.llms.azure_ai.agents.transformation import (
@ -81,19 +82,6 @@ class _AzureMessagesData(TypedDict, total=False):
data: ReadOnly[list[_AzureMessage]]
class _CreatedObject(TypedDict):
id: ReadOnly[str]
class _RunError(TypedDict, total=False):
message: ReadOnly[str]
class _RunStatus(TypedDict, total=False):
status: ReadOnly[str]
last_error: ReadOnly[_RunError]
class _SSEDelta(TypedDict, total=False):
content: ReadOnly[list[_AzureContentItem]]
@ -114,6 +102,14 @@ class _AsyncAgentRequest(Protocol):
) -> Awaitable[httpx.Response]: ...
class _JsonResponse(Protocol):
def json(self) -> object: ...
def _json_body(response: _JsonResponse) -> object:
return response.json()
class AzureAIAgentsHandler:
"""
Handler for Azure AI Agent Service.
@ -133,21 +129,21 @@ class AzureAIAgentsHandler:
def _build_thread_url(self, api_base: str, api_version: str) -> str:
return f"{join_api_base(api_base, 'threads')}?api-version={api_version}"
def _build_messages_url(self, api_base: str, thread_id: str, api_version: str) -> str:
def _build_messages_url(self, api_base: str, thread_id: object, api_version: str) -> str:
encoded_thread_id: Final = encode_url_path_segment(thread_id, field_name="thread_id")
return f"{join_api_base(api_base, f'threads/{encoded_thread_id}/messages')}?api-version={api_version}"
def _build_runs_url(self, api_base: str, thread_id: str, api_version: str) -> str:
def _build_runs_url(self, api_base: str, thread_id: object, api_version: str) -> str:
encoded_thread_id: Final = encode_url_path_segment(thread_id, field_name="thread_id")
return f"{join_api_base(api_base, f'threads/{encoded_thread_id}/runs')}?api-version={api_version}"
def _build_run_status_url(self, api_base: str, thread_id: str, run_id: str, api_version: str) -> str:
def _build_run_status_url(self, api_base: str, thread_id: object, run_id: object, api_version: str) -> str:
encoded_thread_id: Final = encode_url_path_segment(thread_id, field_name="thread_id")
encoded_run_id: Final = encode_url_path_segment(run_id, field_name="run_id")
run_status_route: Final = f"threads/{encoded_thread_id}/runs/{encoded_run_id}"
return f"{join_api_base(api_base, run_status_route)}?api-version={api_version}"
def _build_list_messages_url(self, api_base: str, thread_id: str, api_version: str) -> str:
def _build_list_messages_url(self, api_base: str, thread_id: object, api_version: str) -> str:
encoded_thread_id: Final = encode_url_path_segment(thread_id, field_name="thread_id")
return f"{join_api_base(api_base, f'threads/{encoded_thread_id}/messages')}?api-version={api_version}"
@ -217,7 +213,7 @@ class AzureAIAgentsHandler:
model: str,
content: str,
model_response: ModelResponse,
thread_id: str,
thread_id: object,
messages: list[dict[str, object]],
annotations: list[_TransformedAnnotation] | None = None,
) -> ModelResponse:
@ -354,10 +350,10 @@ class AzureAIAgentsHandler:
api_base: str,
api_version: str,
agent_id: str,
thread_id: str | None,
thread_id: object,
messages: list[dict[str, object]],
optional_params: dict,
) -> tuple[str, str, list[_TransformedAnnotation] | None]:
) -> tuple[object, str, list[_TransformedAnnotation] | None]:
"""Execute the agent flow synchronously. Returns (thread_id, content, annotations)."""
# Step 1: Create thread if not provided
@ -365,8 +361,7 @@ class AzureAIAgentsHandler:
verbose_logger.debug("Creating thread at: %s", self._build_thread_url(api_base, api_version))
response = make_request("POST", self._build_thread_url(api_base, api_version), {})
self._check_response(response, [200, 201], "Failed to create thread")
thread_data: Final[_CreatedObject] = response.json()
thread_id = thread_data["id"]
thread_id = subscript(_json_body(response), "id")
verbose_logger.debug("Created thread: %s", thread_id)
# At this point thread_id is guaranteed to be a string
@ -386,8 +381,7 @@ class AzureAIAgentsHandler:
response = make_request("POST", self._build_runs_url(api_base, thread_id, api_version), run_payload)
self._check_response(response, [200, 201], "Failed to create run")
run_data: Final[_CreatedObject] = response.json()
run_id: Final = run_data["id"]
run_id: Final = subscript(_json_body(response), "id")
verbose_logger.debug("Created run: %s", run_id)
# Step 4: Poll for completion
@ -396,15 +390,15 @@ class AzureAIAgentsHandler:
response = make_request("GET", status_url)
self._check_response(response, [200], "Failed to get run status")
status_data: _RunStatus = response.json()
status_data = decoded_json_object(_json_body(response))
status = status_data.get("status")
verbose_logger.debug("Run status: %s", status)
if status == "completed":
break
elif status in ["failed", "cancelled", "expired"]:
error_data: _RunStatus = response.json()
error_msg = error_data.get("last_error", {}).get("message", "Unknown error")
error_data = decoded_json_object(_json_body(response))
error_msg = decoded_json_object(error_data.get("last_error", {})).get("message", "Unknown error")
raise AzureAIAgentsError(status_code=500, message=f"Run {status}: {error_msg}")
time.sleep(self.config.POLL_INTERVAL_SECONDS)
@ -482,10 +476,10 @@ class AzureAIAgentsHandler:
api_base: str,
api_version: str,
agent_id: str,
thread_id: str | None,
thread_id: object,
messages: list[dict[str, object]],
optional_params: dict,
) -> tuple[str, str, list[_TransformedAnnotation] | None]:
) -> tuple[object, str, list[_TransformedAnnotation] | None]:
"""Execute the agent flow asynchronously. Returns (thread_id, content, annotations)."""
# Step 1: Create thread if not provided
@ -493,8 +487,7 @@ class AzureAIAgentsHandler:
verbose_logger.debug("Creating thread at: %s", self._build_thread_url(api_base, api_version))
response = await make_request("POST", self._build_thread_url(api_base, api_version), {})
self._check_response(response, [200, 201], "Failed to create thread")
thread_data: Final[_CreatedObject] = response.json()
thread_id = thread_data["id"]
thread_id = subscript(_json_body(response), "id")
verbose_logger.debug("Created thread: %s", thread_id)
# At this point thread_id is guaranteed to be a string
@ -514,8 +507,7 @@ class AzureAIAgentsHandler:
response = await make_request("POST", self._build_runs_url(api_base, thread_id, api_version), run_payload)
self._check_response(response, [200, 201], "Failed to create run")
run_data: Final[_CreatedObject] = response.json()
run_id: Final = run_data["id"]
run_id: Final = subscript(_json_body(response), "id")
verbose_logger.debug("Created run: %s", run_id)
# Step 4: Poll for completion
@ -524,15 +516,15 @@ class AzureAIAgentsHandler:
response = await make_request("GET", status_url)
self._check_response(response, [200], "Failed to get run status")
status_data: _RunStatus = response.json()
status_data = decoded_json_object(_json_body(response))
status = status_data.get("status")
verbose_logger.debug("Run status: %s", status)
if status == "completed":
break
elif status in ["failed", "cancelled", "expired"]:
error_data: _RunStatus = response.json()
error_msg = error_data.get("last_error", {}).get("message", "Unknown error")
error_data = decoded_json_object(_json_body(response))
error_msg = decoded_json_object(error_data.get("last_error", {})).get("message", "Unknown error")
raise AzureAIAgentsError(status_code=500, message=f"Run {status}: {error_msg}")
await asyncio.sleep(self.config.POLL_INTERVAL_SECONDS)

15
litellm/llms/azure_ai/vector_stores/transformation.py
View file

@ -4,7 +4,9 @@ from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final
import httpx
from pydantic import TypeAdapter
from litellm.litellm_core_utils.core_helpers import decoded_json_object
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.llms.base_llm.vector_store.transformation import (
@ -30,6 +32,8 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObj = Any
_DECODED_JSON: Final = TypeAdapter(object)
class AzureAIVectorStoreConfig(BaseQueryEmbeddingVectorStoreConfig, BaseAzureLLM):
"""
@ -230,13 +234,14 @@ class AzureAIVectorStoreConfig(BaseQueryEmbeddingVectorStoreConfig, BaseAzureLLM
# Build attributes with all available metadata
# Exclude system fields and already-processed fields
attributes = {}
for key, value in result.items():
if key not in ["id", "content", "contentVector", "@search.score"]:
attributes[key] = value
metadata = {
key: value
for key, value in decoded_json_object(_DECODED_JSON.validate_python(result)).items()
if key not in ["id", "content", "contentVector", "@search.score"]
}
# Always include document_id in attributes
attributes["document_id"] = document_id
attributes = {**metadata, "document_id": document_id}
result_obj = VectorStoreSearchResult(
score=score,

13
litellm/llms/base_llm/base_model_iterator.py
View file

@ -1,7 +1,10 @@
import json
from abc import abstractmethod
from collections.abc import Mapping
from typing import TYPE_CHECKING, Final, cast
from typing_extensions import TypedDict
import litellm
from litellm.constants import STREAM_SSE_DONE_STRING
@ -27,10 +30,18 @@ def _indexed_thinking_block(
return {**block, "index": index}
class _DeltaFields(TypedDict, total=False, extra_items=object):
pass
def _message_fields_without_thinking_blocks(message: Message) -> Mapping[str, object]:
return message.model_dump(exclude={"thinking_blocks"})
def _streaming_delta(message: Message) -> Delta:
thinking_blocks: Final = message.thinking_blocks if hasattr(message, "thinking_blocks") else None
return Delta(
**message.model_dump(exclude={"thinking_blocks"}),
**_DeltaFields(**_message_fields_without_thinking_blocks(message)),
thinking_blocks=(
None
if thinking_blocks is None

4
litellm/llms/base_llm/chat/transformation.py
View file

@ -73,8 +73,8 @@ _NO_ATTRIBUTION_HEADERS: Final[Mapping[str, str]] = types.MappingProxyType({})
def with_attribution_headers(
attribution_headers: Mapping[str, str],
headers: dict[str, str] | None, # mutable-ok: returned as-is when there is nothing to add
) -> dict[str, str] | None: # mutable-ok: becomes the request's outbound headers
headers: dict[str, object] | None, # mutable-ok: returned as-is when there is nothing to add
) -> dict[str, object] | None: # mutable-ok: becomes the request's outbound headers
"""
`headers` plus any attribution header the caller didn't already set (names
compared case-insensitively). Builds a new dict; `headers` is never mutated.

2
litellm/llms/base_llm/videos/transformation.py
View file

@ -235,7 +235,7 @@ class BaseVideoConfig(ABC):
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
custom_llm_provider: str | None = None,
) -> dict[str, str]:
) -> object:
pass
@abstractmethod

16
litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py
View file

@ -7,10 +7,11 @@ https://docs.twelvelabs.io/docs/models/pegasus
import json
import time
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
import httpx
from pydantic import ConfigDict, TypeAdapter, with_config
from pydantic import BaseModel, ConfigDict, TypeAdapter, with_config
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
@ -43,6 +44,7 @@ class _PegasusResponse(TypedDict):
_PEGASUS_RESPONSE: Final = TypeAdapter(_PegasusResponse)
_OPTIONAL_PARAM: Final = TypeAdapter(object)
class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
@ -65,7 +67,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
def map_openai_params(
self,
non_default_params: dict,
non_default_params: Mapping[str, object],
optional_params: dict,
model: str,
drop_params: bool,
@ -79,7 +81,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
optional_params["responseFormat"] = self._normalize_response_format(value)
return optional_params
def _normalize_response_format(self, value: Any) -> object:
def _normalize_response_format(self, value: object) -> object:
"""Normalize response_format to TwelveLabs format.
TwelveLabs expects:
@ -110,7 +112,11 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
return value
# Otherwise return the dict as is
return value
return type_to_response_format_param(response_format=value) or value
if value is None:
return None
if isinstance(value, type) and issubclass(value, BaseModel):
return type_to_response_format_param(response_format=value) or value
raise TypeError(f"Unsupported response_format type - {value}")
def transform_request(
self,
@ -134,7 +140,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
# Handle responseFormat - transform to TwelveLabs format
if "responseFormat" in optional_params:
response_format: Final = optional_params["responseFormat"]
response_format: Final = _OPTIONAL_PARAM.validate_python(optional_params["responseFormat"])
transformed_format: Final = self._normalize_response_format(response_format)
if transformed_format:
request_data["responseFormat"] = transformed_format

23
litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py
View file

@ -1,8 +1,9 @@
import types
from collections.abc import Mapping
from typing import Any, Final
from typing import Final
from openai.types.image import Image
from typing_extensions import TypedDict
from litellm.llms.bedrock.common_utils import get_cached_model_info
from litellm.types.llms.bedrock import (
@ -19,6 +20,10 @@ from litellm.types.llms.bedrock import (
from litellm.types.utils import ImageResponse
class _NovaCanvasTaskFields(TypedDict, total=False, extra_items=object):
pass
class AmazonNovaCanvasConfig:
"""
Reference: https://us-east-1.console.aws.amazon.com/bedrock/home?region=us-east-1#/model-catalog/serverless/amazon.nova-canvas-v1:0
@ -82,10 +87,12 @@ class AmazonNovaCanvasConfig:
image_generation_config = {**image_generation_config, **optional_params}
if task_type == "TEXT_IMAGE":
text_to_image_params: dict[str, Any] = image_generation_config.pop("textToImageParams", {})
text_to_image_params: Mapping[str, object] = image_generation_config.pop("textToImageParams", {})
text_to_image_params = {"text": text, **text_to_image_params}
try:
text_to_image_params_typed: Final = AmazonNovaCanvasTextToImageParams(**text_to_image_params)
text_to_image_params_typed: Final = AmazonNovaCanvasTextToImageParams(
**_NovaCanvasTaskFields(**text_to_image_params)
)
except Exception as e:
raise ValueError(
f"Error transforming text to image params: {e}. Got params: {text_to_image_params}, Expected params: {AmazonNovaCanvasTextToImageParams.__annotations__}"
@ -104,7 +111,7 @@ class AmazonNovaCanvasConfig:
imageGenerationConfig=image_generation_config_typed,
)
if task_type == "COLOR_GUIDED_GENERATION":
color_guided_generation_params: dict[str, Any] = image_generation_config.pop(
color_guided_generation_params: Mapping[str, object] = image_generation_config.pop(
"colorGuidedGenerationParams", {}
)
color_guided_generation_params = {
@ -113,7 +120,7 @@ class AmazonNovaCanvasConfig:
}
try:
color_guided_generation_params_typed: Final = AmazonNovaCanvasColorGuidedGenerationParams(
**color_guided_generation_params
**_NovaCanvasTaskFields(**color_guided_generation_params)
)
except Exception as e:
raise ValueError(
@ -133,10 +140,12 @@ class AmazonNovaCanvasConfig:
imageGenerationConfig=image_generation_config_typed,
)
if task_type == "INPAINTING":
inpainting_params: dict[str, Any] = image_generation_config.pop("inpaintingParams", {})
inpainting_params: Mapping[str, object] = image_generation_config.pop("inpaintingParams", {})
inpainting_params = {"text": text, **inpainting_params}
try:
inpainting_params_typed: Final = AmazonNovaCanvasInpaintingParams(**inpainting_params)
inpainting_params_typed: Final = AmazonNovaCanvasInpaintingParams(
**_NovaCanvasTaskFields(**inpainting_params)
)
except Exception as e:
raise ValueError(
f"Error transforming inpainting params: {e}. Got params: {inpainting_params}, Expected params: {AmazonNovaCanvasInpaintingParams.__annotations__}"

89
litellm/llms/bedrock/passthrough/guardrail_translation/handler.py
View file

@ -8,13 +8,13 @@ from dataclasses import dataclass
from functools import partial, reduce
from itertools import accumulate, chain
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Optional, Protocol, TypeAlias, TypeVar
from typing import TYPE_CHECKING, Final, Optional, Protocol, TypeAlias, TypeVar, runtime_checkable
from pydantic import InstanceOf, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object, subscript
from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler
from litellm.llms.base_llm.guardrail_translation.base_translation import (
BaseTranslation,
@ -171,10 +171,33 @@ def _reads_flat_texts_only(guardrail: "CustomGuardrail") -> bool:
return is_compression_guardrail(guardrail) or not guardrail.supports_structured_messages()
_StringHolder = tuple[Any, str | int]
_StringHolder: TypeAlias = tuple[object, str | int]
_StringLeaf: TypeAlias = tuple[_StringHolder, str]
_LeafIdentity: TypeAlias = tuple[int, str | int]
@runtime_checkable
class _SupportsSetItem(Protocol):
def __setitem__(self, key: str | int, value: object, /) -> None: ...
@runtime_checkable
class _SupportsDelItem(Protocol):
def __delitem__(self, key: str | int, /) -> None: ...
def _assignable(container: object) -> _SupportsSetItem:
if isinstance(container, _SupportsSetItem):
return container
raise TypeError(f"'{type(container).__name__}' object does not support item assignment")
def _deletable(container: object) -> _SupportsDelItem:
if isinstance(container, _SupportsDelItem):
return container
raise TypeError(f"'{type(container).__name__}' object doesn't support item deletion")
@dataclass(frozen=True, slots=True)
class _Assign:
holder: _StringHolder
@ -194,7 +217,7 @@ _OBJECT_BY_REFERENCE: Final = TypeAdapter(InstanceOf[Mapping[str, object]])
_ARRAY_BY_REFERENCE: Final = TypeAdapter(InstanceOf[list[object]])
def _collect_strings(node: object, holders: list[_StringHolder]) -> None:
def _collect_strings(node: object, leaves: list[_StringLeaf]) -> None:
"""
Record a (container, key) holder for every non-empty string value nested
under an arbitrary JSON node, so prompt content a caller hides in fields
@ -209,28 +232,28 @@ def _collect_strings(node: object, holders: list[_StringHolder]) -> None:
for key, value in current.items():
if isinstance(value, str):
if value:
holders.append((current, key))
leaves.append(((current, key), value))
else:
stack.append(value)
elif isinstance(current, list):
for index, value in enumerate(current):
if isinstance(value, str):
if value:
holders.append((current, index))
leaves.append(((current, index), value))
else:
stack.append(value)
def _collect_block_text(block: dict, holders: list[_StringHolder]) -> None:
def _collect_block_text(block: dict, leaves: list[_StringLeaf]) -> None:
text: Final = block.get("text")
if isinstance(text, str) and text:
holders.append((block, "text"))
leaves.append(((block, "text"), text))
def _request_field_holders(body: Mapping[str, object]) -> tuple[_StringHolder, ...]:
holders: Final = list[_StringHolder]()
_collect_strings(body.get("additionalModelRequestFields"), holders)
return tuple(holders)
def _request_field_leaves(body: Mapping[str, object]) -> tuple[_StringLeaf, ...]:
leaves: Final = list[_StringLeaf]()
_collect_strings(body.get("additionalModelRequestFields"), leaves)
return tuple(leaves)
def _extract_converse_texts(
@ -267,15 +290,15 @@ def _extract_converse_texts(
that model. Scanning it stays the fail-closed default. Its strings come
last, in the same order as they follow the latest user message in the rows.
"""
system_holders: Final = list[_StringHolder]()
system_leaves: Final = list[_StringLeaf]()
if not skip_system and not scan_only_tool_results:
for block in body.get("system") or []:
if isinstance(block, dict):
_collect_block_text(block, system_holders)
_collect_block_text(block, system_leaves)
request_field_holders: Final = () if scan_only_tool_results else _request_field_holders(body)
request_field_leaves: Final = () if scan_only_tool_results else _request_field_leaves(body)
message_holders: Final = list[_StringHolder]()
message_leaves: Final = list[_StringLeaf]()
for message in body.get("messages") or []:
if not isinstance(message, dict):
continue
@ -286,20 +309,19 @@ def _extract_converse_texts(
continue
if scan_only_tool_results and "toolResult" not in block:
continue
_collect_block_text(block, message_holders)
_collect_block_text(block, message_leaves)
tool_use = block.get("toolUse")
if isinstance(tool_use, dict):
_collect_strings(tool_use.get("input"), message_holders)
_collect_strings(tool_use.get("input"), message_leaves)
tool_result = block.get("toolResult")
if isinstance(tool_result, dict):
for inner in tool_result.get("content") or []:
if isinstance(inner, dict):
_collect_block_text(inner, message_holders)
_collect_strings(inner.get("json"), message_holders)
_collect_block_text(inner, message_leaves)
_collect_strings(inner.get("json"), message_leaves)
holders: Final = [*system_holders, *message_holders, *request_field_holders]
texts: Final = [container[key] for container, key in holders]
return texts, holders
leaves: Final = (*system_leaves, *message_leaves, *request_field_leaves)
return [text for _, text in leaves], [holder for holder, _ in leaves]
@dataclass(frozen=True, slots=True)
@ -464,9 +486,9 @@ def _apply(writes: Sequence[_Assign | _Remove]) -> None:
for write in writes:
match write:
case _Assign(holder=(container, key), value=value):
container[key] = value
_assignable(container)[key] = value
case _Remove(holder=(container, key)):
del container[key]
del _deletable(container)[key]
def _block_text_holder(block: object) -> _ContentHolder | None:
@ -545,7 +567,7 @@ def _edits_drop_a_text_change(
if writes is None:
return True
_apply(writes)
edited: Final = [container[key] for container, key in holders]
edited: Final = [subscript(container, key) for container, key in holders]
return any(
returned != before and after == before for before, returned, after in zip(texts, guardrailed_texts, edited)
)
@ -562,26 +584,25 @@ def _extract_converse_output_texts(
``citationsContent.content[].text`` -- while leaving structural values such
as reasoning signatures and citation sources untouched.
"""
holders: Final[list[_StringHolder]] = []
leaves: Final[list[_StringLeaf]] = []
for block in content_blocks:
if not isinstance(block, dict):
continue
_collect_block_text(block, holders)
_collect_block_text(block, leaves)
tool_use = block.get("toolUse")
if isinstance(tool_use, dict):
_collect_strings(tool_use.get("input"), holders)
_collect_strings(tool_use.get("input"), leaves)
reasoning = block.get("reasoningContent")
if isinstance(reasoning, dict):
reasoning_text = reasoning.get("reasoningText")
if isinstance(reasoning_text, dict):
_collect_block_text(reasoning_text, holders)
_collect_block_text(reasoning_text, leaves)
citations = block.get("citationsContent")
if isinstance(citations, dict):
for cited in citations.get("content") or []:
if isinstance(cited, dict):
_collect_block_text(cited, holders)
texts: Final = [container[key] for container, key in holders]
return texts, holders
_collect_block_text(cited, leaves)
return [text for _, text in leaves], [holder for holder, _ in leaves]
def _write_back_texts(
@ -598,7 +619,7 @@ def _write_back_texts(
for idx, (container, key) in enumerate(holders):
if idx >= len(guardrailed_texts):
break
container[key] = guardrailed_texts[idx]
_assignable(container)[key] = guardrailed_texts[idx]
_GroupKey: TypeAlias = str | tuple[str, int]

25
litellm/llms/bedrock/realtime/transformation.py
View file

@ -18,6 +18,7 @@ from pydantic import ConfigDict, JsonValue, TypeAdapter
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.core_helpers import decoded_json_object, subscript
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection
from litellm.llms.bedrock.common_utils import BedrockError
@ -918,7 +919,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
def transform_content_start_event(
self,
event: dict,
event: object,
current_response_id: str | None,
current_output_item_id: str | None,
current_conversation_id: str | None,
@ -942,7 +943,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
Returns:
Tuple of (events, response_id, output_item_id, conversation_id, delta_type)
"""
content_start: Final = event["contentStart"]
content_start: Final = decoded_json_object(subscript(event, "contentStart"))
role: Final = content_start.get("role")
if role != "ASSISTANT":
@ -1228,7 +1229,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
def transform_content_end_event(
self,
event: dict,
event: object,
current_output_item_id: str | None,
current_response_id: str | None,
current_delta_type: str | None,
@ -1247,7 +1248,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
Returns:
Tuple of (events, reset_delta_chunks)
"""
content_end: Final = event["contentEnd"]
content_end: Final = subscript(event, "contentEnd")
verbose_logger.debug("Handling contentEnd: %s", content_end)
if self._assistant_final_text_active:
@ -1338,7 +1339,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
def transform_prompt_end_event(
self,
event: dict,
event: object,
current_response_id: str | None,
current_conversation_id: str | None,
) -> tuple[
@ -1400,7 +1401,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
def transform_tool_use_event(
self,
event: dict,
event: object,
current_response_id: str | None,
current_conversation_id: str | None,
) -> tuple[list[OpenAIRealtimeEvents], str, str]:
@ -1416,7 +1417,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
Tuple of (events, response_id, conversation_id)
"""
verbose_logger.debug("Handling toolUse")
tool_use: Final = BedrockToolUse.model_validate(event["toolUse"])
tool_use: Final = BedrockToolUse.model_validate(subscript(event, "toolUse"))
verbose_logger.debug("Tool use event: %s (ID: %s)", tool_use.toolName, tool_use.toolUseId)
response_id: Final = current_response_id or f"resp_{uuid.uuid4()}"
@ -1668,7 +1669,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
current_conversation_id,
current_delta_type,
) = self.transform_content_start_event(
event,
_JSON_VALUE.validate_python(event),
current_response_id,
current_output_item_id,
current_conversation_id,
@ -1710,7 +1711,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
elif "contentEnd" in event:
events, current_delta_chunks = self.transform_content_end_event(
event,
_JSON_VALUE.validate_python(event),
current_output_item_id,
current_response_id,
current_delta_type,
@ -1735,7 +1736,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
elif "toolUse" in event:
events, current_response_id, current_conversation_id = self.transform_tool_use_event(
event, current_response_id, current_conversation_id
_JSON_VALUE.validate_python(event), current_response_id, current_conversation_id
)
returned_messages.extend(events)
@ -1745,7 +1746,9 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
current_output_item_id,
current_response_id,
current_delta_type,
) = self.transform_prompt_end_event(event, current_response_id, current_conversation_id)
) = self.transform_prompt_end_event(
_JSON_VALUE.validate_python(event), current_response_id, current_conversation_id
)
returned_messages.extend(events)
return {

17
litellm/llms/bedrock/vector_stores/transformation.py
View file

@ -4,9 +4,11 @@ from typing import TYPE_CHECKING, Any, Final, Protocol, cast, runtime_checkable
from urllib.parse import urlparse
import httpx
from pydantic import TypeAdapter
from typing_extensions import TypedDict
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import decoded_json_object
from litellm.litellm_core_utils.core_helpers import decoded_json_object, keyword_arguments
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
@ -61,6 +63,13 @@ def _vector_search_configuration(retrieval_config: object) -> Mapping[str, objec
return decoded_json_object(decoded_json_object(retrieval_config).get("vectorSearchConfiguration", {}))
_DECODED_JSON: Final = TypeAdapter(object)
class _KBResponseFields(TypedDict, total=False, extra_items=object):
pass
class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
"""Vector store configuration for AWS Bedrock Knowledge Bases."""
@ -361,7 +370,11 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
self, response: httpx.Response, litellm_logging_obj: LiteLLMLoggingObj
) -> VectorStoreSearchResponse:
try:
response_data: Final = BedrockKBResponse(**response.json())
response_data: Final = BedrockKBResponse(
**_KBResponseFields(
**keyword_arguments(_DECODED_JSON.validate_python(response.json()), BedrockKBResponse)
)
)
results: Final[list[VectorStoreSearchResult]] = []
for item in response_data.get("retrievalResults", []) or []:
content: BedrockKBContent | None = item.get("content")

16
litellm/llms/codestral/completion/transformation.py
View file

@ -1,10 +1,20 @@
import json
from typing import Final
from pydantic import TypeAdapter
from typing_extensions import TypedDict
import litellm
from litellm.litellm_core_utils.core_helpers import keyword_arguments
from litellm.llms.openai.completion.transformation import OpenAITextCompletionConfig
from litellm.types.llms.databricks import GenericStreamingChunk
_DECODED_JSON: Final = TypeAdapter(object)
class _StreamChunkFields(TypedDict, total=False, extra_items=object):
pass
class CodestralTextCompletionConfig(OpenAITextCompletionConfig):
"""
@ -100,7 +110,11 @@ class CodestralTextCompletionConfig(OpenAITextCompletionConfig):
"finish_reason": finish_reason,
}
original_chunk: Final = litellm.ModelResponseStream(**chunk_data_dict)
original_chunk: Final = litellm.ModelResponseStream(
**_StreamChunkFields(
**keyword_arguments(_DECODED_JSON.validate_python(chunk_data_dict), litellm.ModelResponseStream)
)
)
_choices: Final = chunk_data_dict.get("choices", []) or []
if len(_choices) == 0:
return {

8
litellm/llms/cohere/common_utils.py
View file

@ -397,8 +397,12 @@ class CohereV2ModelResponseIterator:
if index != -1:
str_line = str_line[index:]
data_json: Final = json.loads(str_line)
return self.chunk_parser(chunk=data_json)
data_json: Final = _DECODED_JSON.validate_python(json.loads(str_line))
if isinstance(data_json, dict):
return self.chunk_parser(chunk=data_json)
raise ValueError(
f"Failed to parse v2 chunk: {type(data_json).__name__!r} object has no attribute 'get', chunk: {data_json}"
)
# Async iterator
def __aiter__(self):

22
litellm/llms/cohere/embed/v1_transformation.py
View file

@ -2,12 +2,14 @@
Legacy /v1/embedding transformation logic for Bedrock Cohere.
"""
from collections.abc import Sized
from collections.abc import Mapping, Sized
from typing import Final, Protocol
import httpx
from pydantic import TypeAdapter
from litellm import COHERE_DEFAULT_EMBEDDING_INPUT_TYPE
from litellm.litellm_core_utils.core_helpers import as_str_mapping, subscript
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.custom_httpx.http_handler import header_value
from litellm.types.llms.bedrock import (
@ -24,6 +26,17 @@ class _SupportsEncode(Protocol):
def encode(self, text: str, /) -> Sized: ...
_DECODED_JSON: Final = TypeAdapter(object)
def _embed_response_body(response_json: object) -> Mapping[str, object]:
body: Final = as_str_mapping(response_json)
if body is None:
subscript(response_json, "embeddings")
raise TypeError(f"'{type(response_json).__name__}' object is not subscriptable")
return body
def _bedrock_billed_input_tokens(response: httpx.Response) -> int | None:
count: Final = header_value(response.headers, "x-amzn-bedrock-input-token-count")
return int(count) if count is not None and count.isdigit() else None
@ -119,7 +132,7 @@ class CohereEmbeddingConfig:
encoding: _SupportsEncode,
input: list,
) -> EmbeddingResponse:
response_json: Final = response.json()
response_json: Final = _DECODED_JSON.validate_python(response.json())
## LOGGING
logging_obj.post_call(
input=input,
@ -127,13 +140,14 @@ class CohereEmbeddingConfig:
additional_args={"complete_input_dict": data},
original_response=response_json,
)
billed_input_tokens: Final = _bedrock_billed_input_tokens(response) if "images" in data else None
return self._populate_embedding_response(
response_json=response_json,
response_json=dict(_embed_response_body(response_json)),
model_response=model_response,
model=model,
encoding=encoding,
input=input,
billed_input_tokens=_bedrock_billed_input_tokens(response) if "images" in data else None,
billed_input_tokens=billed_input_tokens,
)
def transform_response(

72
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -632,6 +632,7 @@ _OPTIONAL_PARAMS: Final = TypeAdapter(Mapping[str, object])
_JSON_OBJECT: Final = TypeAdapter(dict[str, object])
_JSON_OBJECTS: Final = TypeAdapter(list[dict[str, object]])
_DECODED_JSON: Final = TypeAdapter(object)
_AGENTIC_HOOK_RESULT: Final = TypeAdapter(object)
def _decoded_body_headers(response: httpx.Response) -> httpx.Headers:
@ -906,7 +907,7 @@ class BaseLLMHTTPHandler:
stream: bool | None = False,
fake_stream: bool = False,
api_key: str | None = None,
headers: dict[str, Any] | None = None,
headers: dict[str, object] | None = None,
client: HTTPHandler | AsyncHTTPHandler | None = None,
provider_config: BaseConfig | None = None,
shared_session: Optional["ClientSession"] = None,
@ -1407,7 +1408,7 @@ class BaseLLMHTTPHandler:
api_key: str | None = None,
client: HTTPHandler | AsyncHTTPHandler | None = None,
aembedding: bool | None = False,
headers: dict[str, Any] | None = None,
headers: dict[str, object] | None = None,
) -> EmbeddingResponse:
provider_config: Final = ProviderConfigManager.get_provider_embedding_config(
model=model, provider=litellm.LlmProviders(custom_llm_provider)
@ -1588,7 +1589,7 @@ class BaseLLMHTTPHandler:
api_key: str | None = None,
api_base: str | None = None,
client: HTTPHandler | AsyncHTTPHandler | None = None,
litellm_params: dict[str, Any] | None = None,
litellm_params: dict[str, object] | None = None,
) -> RerankResponse:
# get config from model, custom llm provider
headers = provider_config.validate_environment(
@ -6146,7 +6147,7 @@ class BaseLLMHTTPHandler:
fingerprint: str,
stream: bool = False,
callback: Optional["CustomLogger"] = None,
) -> AnthropicMessagesResponse | AsyncIterator[object]:
) -> object:
from litellm.anthropic_interface import messages as anthropic_messages
patch: Final = plan.request_patch or AgenticLoopRequestPatch()
@ -6187,7 +6188,7 @@ class BaseLLMHTTPHandler:
kwargs_for_followup["max_agentic_loops"] = max_loops
kwargs_for_followup["_agentic_loop_fingerprints"] = [*fingerprints, fingerprint]
response: AnthropicMessagesResponse | AsyncIterator[object] = await anthropic_messages.acreate(
response: Final[AnthropicMessagesResponse | AsyncIterator[object]] = await anthropic_messages.acreate(
**{
"max_tokens": max_tokens,
"messages": patch.messages,
@ -6200,9 +6201,7 @@ class BaseLLMHTTPHandler:
if callback is not None:
try:
response = await callback.async_post_agentic_loop_response_hook(
response=response, plan=plan, kwargs=kwargs
)
return await callback.async_post_agentic_loop_response_hook(response=response, plan=plan, kwargs=kwargs)
except Exception as e:
_call_id: Final = getattr(logging_obj, "litellm_call_id", "unknown")
verbose_logger.exception(
@ -6227,7 +6226,7 @@ class BaseLLMHTTPHandler:
fingerprints: Sequence[str],
fingerprint: str,
callback: Optional["CustomLogger"] = None,
) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator:
) -> object:
patch: Final = plan.request_patch or AgenticLoopRequestPatch()
if patch.messages is None:
raise ValueError("Agentic loop plan missing patched responses input")
@ -6263,7 +6262,7 @@ class BaseLLMHTTPHandler:
try:
with internal_sub_call(logging_obj.litellm_call_id):
response: ResponsesAPIResponse | BaseResponsesAPIStreamingIterator = await litellm.aresponses(
response: Final[ResponsesAPIResponse | BaseResponsesAPIStreamingIterator] = await litellm.aresponses(
model=patch.model or model,
input=patch.messages, # pyright: ignore[reportArgumentType] # pre-existing mismatch surfaced by the Router import; patch messages are valid response input at runtime
**optional_params,
@ -6272,7 +6271,7 @@ class BaseLLMHTTPHandler:
if callback is not None:
try:
response = await callback.async_post_agentic_loop_response_hook(
return await callback.async_post_agentic_loop_response_hook(
response=response, plan=plan, kwargs=kwargs
)
except Exception as e:
@ -6501,21 +6500,23 @@ class BaseLLMHTTPHandler:
callback.__class__.async_build_agentic_loop_plan is not CustomLogger.async_build_agentic_loop_plan
)
if not build_plan_overridden:
agentic_result: object = await bill_first_round_on_failure(
callback.async_run_agentic_loop(
tools=tool_calls,
model=model,
messages=messages,
response=response,
anthropic_messages_provider_config=anthropic_messages_provider_config,
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
agentic_result: object = _AGENTIC_HOOK_RESULT.validate_python(
await bill_first_round_on_failure(
callback.async_run_agentic_loop(
tools=tool_calls,
model=model,
messages=messages,
response=response,
anthropic_messages_provider_config=anthropic_messages_provider_config,
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
logging_obj=logging_obj,
stream=stream,
kwargs=kwargs_with_provider,
),
logging_obj=logging_obj,
response=response,
stream=stream,
kwargs=kwargs_with_provider,
),
logging_obj=logging_obj,
response=response,
stream=stream,
)
)
return self._maybe_wrap_in_fake_stream(agentic_result, logging_obj, api_surface)
@ -6536,8 +6537,9 @@ class BaseLLMHTTPHandler:
stream=stream,
)
if plan.response_override is not None:
return self._maybe_wrap_in_fake_stream(plan.response_override, logging_obj, api_surface)
response_override: object = plan.response_override
if response_override is not None:
return self._maybe_wrap_in_fake_stream(response_override, logging_obj, api_surface)
if plan.terminate:
verbose_logger.debug(
"Agentic loop terminated by callback=%s reason=%s",
@ -6545,10 +6547,12 @@ class BaseLLMHTTPHandler:
plan.stop_reason,
)
return self._maybe_wrap_in_fake_stream(
await callback.async_post_agentic_loop_response_hook(
response=self._finalize_refused_agentic_response(response=response, tool_calls=tool_calls),
plan=plan,
kwargs=kwargs_with_provider,
_AGENTIC_HOOK_RESULT.validate_python(
await callback.async_post_agentic_loop_response_hook(
response=self._finalize_refused_agentic_response(response=response, tool_calls=tool_calls),
plan=plan,
kwargs=kwargs_with_provider,
)
),
logging_obj,
api_surface,
@ -12396,10 +12400,10 @@ class BaseLLMHTTPHandler:
def generate_content_handler(
self,
model: str,
contents: Any,
contents: object,
generate_content_provider_config: BaseGoogleGenAIGenerateContentConfig,
generate_content_config_dict: dict,
tools: Any,
tools: object,
custom_llm_provider: str,
litellm_params: GenericLiteLLMParams,
logging_obj: LiteLLMLoggingObj,
@ -12517,10 +12521,10 @@ class BaseLLMHTTPHandler:
async def async_generate_content_handler(
self,
model: str,
contents: Any,
contents: object,
generate_content_provider_config: BaseGoogleGenAIGenerateContentConfig,
generate_content_config_dict: dict,
tools: Any,
tools: object,
custom_llm_provider: str,
litellm_params: GenericLiteLLMParams,
logging_obj: LiteLLMLoggingObj,

33
litellm/llms/deprecated_providers/aleph_alpha.py
View file

@ -1,14 +1,30 @@
import json
import time
import types
from collections.abc import Callable, Mapping
from collections.abc import Callable, Mapping, Sized
from typing import Final
import httpx
from pydantic import TypeAdapter
import litellm
from litellm.litellm_core_utils.core_helpers import iter_items, json_contains, subscript
from litellm.utils import Choices, Message, ModelResponse, Usage
_DECODED_JSON: Final = TypeAdapter(object)
_STR: Final = TypeAdapter(str)
def _has_length(value: object) -> bool:
if isinstance(value, Sized):
return len(value) > 0
raise TypeError(f"object of type '{type(value).__name__}' has no len()")
def _completion_message(item: object) -> Message:
completion: Final = subscript(item, "completion")
return Message(content=_STR.validate_python(completion) if _has_length(completion) else None)
class AlephAlphaError(Exception):
def __init__(self, status_code, message):
@ -249,22 +265,19 @@ def completion(
)
print_verbose(f"raw model_response: {response.text}")
## RESPONSE OBJECT
completion_response: Final = response.json()
if "error" in completion_response:
completion_response: Final = _DECODED_JSON.validate_python(response.json())
if json_contains(completion_response, "error"):
raise AlephAlphaError(
message=completion_response["error"],
message=subscript(completion_response, "error"),
status_code=response.status_code,
)
else:
try:
choices_list: Final = []
for idx, item in enumerate(completion_response["completions"]):
if len(item["completion"]) > 0:
message_obj = Message(content=item["completion"])
else:
message_obj = Message(content=None)
for idx, item in enumerate(iter_items(subscript(completion_response, "completions"))):
message_obj = _completion_message(item)
choice_obj = Choices(
finish_reason=item["finish_reason"],
finish_reason=subscript(item, "finish_reason"),
index=idx + 1,
message=message_obj,
)

2
litellm/llms/edenai/videos/transformation.py
View file

@ -142,7 +142,7 @@ class EdenAIVideoConfig(OpenAIVideoConfig):
raw_response: httpx.Response,
logging_obj: "LiteLLMLoggingObj",
custom_llm_provider: str | None = None,
) -> dict[str, str]: # mutable-ok: inherited contract
) -> object:
raw_response.raise_for_status() # the shared GET helpers return error bodies instead of raising
return super().transform_video_list_response(
raw_response=raw_response, logging_obj=logging_obj, custom_llm_provider=custom_llm_provider

7
litellm/llms/fireworks_ai/chat/transformation.py
View file

@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Final, Literal, cast
import httpx
from pydantic import ConfigDict, TypeAdapter
from typing_extensions import TypedDict
import litellm
from litellm._logging import verbose_logger
@ -65,6 +66,10 @@ _STR: Final = TypeAdapter(str)
_DECODED_JSON: Final = TypeAdapter(object)
class _FunctionFields(TypedDict, total=False, extra_items=object):
pass
def _map_reasoning_effort(value: object) -> object:
effort: Final[object] = cast(Mapping[str, object], value).get("effort") if isinstance(value, Mapping) else value
if effort is True:
@ -718,7 +723,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
"""
if tool_calls is not None and message.content is not None and message.tool_calls is None:
try:
function: Final = Function(**json.loads(message.content))
function: Final = Function(**_FunctionFields(**_JSON_DICT.validate_python(json.loads(message.content))))
if function.name != RESPONSE_FORMAT_TOOL_NAME and function.name in [
tool["function"]["name"] for tool in tool_calls
]:

30
litellm/llms/gigachat/chat/transformation.py
View file

@ -17,6 +17,7 @@ import httpx
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object
from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
@ -452,36 +453,37 @@ class GigaChatConfig(BaseConfig):
choices: Final[list[Choices]] = [] # mutable-ok: accumulator for building response choices
for choice in response_json.get("choices", []):
message_data = choice.get("message", {})
finish_reason = choice.get("finish_reason", "stop")
finish_reason = _JSON_VALUE.validate_python(choice.get("finish_reason", "stop"))
if finish_reason == "function_call" and message_data.get("function_call"):
func_call = message_data["function_call"]
function_call = (
decoded_json_object(_JSON_VALUE.validate_python(message_data)).get("function_call")
if finish_reason == "function_call"
else None
)
if function_call:
func_call = decoded_json_object(function_call)
args = func_call.get("arguments", {})
args_mapping = as_str_mapping(args)
if is_structured_output:
if isinstance(args, dict):
content = json.dumps(args, ensure_ascii=False)
else:
content = str(args)
message_data["content"] = content
message_data.pop("function_call", None)
message_data.pop("functions_state_id", None)
message_data["content"] = (
json.dumps(args_mapping, ensure_ascii=False) if args_mapping is not None else str(args)
)
finish_reason = "stop"
else:
if isinstance(args, dict):
args = json.dumps(args, ensure_ascii=False)
arguments = json.dumps(args_mapping, ensure_ascii=False) if args_mapping is not None else args
message_data["tool_calls"] = [
{
"id": f"call_{uuid.uuid4().hex[:24]}",
"type": "function",
"function": {
"name": func_call.get("name", ""),
"arguments": args,
"arguments": arguments,
},
}
]
message_data.pop("function_call", None)
finish_reason = "tool_calls"
message_data.pop("function_call", None)
message_data.pop("functions_state_id", None)

42
litellm/llms/gigachat/embedding/transformation.py
View file

@ -9,12 +9,13 @@ from __future__ import annotations
import types
from collections.abc import Mapping
from typing import Final
from typing import Final, Protocol, runtime_checkable
import httpx
from pydantic import ConfigDict, TypeAdapter
from litellm import LlmProviders
from litellm.litellm_core_utils.core_helpers import iter_items, json_contains, subscript
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@ -26,6 +27,28 @@ from litellm.types.utils import EmbeddingResponse
from ..authenticator import get_access_token
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
_DECODED_JSON: Final = TypeAdapter(object)
@runtime_checkable
class _SupportsDelItem(Protocol):
def __delitem__(self, key: str, /) -> None: ...
@runtime_checkable
class _SupportsSetItem(Protocol):
def __setitem__(self, key: str, value: object, /) -> None: ...
def _add_prompt_tokens(total_tokens: float, embedding: object) -> float:
if not (json_contains(embedding, "usage") and json_contains(subscript(embedding, "usage"), "prompt_tokens")):
return total_tokens
prompt_tokens: Final = subscript(subscript(embedding, "usage"), "prompt_tokens")
if isinstance(prompt_tokens, int | float):
return total_tokens + prompt_tokens
raise TypeError(
f"unsupported operand type(s) for +=: '{type(total_tokens).__name__}' and '{type(prompt_tokens).__name__}'"
)
class GigaChatEmbeddingError(BaseLLMException):
@ -144,7 +167,7 @@ class GigaChatEmbeddingConfig(BaseEmbeddingConfig):
"model": "Embeddings"
}
"""
response_json: Final = raw_response.json()
response_json: Final = _DECODED_JSON.validate_python(raw_response.json())
# Log response
logging_obj.post_call(
@ -155,16 +178,19 @@ class GigaChatEmbeddingConfig(BaseEmbeddingConfig):
)
# Calculate total tokens from individual embeddings
total_tokens = 0
if "data" in response_json:
for emb in response_json["data"]:
if "usage" in emb and "prompt_tokens" in emb["usage"]:
total_tokens += emb["usage"]["prompt_tokens"]
total_tokens: float = 0
if json_contains(response_json, "data"):
for emb in iter_items(subscript(response_json, "data")):
total_tokens = _add_prompt_tokens(total_tokens, emb)
# Remove usage from individual embeddings (not part of OpenAI format)
if "usage" in emb:
if json_contains(emb, "usage"):
if not isinstance(emb, _SupportsDelItem):
raise TypeError(f"'{type(emb).__name__}' object does not support item deletion")
del emb["usage"]
# Set overall usage
if not isinstance(response_json, _SupportsSetItem):
raise TypeError(f"'{type(response_json).__name__}' object does not support item assignment")
response_json["usage"] = {
"prompt_tokens": total_tokens,
"total_tokens": total_tokens,

66
litellm/llms/huggingface/embedding/transformation.py
View file

@ -1,13 +1,15 @@
import json
import os
import time
from collections.abc import Mapping, Sequence
from collections.abc import Mapping, Sequence, Sized
from copy import deepcopy
from typing import TYPE_CHECKING, Any, Final, Protocol
import httpx
from pydantic import TypeAdapter
import litellm
from litellm.litellm_core_utils.core_helpers import iter_items, subscript
from litellm.litellm_core_utils.prompt_templates.common_utils import (
convert_content_list_to_str,
)
@ -44,6 +46,27 @@ def _header_value(headers: Mapping[str, str], key: str, default: str) -> str:
return headers.get(key, default)
_DECODED_JSON: Final = TypeAdapter(object)
def _json_len(value: object) -> int:
if isinstance(value, Sized):
return len(value)
raise TypeError(f"object of type '{type(value).__name__}' has no len()")
def _generated_text(value: object) -> str:
if isinstance(value, str):
return value
raise AttributeError(f"'{type(value).__name__}' object has no attribute 'strip'", name="strip", obj=value)
def _add_logprob(total: float, logprob: object) -> float:
if isinstance(logprob, (int, float)):
return total + logprob
raise TypeError(f"unsupported operand type(s) for +=: '{type(total).__name__}' and '{type(logprob).__name__}'")
tgi_models_cache = None
conv_models_cache = None
@ -407,33 +430,41 @@ class HuggingFaceEmbeddingConfig(BaseConfig):
headers=None,
)
if len(completion_response[0]["generated_text"]) > 0:
model_response.choices[0].message.content = output_parser(completion_response[0]["generated_text"])
if _json_len(_DECODED_JSON.validate_python(completion_response[0]["generated_text"])) > 0:
model_response.choices[0].message.content = output_parser(
_generated_text(_DECODED_JSON.validate_python(completion_response[0]["generated_text"]))
)
## GETTING LOGPROBS + FINISH REASON
if "details" in completion_response[0] and "tokens" in completion_response[0]["details"]:
model_response.choices[0].finish_reason = completion_response[0]["details"]["finish_reason"]
sum_logprob = 0
for token in completion_response[0]["details"]["tokens"]:
if token["logprob"] is not None:
sum_logprob += token["logprob"]
for token in iter_items(_DECODED_JSON.validate_python(completion_response[0]["details"]["tokens"])):
logprob = subscript(token, "logprob")
if logprob is not None:
sum_logprob = _add_logprob(sum_logprob, logprob)
setattr(model_response.choices[0].message, "_logprob", sum_logprob)
if "best_of" in optional_params and optional_params["best_of"] > 1:
if "details" in completion_response[0] and "best_of_sequences" in completion_response[0]["details"]:
choices_list: Final = []
for idx, item in enumerate(completion_response[0]["details"]["best_of_sequences"]):
for idx, item in enumerate(
iter_items(
_DECODED_JSON.validate_python(completion_response[0]["details"]["best_of_sequences"])
)
):
sum_logprob = 0
for token in item["tokens"]:
if token["logprob"] is not None:
sum_logprob += token["logprob"]
if len(item["generated_text"]) > 0:
for token in iter_items(subscript(item, "tokens")):
logprob = subscript(token, "logprob")
if logprob is not None:
sum_logprob = _add_logprob(sum_logprob, logprob)
if _json_len(subscript(item, "generated_text")) > 0:
message_obj = Message(
content=output_parser(item["generated_text"]),
content=output_parser(_generated_text(subscript(item, "generated_text"))),
logprobs=sum_logprob,
)
else:
message_obj = Message(content=None)
choice_obj = Choices(
finish_reason=item["finish_reason"],
finish_reason=subscript(item, "finish_reason"),
index=idx + 1,
message=message_obj,
)
@ -442,8 +473,13 @@ class HuggingFaceEmbeddingConfig(BaseConfig):
elif task == "text-classification":
model_response.choices[0].message.content = json.dumps(completion_response)
else:
if isinstance(completion_response, list) and len(completion_response[0]["generated_text"]) > 0:
model_response.choices[0].message.content = output_parser(completion_response[0]["generated_text"])
if (
isinstance(completion_response, list)
and _json_len(_DECODED_JSON.validate_python(completion_response[0]["generated_text"])) > 0
):
model_response.choices[0].message.content = output_parser(
_generated_text(_DECODED_JSON.validate_python(completion_response[0]["generated_text"]))
)
## CALCULATING USAGE
prompt_tokens = 0
try:

2
litellm/llms/minimax/videos/transformation.py
View file

@ -803,7 +803,7 @@ class MinimaxVideoConfig(BaseVideoConfig):
params.update(extra_query)
return f"{api_base}/v2/query/video_generation", params
def transform_video_list_response( # pyright: ignore[reportIncompatibleMethodOverride] # base declares dict[str, str] but the payload is a heterogeneous list body
def transform_video_list_response(
self,
raw_response: httpx.Response,
logging_obj: "Logging",

14
litellm/llms/nlp_cloud/chat/transformation.py
View file

@ -1,9 +1,10 @@
import json
import time
from collections.abc import Mapping
from collections.abc import Mapping, Sized
from typing import TYPE_CHECKING, Any, Final
import httpx
from pydantic import TypeAdapter
from litellm.litellm_core_utils.prompt_templates.common_utils import (
convert_content_list_to_str,
@ -23,6 +24,15 @@ else:
LoggingClass = Any
_DECODED_JSON: Final = TypeAdapter(object)
def _json_len(value: object) -> int:
if isinstance(value, Sized):
return len(value)
raise TypeError(f"object of type '{type(value).__name__}' has no len()")
class NLPCloudConfig(BaseConfig):
"""
Reference: https://docs.nlpcloud.com/#generation
@ -199,7 +209,7 @@ class NLPCloudConfig(BaseConfig):
)
else:
try:
if len(completion_response["generated_text"]) > 0:
if _json_len(_DECODED_JSON.validate_python(completion_response["generated_text"])) > 0:
model_response.choices[0].message.content = completion_response["generated_text"]
except Exception:
raise NLPCloudError(

24
litellm/llms/nvidia_riva/audio_transcription/riva_grpc.py
View file

@ -6,11 +6,12 @@ request carries them exactly once over both plaintext and TLS.
"""
from collections.abc import Iterator, Mapping, Sequence
from typing import Any, Final, TypeAlias
from typing import Final, Protocol, SupportsIndex, SupportsInt, TypeAlias, runtime_checkable
import grpc
from google.protobuf.message import Message
from pydantic import TypeAdapter
from typing_extensions import Buffer
from litellm.llms.nvidia_riva.audio_transcription import riva_asr_schema as schema
from litellm.llms.nvidia_riva.common_utils import NvidiaRivaException
@ -38,16 +39,29 @@ def open_channel(api_base: str, use_ssl: bool) -> grpc.Channel:
return grpc.insecure_channel(api_base)
def recognition_config(config: Mapping[str, Any]) -> Message:
@runtime_checkable
class _SupportsTrunc(Protocol):
def __trunc__(self) -> int: ...
def _native_int(value: object) -> int:
if isinstance(value, (str, Buffer, SupportsInt, SupportsIndex, _SupportsTrunc)):
return int(value)
raise TypeError(
f"int() argument must be a string, a bytes-like object or a real number, not '{type(value).__name__}'"
)
def recognition_config(config: Mapping[str, object]) -> Message:
endpointing: Final = _endpointing_config(config.get("endpointing_config"))
return schema.RecognitionConfig(
encoding=schema.LINEAR_PCM,
sample_rate_hertz=int(config["sample_rate_hertz"]),
sample_rate_hertz=_native_int(config["sample_rate_hertz"]),
language_code=config["language_code"],
audio_channel_count=int(config["audio_channel_count"]),
audio_channel_count=_native_int(config["audio_channel_count"]),
enable_automatic_punctuation=bool(config.get("enable_automatic_punctuation", True)),
enable_word_time_offsets=bool(config.get("enable_word_time_offsets", False)),
max_alternatives=int(config.get("max_alternatives", 1)),
max_alternatives=_native_int(config.get("max_alternatives", 1)),
model=config.get("model", "") or "",
verbatim_transcripts=bool(config.get("verbatim_transcripts", False)),
profanity_filter=bool(config.get("profanity_filter", False)),

23
litellm/llms/ollama/chat/transformation.py
View file

@ -8,7 +8,7 @@ from pydantic import BaseModel, TypeAdapter
import litellm
from litellm._uuid import uuid
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.core_helpers import decoded_json_object, map_finish_reason
from litellm.litellm_core_utils.prompt_templates.common_utils import (
convert_content_list_to_str,
extract_images_from_message,
@ -47,6 +47,15 @@ else:
LiteLLMLoggingObj = Any
_REQUEST_PARAMS: Final = TypeAdapter(dict[str, object])
_DECODED_JSON: Final = TypeAdapter(object)
def _json_document(value: object) -> str | bytes | bytearray:
if isinstance(value, (str, bytes, bytearray)):
return value
raise TypeError(f"the JSON object must be str, bytes or bytearray, not {value.__class__.__name__}")
_TOOL_CALL_ARGUMENTS: Final = TypeAdapter(dict[str, object])
@ -372,7 +381,7 @@ class OllamaChatConfig(BaseConfig):
# remap 'thinking' to 'reasoning_content'
response_json_message["reasoning_content"] = response_json_message["thinking"]
del response_json_message["thinking"]
elif response_json_message.get("content") is not None:
elif decoded_json_object(_DECODED_JSON.validate_python(response_json_message)).get("content") is not None:
# parse reasoning content from content
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
parse_content_for_reasoning,
@ -387,14 +396,20 @@ class OllamaChatConfig(BaseConfig):
and litellm_params.get("function_name") is not None
and response_json_message is not None
):
function_call: Final = json.loads(response_json_message["content"])
function_call: Final = decoded_json_object(
_DECODED_JSON.validate_python(
json.loads(_json_document(_DECODED_JSON.validate_python(response_json_message["content"])))
)
)
message: Final = litellm.Message(
content=None,
tool_calls=[
{
"id": f"call_{uuid.uuid4()}",
"function": {
"name": function_call.get("name", litellm_params.get("function_name")),
"name": function_call.get(
"name", _DECODED_JSON.validate_python(litellm_params.get("function_name"))
),
"arguments": json.dumps(function_call.get("arguments", function_call)),
},
"type": "function",

13
litellm/llms/oobabooga/chat/oobabooga.py
View file

@ -1,7 +1,9 @@
import json
from collections.abc import Callable
from collections.abc import Callable, Sized
from typing import TYPE_CHECKING, Final
from pydantic import TypeAdapter
import litellm
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.custom_httpx.http_handler import get_httpx_client
@ -15,6 +17,13 @@ if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
oobabooga_config: Final = OobaboogaConfig()
_DECODED_JSON: Final = TypeAdapter(object)
def _json_len(value: object) -> int:
if isinstance(value, Sized):
return len(value)
raise TypeError(f"object of type '{type(value).__name__}' has no len()")
def completion(
@ -149,7 +158,7 @@ def embedding(
}
]
num_tokens: Final = len(completion_response["data"][0]["embedding"])
num_tokens: Final = _json_len(_DECODED_JSON.validate_python(completion_response["data"][0]["embedding"]))
# Adding metadata to response
setattr(
model_response,

27
litellm/llms/openai/chat/gpt_transformation.py
View file

@ -8,15 +8,21 @@ import os
from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
from contextvars import ContextVar
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast, overload
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, cast, overload
from urllib.parse import urlparse
import httpx
from pydantic import TypeAdapter, ValidationError
from typing_extensions import TypedDict
import litellm
from litellm.constants import OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason
from litellm.litellm_core_utils.core_helpers import (
drops_unsupported_params,
map_finish_reason,
require_attributes,
subscript,
)
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
extract_reasoning_content,
handle_invalid_parallel_tool_calls,
@ -147,6 +153,14 @@ _CHOICE_MAPPINGS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
_DECODED_JSON: Final = TypeAdapter(object)
class _ModelResponseStreamFields(TypedDict, total=False, extra_items=object):
pass
class _SupportsGetWithDefault(Protocol):
def get(self, key: str, default: str, /) -> object: ...
def _litellm_params_str(litellm_params: Mapping[str, object] | None, key: str) -> str | None:
value: Final = litellm_params.get(key) if litellm_params is not None else None
return value if isinstance(value, str) else None
@ -808,7 +822,10 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
convert_tool_call_to_json_mode=json_mode,
):
# to support response_format on claude models
json_mode_content_str: str | None = str(new_tool_calls[0]["function"].get("arguments", "")) or None
tool_call_function = require_attributes(
subscript(new_tool_calls[0], "function"), _SupportsGetWithDefault, "get"
)
json_mode_content_str: str | None = str(tool_call_function.get("arguments", "")) or None
if json_mode_content_str is not None:
translated_message = Message(content=json_mode_content_str)
finish_reason = "stop"
@ -1086,7 +1103,7 @@ class OpenAIChatCompletionStreamingHandler(BaseModelResponseIterator):
choices: Final = self._map_reasoning_to_reasoning_content(raw_choices)
fingerprint: Final = _OBJECT_MAPPING.validate_python(chunk).get("system_fingerprint")
kwargs: Final[dict[str, Any]] = {
kwargs: Final[dict[str, object]] = {
"id": chunk.get("id"),
"system_fingerprint": fingerprint,
"object": "chat.completion.chunk",
@ -1099,6 +1116,6 @@ class OpenAIChatCompletionStreamingHandler(BaseModelResponseIterator):
service_tier: Final = chunk.get("service_tier")
if isinstance(service_tier, str) and service_tier:
kwargs["service_tier"] = service_tier
return ModelResponseStream(**kwargs)
return ModelResponseStream(**_ModelResponseStreamFields(**kwargs))
except Exception as e:
raise e

16
litellm/llms/openai/cost_calculation.py
View file

@ -4,7 +4,9 @@ Helper util for handling openai-specific cost calculation
"""
from collections.abc import Mapping
from typing import Any, Final, Literal
from typing import Final, Literal, SupportsFloat, SupportsIndex
from typing_extensions import Buffer
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
@ -100,8 +102,14 @@ def _video_resolution_to_cost_field_suffix(resolution: str) -> str | None:
return safe
def _native_float(value: object) -> float:
if isinstance(value, (str, Buffer, SupportsFloat, SupportsIndex)):
return float(value)
raise TypeError(f"float() argument must be a string or a real number, not '{type(value).__name__}'")
def video_output_cost_per_second(
model_info: Mapping[str, Any],
model_info: Mapping[str, object],
video_resolution: str | None,
) -> float | None:
"""
@ -118,10 +126,10 @@ def video_output_cost_per_second(
tier_key: Final = f"output_cost_per_second_{suffix}"
tier_rate: Final = model_info.get(tier_key)
if tier_rate is not None:
return float(tier_rate)
return _native_float(tier_rate)
out: Final = model_info.get("output_cost_per_second")
if out is not None:
return float(out)
return _native_float(out)
return None

14
litellm/llms/openai/image_edit/transformation.py
View file

@ -3,9 +3,12 @@ from typing import TYPE_CHECKING, Any, Final, cast
import httpx
from httpx._types import RequestFiles
from pydantic import TypeAdapter
from typing_extensions import TypedDict
import litellm
from litellm.images.utils import ImageEditRequestUtils
from litellm.litellm_core_utils.core_helpers import keyword_arguments
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.secret_managers.main import get_secret_str
@ -28,6 +31,13 @@ else:
LiteLLMLoggingObj = Any
_DECODED_JSON: Final = TypeAdapter(object)
class _ImageResponseFields(TypedDict, total=False, extra_items=object):
pass
class OpenAIImageEditConfig(BaseImageEditConfig):
"""
Base configuration for OpenAI image edit API.
@ -149,10 +159,10 @@ class OpenAIImageEditConfig(BaseImageEditConfig):
) -> ImageResponse:
"""No transform applied since outputs are in OpenAI spec already"""
try:
raw_response_json: Final = raw_response.json()
raw_response_json: Final = _DECODED_JSON.validate_python(raw_response.json())
except Exception:
raise OpenAIError(message=raw_response.text, status_code=raw_response.status_code)
return ImageResponse(**raw_response_json)
return ImageResponse(**_ImageResponseFields(**keyword_arguments(raw_response_json, ImageResponse)))
def validate_environment(
self,

18
litellm/llms/openai/image_variations/handler.py
View file

@ -7,6 +7,8 @@ from typing import Final
import httpx
from openai import AsyncOpenAI, OpenAI
from pydantic import BaseModel
from typing_extensions import TypedDict
import litellm
from litellm.types.utils import FileTypes, ImageResponse, LlmProviders
@ -18,6 +20,14 @@ from ..common_utils import OpenAIError, upstream_error_response
from ..workload_identity import build_async_openai_client, build_openai_client
class _ImageResponseFields(TypedDict, total=False, extra_items=object):
pass
def _dumped_fields(model: BaseModel) -> Mapping[str, object]:
return model.model_dump()
class OpenAIImageVariationsHandler:
def get_sync_client(
self,
@ -94,7 +104,7 @@ class OpenAIImageVariationsHandler:
raw_response: Final = await client.images.with_raw_response.create_variation(**data)
response: Final = raw_response.parse()
response_json: Final = response.model_dump()
response_json: Final = _dumped_fields(response)
## LOGGING
logging_obj.post_call(
@ -109,7 +119,7 @@ class OpenAIImageVariationsHandler:
## RESPONSE OBJECT
return provider_config.transform_response_image_variation(
model=model,
model_response=ImageResponse(**response_json),
model_response=ImageResponse(**_ImageResponseFields(**response_json)),
raw_response=httpx.Response(
status_code=200,
request=httpx.Request(method="GET", url="https://litellm.ai"), # mock request object
@ -215,7 +225,7 @@ class OpenAIImageVariationsHandler:
raw_response: Final = client.images.with_raw_response.create_variation(**json_data)
response: Final = raw_response.parse()
response_json: Final = response.model_dump()
response_json: Final = _dumped_fields(response)
## LOGGING
logging_obj.post_call(
@ -230,7 +240,7 @@ class OpenAIImageVariationsHandler:
## RESPONSE OBJECT
return provider_config.transform_response_image_variation(
model=model,
model_response=ImageResponse(**response_json),
model_response=ImageResponse(**_ImageResponseFields(**response_json)),
raw_response=httpx.Response(
status_code=200,
request=httpx.Request(method="GET", url="https://litellm.ai"), # mock request object

47
litellm/llms/openai/responses/guardrail_translation/handler.py
View file

@ -36,7 +36,7 @@ from collections.abc import Iterable, Mapping, MutableSequence, Sequence
from dataclasses import dataclass
from itertools import accumulate, chain, groupby, repeat
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, TypeVar, Union, cast
from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, TypeVar, Union, cast, runtime_checkable
from openai.types.responses.response_output_refusal import ResponseOutputRefusal
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
@ -52,7 +52,7 @@ from litellm.completion_extras.litellm_responses_transformation.transformation i
OpenAiResponsesToChatCompletionStreamIterator,
tool_call_dict_from_output_item,
)
from litellm.litellm_core_utils.core_helpers import has_attributes
from litellm.litellm_core_utils.core_helpers import as_object_list, has_attributes
from litellm.llms.base_llm.guardrail_translation.base_translation import (
BaseTranslation,
StreamingScanKey,
@ -694,6 +694,24 @@ def _is_textless_tool_row(message: Mapping[str, object], texts: tuple[str, ...]
return _is_string_tool_row(message) and (not message.get("content") or texts == ())
@runtime_checkable
class _SupportsSetItem(Protocol):
def __setitem__(self, key: str, value: object, /) -> None: ...
def _assignable(item: object) -> _SupportsSetItem:
if isinstance(item, _SupportsSetItem):
return item
raise TypeError(f"'{type(item).__name__}' object does not support item assignment")
def _summarized_encrypted_content(item: Mapping[str, object]) -> str | None:
if compaction_summary(item) is None:
return None
encrypted_content: Final = item["encrypted_content"]
return encrypted_content if isinstance(encrypted_content, str) else None
def _rewritten_whole_text(item: object, field: str, guardrail_response: str) -> object | None:
if isinstance(stream_item_field(item, field), str):
return guardrail_response
@ -1134,7 +1152,7 @@ class OpenAIResponsesHandler(BaseTranslation):
async def _apply_guardrail_responses_to_input(
self,
messages: Any, # Can be List[Dict[str, Any]] or ResponseInputParam
messages: Sequence[Mapping[str, object]],
responses: Sequence[str],
task_mappings: Sequence[tuple[int, int | None]],
) -> None:
@ -1149,26 +1167,27 @@ class OpenAIResponsesHandler(BaseTranslation):
for guardrail_response, mapping in zip(responses, task_mappings):
msg_idx = cast(int, mapping[0])
content_idx_optional = cast(int | None, mapping[1])
item = messages[msg_idx]
if compaction_summary(messages[msg_idx]) is not None:
messages[msg_idx]["encrypted_content"] = with_compaction_summary( # rebind-ok: in-place input rewrite
messages[msg_idx]["encrypted_content"], guardrail_response
)
encrypted_content = _summarized_encrypted_content(item)
if encrypted_content is not None:
_assignable(item)["encrypted_content"] = with_compaction_summary(encrypted_content, guardrail_response)
continue
field = _scanned_text_field(messages[msg_idx])
content = messages[msg_idx].get(field, None)
field = _scanned_text_field(item)
content = item.get(field, None)
if content is None:
continue
content_parts = as_object_list(content)
if content_idx_optional is None:
rewritten = _rewritten_whole_text(messages[msg_idx], field, guardrail_response)
rewritten = _rewritten_whole_text(item, field, guardrail_response)
if rewritten is not None:
messages[msg_idx][field] = rewritten
_assignable(item)[field] = rewritten
elif isinstance(content, list):
elif content_parts is not None:
# Replace specific text item in list content
self._write_content_part_text(content[content_idx_optional], guardrail_response)
self._write_content_part_text(content_parts[content_idx_optional], guardrail_response)
async def process_output_response(
self,
@ -1299,7 +1318,7 @@ class OpenAIResponsesHandler(BaseTranslation):
request_data: dict | None = None,
stream_transform_sink: StreamTransformSink | None = None,
deliver_ended_stream_rewrites: bool = False,
) -> list[Any]:
) -> Sequence[object]:
"""
Process output streaming response by applying guardrails to text content.

17
litellm/llms/openai/transcriptions/whisper_transformation.py
View file

@ -3,7 +3,10 @@ from collections.abc import Mapping
from typing import Final
from httpx import Headers, Response
from pydantic import TypeAdapter
from typing_extensions import TypedDict
from litellm.litellm_core_utils.core_helpers import json_contains, keyword_arguments
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.audio_transcription.transformation import (
AudioTranscriptionRequestData,
@ -19,6 +22,12 @@ from litellm.types.utils import FileTypes, TranscriptionResponse
from ..common_utils import OpenAIError
_DECODED_JSON: Final = TypeAdapter(object)
class _TranscriptionResponseFields(TypedDict, total=False, extra_items=object):
pass
def _header_value(headers: Mapping[str, str], key: str, default: str) -> str:
return headers.get(key, default)
@ -131,14 +140,16 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
raw_response: Response,
) -> TranscriptionResponse:
try:
raw_response_json: Final = raw_response.json()
raw_response_json: Final = _DECODED_JSON.validate_python(raw_response.json())
except json.JSONDecodeError:
content_type: Final = _header_value(raw_response.headers, "content-type", "").lower()
if "application/json" in content_type:
raise
return TranscriptionResponse(text=raw_response.text)
if any(key in raw_response_json for key in TranscriptionResponse.model_fields):
return TranscriptionResponse(**raw_response_json)
if any(json_contains(raw_response_json, key) for key in TranscriptionResponse.model_fields):
return TranscriptionResponse(
**_TranscriptionResponseFields(**keyword_arguments(raw_response_json, TranscriptionResponse))
)
else:
raise ValueError("Invalid response format. Received response does not match the expected format.")

21
litellm/llms/openai/vector_stores/transformation.py
View file

@ -1,8 +1,11 @@
from typing import TYPE_CHECKING, Any, Final, cast
import httpx
from pydantic import TypeAdapter
from typing_extensions import TypedDict
import litellm
from litellm.litellm_core_utils.core_helpers import keyword_arguments
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig
from litellm.llms.openai.workload_identity import resolve_openai_bearer_token
@ -27,6 +30,12 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObj = Any
_DECODED_JSON: Final = TypeAdapter(object)
class _VectorStoreResponseFields(TypedDict, total=False, extra_items=object):
pass
class OpenAIVectorStoreConfig(BaseVectorStoreConfig):
ASSISTANTS_HEADER_KEY = "OpenAI-Beta"
@ -118,8 +127,10 @@ class OpenAIVectorStoreConfig(BaseVectorStoreConfig):
self, response: httpx.Response, litellm_logging_obj: LiteLLMLoggingObj
) -> VectorStoreSearchResponse:
try:
response_json: Final = response.json()
return VectorStoreSearchResponse(**response_json)
response_json: Final = _DECODED_JSON.validate_python(response.json())
return VectorStoreSearchResponse(
**_VectorStoreResponseFields(**keyword_arguments(response_json, VectorStoreSearchResponse))
)
except Exception as e:
raise self.get_error_class(
error_message=str(e),
@ -149,8 +160,10 @@ class OpenAIVectorStoreConfig(BaseVectorStoreConfig):
def transform_create_vector_store_response(self, response: httpx.Response) -> VectorStoreCreateResponse:
try:
response_json: Final = response.json()
return VectorStoreCreateResponse(**response_json)
response_json: Final = _DECODED_JSON.validate_python(response.json())
return VectorStoreCreateResponse(
**_VectorStoreResponseFields(**keyword_arguments(response_json, VectorStoreCreateResponse))
)
except Exception as e:
raise self.get_error_class(
error_message=str(e),

136
litellm/llms/openai/videos/transformation.py
View file

@ -2,12 +2,21 @@ import mimetypes
from collections.abc import Mapping
from io import BufferedReader, BytesIO
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, cast
from typing import TYPE_CHECKING, Any, Final, Protocol, cast
from urllib.parse import quote
import httpx
from httpx._types import FileContent, FileTypes, RequestFiles
from pydantic import TypeAdapter
from litellm.litellm_core_utils.core_helpers import (
as_object_list,
decoded_json_object,
iter_items,
json_contains,
require_attributes,
subscript,
)
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
from litellm.llms.base_llm.videos.async_task_utils import unsupported_video_fields
from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
@ -39,6 +48,59 @@ else:
BaseLLMException = Any
_REQUEST_VALUE: Final = TypeAdapter(object)
def _dict_fields(value: object) -> Mapping[str, object] | None:
return value if isinstance(value, dict) else None
class _JsonResponse(Protocol):
def json(self) -> object: ...
def _json_body(response: _JsonResponse) -> object:
return response.json()
def _model_label(model: object) -> str | None:
return format(model) if model else None
def _encoded_video_id(video_id: object, provider: str, model: object) -> object:
if not video_id:
return video_id
return encode_video_id_with_provider(
require_attributes(video_id, str, "startswith"),
provider,
_model_label(model),
)
def _with_encoded_id(video: object, provider: str) -> object:
fields: Final = _dict_fields(video)
if fields is None or "id" not in fields:
return video
return {**fields, "id": _encoded_video_id(fields["id"], provider, fields.get("model"))}
def _edge_model(data: object, index: int) -> object:
if not data:
return None
edge: Final = _dict_fields(subscript(data, index))
return None if edge is None else edge.get("model")
def _decoded_character(character: object) -> object:
fields: Final = _dict_fields(character)
if fields is None:
return character
character_id: Final = fields.get("id")
if not isinstance(character_id, str):
return character
return {**fields, "id": extract_original_character_id(character_id)}
class OpenAIVideoConfig(BaseVideoConfig):
"""
Configuration class for OpenAI video generation.
@ -157,25 +219,11 @@ class OpenAIVideoConfig(BaseVideoConfig):
`character_<base64-encoded-provider-payload>`, convert it back to the
original provider id before forwarding upstream.
"""
raw_characters: Final = request_dict.get("characters")
if not isinstance(raw_characters, list):
raw_characters: Final = as_object_list(_REQUEST_VALUE.validate_python(request_dict.get("characters")))
if raw_characters is None:
return request_dict
decoded_characters: Final[list[Any]] = []
for character in raw_characters:
if not isinstance(character, dict):
decoded_characters.append(character)
continue
character_id = character.get("id")
if isinstance(character_id, str):
decoded_character = dict(character)
decoded_character["id"] = extract_original_character_id(character_id)
decoded_characters.append(decoded_character)
else:
decoded_characters.append(character)
request_dict["characters"] = decoded_characters
request_dict["characters"] = [_decoded_character(character) for character in raw_characters]
return request_dict
def transform_video_create_response(
@ -343,41 +391,27 @@ class OpenAIVideoConfig(BaseVideoConfig):
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
custom_llm_provider: str | None = None,
) -> dict[str, str]:
response_data: Final = raw_response.json()
) -> object:
response_data: Final = _json_body(raw_response)
if not custom_llm_provider or not json_contains(response_data, "data"):
return response_data
if custom_llm_provider and "data" in response_data:
for video_obj in response_data.get("data", []):
if isinstance(video_obj, dict) and "id" in video_obj:
video_obj["id"] = encode_video_id_with_provider(
video_obj["id"],
custom_llm_provider,
video_obj.get("model"),
)
fields: Final = decoded_json_object(response_data)
data: Final = fields.get("data", [])
encoded_items: Final = [_with_encoded_id(item, custom_llm_provider) for item in iter_items(data)]
encoded_data: Final = encoded_items if as_object_list(data) is not None else data
# Encode pagination cursor IDs so they remain consistent
# with the wrapped data[].id format
data_list: Final = response_data.get("data", [])
if response_data.get("first_id"):
first_model = None
if data_list and isinstance(data_list[0], dict):
first_model = data_list[0].get("model")
response_data["first_id"] = encode_video_id_with_provider(
response_data["first_id"],
custom_llm_provider,
first_model,
)
if response_data.get("last_id"):
last_model = None
if data_list and isinstance(data_list[-1], dict):
last_model = data_list[-1].get("model")
response_data["last_id"] = encode_video_id_with_provider(
response_data["last_id"],
custom_llm_provider,
last_model,
)
return response_data
# Encode pagination cursor IDs so they remain consistent
# with the wrapped data[].id format
first_id: Final = fields.get("first_id")
encoded_first: Final[Mapping[str, object]] = (
{"first_id": _encoded_video_id(first_id, custom_llm_provider, _edge_model(data, 0))} if first_id else {}
)
last_id: Final = fields.get("last_id")
encoded_last: Final[Mapping[str, object]] = (
{"last_id": _encoded_video_id(last_id, custom_llm_provider, _edge_model(data, -1))} if last_id else {}
)
return {**fields, "data": encoded_data, **encoded_first, **encoded_last}
def transform_video_delete_request(
self,

32
litellm/llms/openai_like/chat/handler.py
View file

@ -5,14 +5,16 @@ For handling OpenAI-like chat completions, like IBM WatsonX, etc.
"""
import json
from collections.abc import Callable, Mapping, Sequence
from typing import Final, TypedDict
from collections.abc import Callable
from typing import Final
import httpx
from typing_extensions import ReadOnly
from pydantic import TypeAdapter
from typing_extensions import TypedDict
import litellm
from litellm import LlmProviders
from litellm.litellm_core_utils.core_helpers import keyword_arguments
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.base_model_iterator import MockResponseIterator
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
@ -26,22 +28,16 @@ from litellm.utils import CustomStreamWrapper, ProviderConfigManager
from ..common_utils import OpenAILikeBase, OpenAILikeError
from .transformation import OpenAILikeChatConfig
class _OpenAILikeChatCompletion(TypedDict, total=False):
"""The chat-completion JSON body an OpenAI-like provider returns for a non-streamed call."""
id: ReadOnly[str]
choices: ReadOnly[Sequence[Mapping[str, object]]]
created: ReadOnly[int]
model: ReadOnly[str]
system_fingerprint: ReadOnly[str]
usage: ReadOnly[Mapping[str, object]]
object: ReadOnly[str]
_DECODED_JSON: Final = TypeAdapter(object)
def _fake_streamed_model_response(payload: _OpenAILikeChatCompletion) -> ModelResponse:
class _ModelResponseFields(TypedDict, total=False, extra_items=object):
pass
def _fake_streamed_model_response(payload: object) -> ModelResponse:
"""Build the single response a fake-streamed provider call replays as one chunk."""
return ModelResponse(**payload)
return ModelResponse(**_ModelResponseFields(**keyword_arguments(payload, ModelResponse)))
async def make_call(
@ -63,7 +59,7 @@ async def make_call(
if streaming_decoder is not None:
completion_stream = streaming_decoder.aiter_bytes(response.aiter_bytes(chunk_size=1024))
elif fake_stream:
model_response: Final = _fake_streamed_model_response(response.json())
model_response: Final = _fake_streamed_model_response(_DECODED_JSON.validate_python(response.json()))
completion_stream = MockResponseIterator(model_response=model_response)
else:
completion_stream = ModelResponseIterator(
@ -103,7 +99,7 @@ def make_sync_call(
if streaming_decoder is not None:
completion_stream = streaming_decoder.iter_bytes(response.iter_bytes(chunk_size=1024))
elif fake_stream:
model_response: Final = _fake_streamed_model_response(response.json())
model_response: Final = _fake_streamed_model_response(_DECODED_JSON.validate_python(response.json()))
completion_stream = MockResponseIterator(model_response=model_response)
else:
completion_stream = ModelResponseIterator(

11
litellm/llms/openai_like/embedding/handler.py
View file

@ -10,9 +10,11 @@ from typing import Final
import httpx
from pydantic import TypeAdapter
from typing_extensions import TypedDict
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import keyword_arguments
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
@ -40,6 +42,11 @@ _TRANSPORT_HEADER_NAMES: Final = frozenset(
)
_HEADER_NAME: Final = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+")
_HEADER_VALUE: Final = re.compile(r"[\t\x20-\x7e]*")
_DECODED_JSON: Final = TypeAdapter(object)
class _EmbeddingResponseFields(TypedDict, total=False, extra_items=object):
pass
def _header_text(name: str, value: object) -> str | None:
@ -108,7 +115,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
response.raise_for_status()
response_json: Final = response.json()
response_json: Final = _DECODED_JSON.validate_python(response.json())
except httpx.HTTPStatusError as e:
raise OpenAILikeError(
status_code=e.response.status_code,
@ -126,7 +133,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
additional_args={"complete_input_dict": data},
original_response=response_json,
)
return EmbeddingResponse(**response_json)
return EmbeddingResponse(**_EmbeddingResponseFields(**keyword_arguments(response_json, EmbeddingResponse)))
except Exception as e:
## LOGGING
logging_obj.post_call(

11
litellm/llms/predibase/chat/transformation.py
View file

@ -5,10 +5,11 @@ from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final, Literal
from httpx import Headers, Response
from pydantic import TypeAdapter
import litellm
from litellm.constants import DEFAULT_MAX_TOKENS
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.core_helpers import json_contains, map_finish_reason, subscript
from litellm.litellm_core_utils.prompt_templates.factory import (
prompt_factory,
render_custom_prompt_template,
@ -28,6 +29,8 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObj = Any
_DECODED_JSON: Final = TypeAdapter(object)
class PredibaseConfig(BaseConfig):
"""
@ -163,13 +166,13 @@ class PredibaseConfig(BaseConfig):
additional_args={"complete_input_dict": request_data},
)
try:
completion_response: Final = raw_response.json()
completion_response: Final = _DECODED_JSON.validate_python(raw_response.json())
except Exception:
raise PredibaseError(message=raw_response.text, status_code=422)
if "error" in completion_response:
if json_contains(completion_response, "error"):
raise PredibaseError(
message=str(completion_response["error"]),
message=str(subscript(completion_response, "error")),
status_code=raw_response.status_code,
)
elif not isinstance(completion_response, dict):

7
litellm/llms/ragflow/vector_stores/transformation.py
View file

@ -41,6 +41,11 @@ class _RAGFlowCreateResponseRoot(RootModel[_RAGFlowCreateResponse]):
model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
class _RAGFlowStatus(TypedDict):
code: ReadOnly[NotRequired[int]]
message: ReadOnly[NotRequired[str]]
_RAGFLOW_EMPTY_DATA: Final[_RAGFlowDataset] = {}
@ -197,7 +202,7 @@ class RAGFlowVectorStoreConfig(BaseVectorStoreConfig):
}
"""
try:
response_json: Final = response.json()
response_json: Final[_RAGFlowStatus] = response.json()
# Check for RAGFlow error response
if response_json.get("code") != 0:

7
litellm/llms/replicate/chat/transformation.py
View file

@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final
import httpx
from pydantic import ConfigDict, TypeAdapter
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm.constants import REPLICATE_MODEL_NAME_WITH_ID_LENGTH
@ -33,6 +34,10 @@ _TEXTS: Final = TypeAdapter(Iterable[str], config=ConfigDict(strict=True, hide_i
_JSON_DICT: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True))
class _ReplicatePrediction(TypedDict):
urls: ReadOnly[NotRequired[Mapping[str, str]]]
class ReplicateConfig(BaseConfig):
"""
Reference: https://replicate.com/meta/llama-2-70b-chat/api
@ -292,7 +297,7 @@ class ReplicateConfig(BaseConfig):
"urls":{"cancel":"https://api.replicate.com/v1/predictions/gqsmqmp1pdrj00cknr08dgmvb4/cancel","get":"https://api.replicate.com/v1/predictions/gqsmqmp1pdrj00cknr08dgmvb4","stream":"https://stream-b.svc.rno2.c.replicate.net/v1/streams/eot4gbydowuin4snhncydwxt57dfwgsc3w3snycx5nid7oef7jga"}
}
"""
response_json: Final = response.json()
response_json: Final[_ReplicatePrediction] = response.json()
prediction_url: Final = response_json.get("urls", {}).get("get")
if prediction_url is None:
raise ReplicateError(

32
litellm/llms/sagemaker/completion/transformation.py
View file

@ -11,9 +11,11 @@ from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
from httpx._models import Headers, Response
from pydantic import TypeAdapter
import litellm
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.core_helpers import as_object_list, json_contains, subscript
from litellm.litellm_core_utils.prompt_templates.factory import (
prompt_factory,
render_custom_prompt_template,
@ -33,6 +35,14 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObj = Any
_DECODED_JSON: Final = TypeAdapter(object)
def _concatenated(prefix: str, value: object) -> str:
if isinstance(value, str):
return prefix + value
raise TypeError(f'can only concatenate str (not "{type(value).__name__}") to str')
class SagemakerConfig(BaseConfig):
"""
@ -199,7 +209,7 @@ class SagemakerConfig(BaseConfig):
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:
completion_response: Final = raw_response.json()
completion_response: Final = _DECODED_JSON.validate_python(raw_response.json())
## LOGGING
logging_obj.post_call(
input=messages,
@ -212,15 +222,19 @@ class SagemakerConfig(BaseConfig):
## RESPONSE OBJECT
try:
if isinstance(completion_response, list):
completion_response_choices = completion_response[0]
else:
completion_response_choices = completion_response
completion_response_list: Final = as_object_list(completion_response)
completion_response_choices: Final = (
completion_response if completion_response_list is None else completion_response_list[0]
)
completion_output = ""
if "generation" in completion_response_choices:
completion_output += completion_response_choices["generation"]
elif "generated_text" in completion_response_choices:
completion_output += completion_response_choices["generated_text"]
if json_contains(completion_response_choices, "generation"):
completion_output = _concatenated(
completion_output, subscript(completion_response_choices, "generation")
)
elif json_contains(completion_response_choices, "generated_text"):
completion_output = _concatenated(
completion_output, subscript(completion_response_choices, "generated_text")
)
# check if the prompt template is part of output, if so - filter it out
if completion_output.startswith(prompt) and "<s>" in prompt:

8
litellm/llms/snowflake/chat/transformation.py
View file

@ -17,7 +17,7 @@ from typing import TYPE_CHECKING, Any, Final, Protocol, TypedDict
import httpx
from typing_extensions import ReadOnly
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
from litellm.litellm_core_utils.prompt_templates.factory import (
anthropic_process_openai_file_message,
convert_to_anthropic_tool_result,
@ -354,13 +354,15 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig):
if isinstance(block, Mapping) and block.get("type") == "text"
)
elif role == "assistant":
tool_calls = msg.get("tool_calls") if isinstance(msg, dict) else getattr(msg, "tool_calls", None)
tool_calls: object = (
msg.get("tool_calls") if isinstance(msg, dict) else getattr(msg, "tool_calls", None)
)
thinking_blocks = _signed_thinking_blocks(msg)
if tool_calls:
content_blocks: list[dict[str, object]] = list(thinking_blocks) # mutable-ok: JSON wire blocks
if content:
content_blocks.append({"type": "text", "text": content})
for tc in tool_calls:
for tc in iter_items(tool_calls):
func = tc.get("function", {}) if isinstance(tc, dict) else getattr(tc, "function", {})
tc_id = tc.get("id", "") if isinstance(tc, dict) else getattr(tc, "id", "")
func_name = func.get("name", "") if isinstance(func, dict) else getattr(func, "name", "")

27
litellm/llms/soniox/audio_transcription/transformation.py
View file

@ -13,11 +13,13 @@ import re
from collections.abc import Mapping
from dataclasses import dataclass
from types import MappingProxyType
from typing import Any, Final, NoReturn
from typing import Any, Final, NoReturn, SupportsFloat, SupportsIndex
from httpx import Headers, Response
from pydantic import TypeAdapter, ValidationError
from typing_extensions import Buffer
from litellm.litellm_core_utils.core_helpers import decoded_json_object
from litellm.llms.base_llm.audio_transcription.transformation import (
AudioTranscriptionRequestData,
BaseAudioTranscriptionConfig,
@ -172,6 +174,12 @@ def _nested_form_param(param: str, fields: tuple[_BracketedField, ...]) -> Mappi
}
def _as_float(value: object) -> float:
if isinstance(value, (str, Buffer, SupportsFloat, SupportsIndex)):
return float(value)
raise TypeError(f"float() argument must be a string or a real number, not {type(value).__name__!r}")
def _is_soniox_context(value: object) -> bool:
try:
_STRUCTURED_CONTEXT.validate_python(value)
@ -345,7 +353,7 @@ class SonioxAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
response_format: str | None = None,
) -> TranscriptionResponse:
"""Shared response-building logic (also used by the handler)."""
transcription_meta: dict[str, Any] = {}
transcription_meta: object = {}
transcript: dict[str, Any]
if isinstance(payload, dict) and "transcript" in payload:
@ -385,9 +393,10 @@ class SonioxAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
response["task"] = "transcribe"
# Best-effort metadata fields matching OpenAI's verbose_json shape.
if transcription_meta.get("audio_duration_ms") is not None:
audio_duration_ms: Final = decoded_json_object(transcription_meta).get("audio_duration_ms")
if audio_duration_ms is not None:
try:
response["duration"] = float(transcription_meta["audio_duration_ms"]) / 1000.0
response["duration"] = _as_float(audio_duration_ms) / 1000.0
except (TypeError, ValueError):
pass
@ -403,10 +412,12 @@ class SonioxAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
words: Final[list[dict[str, object]]] = []
for token in tokens:
word_entry: dict[str, object] = {"word": token.get("text", "")}
if token.get("start_ms") is not None:
word_entry["start"] = float(token["start_ms"]) / 1000.0
if token.get("end_ms") is not None:
word_entry["end"] = float(token["end_ms"]) / 1000.0
start_ms: object = token.get("start_ms")
if start_ms is not None:
word_entry["start"] = _as_float(start_ms) / 1000.0
end_ms: object = token.get("end_ms")
if end_ms is not None:
word_entry["end"] = _as_float(end_ms) / 1000.0
words.append(word_entry)
if words:
response["words"] = words

18
litellm/llms/stability/image_edit/transformations.py
View file

@ -6,13 +6,13 @@ Handles transformation between OpenAI-compatible format and Stability AI API for
API Reference: https://platform.stability.ai/docs/api-reference
"""
from collections.abc import Sequence
from typing import TYPE_CHECKING, Any, Final, cast
import httpx
from httpx._types import RequestFiles
from typing_extensions import ReadOnly, TypedDict
from pydantic import TypeAdapter
from litellm.litellm_core_utils.core_helpers import decoded_json_object, json_contains, subscript
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.images.main import ImageEditOptionalRequestParams
@ -31,12 +31,7 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObj = Any
class _StabilityImageEditResponse(TypedDict, total=False):
image: ReadOnly[str]
finish_reason: ReadOnly[str]
seed: ReadOnly[int]
errors: ReadOnly[Sequence[str]]
_DECODED_JSON: Final = TypeAdapter(object)
class StabilityImageEditConfig(BaseImageEditConfig):
@ -268,7 +263,7 @@ class StabilityImageEditConfig(BaseImageEditConfig):
OpenAI expects: {"data": [{"b64_json": "base64..."}], "created": timestamp}
"""
try:
response_data: Final[_StabilityImageEditResponse] = raw_response.json()
response_json: Final = _DECODED_JSON.validate_python(raw_response.json())
except Exception as e:
raise self.get_error_class(
error_message=f"Error parsing Stability AI response: {e}",
@ -277,14 +272,15 @@ class StabilityImageEditConfig(BaseImageEditConfig):
)
# Check for errors in response
if "errors" in response_data:
if json_contains(response_json, "errors"):
raise self.get_error_class(
error_message=f"Stability AI error: {response_data['errors']}",
error_message=f"Stability AI error: {subscript(response_json, 'errors')}",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
# Check finish_reason
response_data: Final = decoded_json_object(response_json)
finish_reason: Final = response_data.get("finish_reason", "")
if finish_reason == "CONTENT_FILTERED":
raise self.get_error_class(

48
litellm/llms/vertex_ai/common_utils.py
View file

@ -7,7 +7,7 @@ from enum import Enum
from functools import lru_cache
from itertools import accumulate, chain, repeat, takewhile
from types import MappingProxyType
from typing import Any, Final, Literal, TypeVar, cast, get_type_hints
from typing import Final, Literal, TypeVar, cast, get_type_hints
import httpx
from pydantic import JsonValue, TypeAdapter, ValidationError
@ -1384,6 +1384,13 @@ class VertexAIModelInfo(BaseLLMModelInfo):
raise NotImplementedError("Vertex AI models are not supported yet")
_COUNT_TOKENS_REQUEST_FIELDS: Final = frozenset({"model", "contents", "system_instruction", "tools"})
class _ForwardedCountTokensParams(TypedDict, total=False, extra_items=object):
pass
class VertexAITokenCounter(BaseTokenCounter):
"""Token counter implementation for Google AI Studio provider."""
@ -1446,7 +1453,7 @@ class VertexAITokenCounter(BaseTokenCounter):
model_to_use: str,
messages: list[dict[str, object]] | None,
contents: list[dict[str, object]] | None,
deployment: dict[str, Any] | None = None,
deployment: dict[str, object] | None = None,
request_model: str = "",
tools: list[dict[str, object]] | None = None,
system: object | None = None,
@ -1458,6 +1465,7 @@ class VertexAITokenCounter(BaseTokenCounter):
) -> TokenCountResponse | None:
import copy
from litellm.litellm_core_utils.core_helpers import decoded_json_object
from litellm.llms.anthropic.common_utils import context_management_with_edits
from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler import count_tokens_publisher
from litellm.llms.vertex_ai.vertex_ai_partner_models.main import (
@ -1467,19 +1475,19 @@ class VertexAITokenCounter(BaseTokenCounter):
VertexAIPartnerModels,
)
deployment = deployment or {}
count_tokens_params_request: Final = copy.deepcopy(deployment.get("litellm_params", {}))
resolved_deployment: Final = deployment or {}
raw_litellm_params: Final = copy.deepcopy(resolved_deployment.get("litellm_params", {}))
if not VertexAIPartnerModels.is_vertex_partner_model(model_to_use) and not uses_vertex_gemini_api(
model_to_use, count_tokens_params_request
model_to_use, decoded_json_object(raw_litellm_params) if raw_litellm_params else None
):
return None
without_count_route: Final = (
VertexAIPartnerModels.is_vertex_partner_model(model_to_use) and count_tokens_publisher(model_to_use) is None
)
model_info: Final[Mapping[str, object]] = deployment.get("model_info") or {}
if without_count_route and messages and model_info.get("custom_tokenizer") is not None:
model_info: Final[object] = resolved_deployment.get("model_info") or {}
if without_count_route and messages and decoded_json_object(model_info).get("custom_tokenizer") is not None:
return None
if without_count_route:
return TokenCountResponse(
@ -1497,7 +1505,7 @@ class VertexAITokenCounter(BaseTokenCounter):
translated_count: Final = await self._count_translated_claude_request(
model_to_use=model_to_use,
messages=messages,
deployment=deployment,
deployment=resolved_deployment,
request_model=request_model,
tools=tools,
system=system,
@ -1513,7 +1521,7 @@ class VertexAITokenCounter(BaseTokenCounter):
partner_models_handler: Final = VertexAIPartnerModels()
# Extract vertex-specific params from litellm_params
partner_litellm_params: Final[dict[str, object]] = count_tokens_params_request
partner_litellm_params: Final = dict(decoded_json_object(raw_litellm_params))
vertex_project = partner_litellm_params.get("vertex_project") or partner_litellm_params.get(
"vertex_ai_project"
)
@ -1585,11 +1593,12 @@ class VertexAITokenCounter(BaseTokenCounter):
)
from litellm.llms.vertex_ai.count_tokens.handler import VertexAITokenCounter
gemini_litellm_params: Final = decoded_json_object(raw_litellm_params)
request_view: Final = await inference_request_view(
messages or [],
system,
messages_format,
count_context_edits(context_management, tools, count_tokens_params_request, model_to_use),
count_context_edits(context_management, tools, gemini_litellm_params, model_to_use),
)
if isinstance(request_view, InferenceRequestViewError):
return count_context_error_response(request_view, request_model, model_to_use)
@ -1600,7 +1609,7 @@ class VertexAITokenCounter(BaseTokenCounter):
return count_context_error_response(count_context, request_model, model_to_use)
blocks_error: Final = unconvertible_messages_error(
request_view.messages, messages_format
) or malformed_tool_result_text_count_error(messages, messages_format, deployment)
) or malformed_tool_result_text_count_error(messages, messages_format, resolved_deployment)
if blocks_error is not None:
return count_context_error_response(blocks_error, request_model, model_to_use)
@ -1608,22 +1617,21 @@ class VertexAITokenCounter(BaseTokenCounter):
contents
if contents is not None
else await gemini_count_contents(
request_view.messages, messages_format, model_to_use, "vertex_ai", count_tokens_params_request
request_view.messages, messages_format, model_to_use, "vertex_ai", gemini_litellm_params
)
)
if isinstance(resolved_contents, VertexGeminiCountContextError):
return count_context_error_response(resolved_contents, request_model, model_to_use)
count_tokens_params: Final = {
"model": model_to_use,
"contents": resolved_contents,
"system_instruction": count_context.system_instruction,
"tools": count_context.tools,
}
count_tokens_params_request.update(count_tokens_params)
try:
result = await VertexAITokenCounter().acount_tokens(
**count_tokens_params_request,
**_ForwardedCountTokensParams(
**{k: v for k, v in gemini_litellm_params.items() if k not in _COUNT_TOKENS_REQUEST_FIELDS}
),
model=model_to_use,
contents=resolved_contents,
system_instruction=count_context.system_instruction,
tools=count_context.tools,
)
except litellm.APIError as error:
return provider_error_response(error, request_model, model_to_use)

11
litellm/llms/vertex_ai/files/transformation.py
View file

@ -10,7 +10,7 @@ from contextlib import aclosing
from dataclasses import dataclass
from functools import partial
from types import MappingProxyType
from typing import IO, Any, Final, TypedDict
from typing import IO, Final, TypedDict
from urllib.parse import quote, unquote
import httpx
@ -827,7 +827,7 @@ def _openai_batch_jsonl_entry_to_vertex_anthropic_row(
def _openai_batch_jsonl_entry_to_vertex_rows(
openai_entry: dict[str, Any],
openai_entry: Mapping[str, object],
map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]],
batch_model: str | None = None,
deployment_params: Mapping[str, object] = MappingProxyType({}),
@ -842,7 +842,7 @@ def _openai_batch_jsonl_entry_to_vertex_rows(
Example Vertex jsonl
{"request":{"contents": [{"role": "user", "parts": [{"text": "What is the relation between the following video and image samples?"}, {"fileData": {"fileUri": "gs://cloud-samples-data/generative-ai/video/animals.mp4", "mimeType": "video/mp4"}}, {"fileData": {"fileUri": "gs://cloud-samples-data/generative-ai/image/cricket.jpeg", "mimeType": "image/jpeg"}}]}]}}
"""
entry_body: Final = openai_entry.get("body")
entry_body: Final = as_str_mapping(openai_entry.get("body"))
model: Final = batch_model or (str(entry_body.get("model") or "") if isinstance(entry_body, dict) else "")
is_anthropic_model: Final = is_vertex_anthropic_batch_model(model)
if _is_embeddings_batch_entry(openai_entry):
@ -855,11 +855,10 @@ def _openai_batch_jsonl_entry_to_vertex_rows(
if not isinstance(entry_body, dict):
raise TypeError("`body` on batch requests must be a JSON object, but was missing or not an object")
openai_request_body: Final = openai_entry["body"]
chat_request_body: Final = (
responses_batch_body_to_chat_body(openai_request_body, custom_llm_provider="vertex_ai")
responses_batch_body_to_chat_body(entry_body, custom_llm_provider="vertex_ai")
if _is_responses_batch_entry(openai_entry)
else openai_request_body
else entry_body
)
chat_messages: Final = map_developer_role_to_system_role(chat_request_body.get("messages", []))
if is_anthropic_model:

16
litellm/llms/vertex_ai/image_generation/image_generation_handler.py
View file

@ -7,6 +7,7 @@ from openai.types.image import Image
from pydantic import ConfigDict, TypeAdapter
import litellm
from litellm.litellm_core_utils.core_helpers import json_contains, subscript
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
@ -20,23 +21,24 @@ if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
_PREDICTIONS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True))
_DECODED_JSON: Final = TypeAdapter(object)
class VertexImageGeneration(VertexLLM):
def process_image_generation_response(
self,
json_response: Mapping[str, object],
json_response: object,
model_response: ImageResponse,
model: str | None = None,
) -> ImageResponse:
if "predictions" not in json_response:
if not json_contains(json_response, "predictions"):
raise litellm.InternalServerError(
message=f"image generation response does not contain 'predictions', got {json_response}",
llm_provider="vertex_ai",
model=model,
)
predictions: Final = _PREDICTIONS.validate_python(json_response["predictions"])
predictions: Final = _PREDICTIONS.validate_python(subscript(json_response, "predictions"))
response_data: Final[list[Image]] = []
for prediction in predictions:
@ -168,7 +170,7 @@ class VertexImageGeneration(VertexLLM):
if response.status_code != 200:
raise Exception(f"Error: {response.status_code} {response.text}")
json_response: Final[Mapping[str, object]] = response.json()
json_response: Final = _DECODED_JSON.validate_python(response.json())
return self.process_image_generation_response(json_response, model_response, model)
async def aimage_generation(
@ -272,11 +274,11 @@ class VertexImageGeneration(VertexLLM):
if response.status_code != 200:
raise Exception(f"Error: {response.status_code} {response.text}")
json_response: Final[Mapping[str, object]] = response.json()
json_response: Final = _DECODED_JSON.validate_python(response.json())
return self.process_image_generation_response(json_response, model_response, model)
def is_image_generation_response(self, json_response: dict[str, Any]) -> bool:
def is_image_generation_response(self, json_response: Mapping[str, object]) -> bool:
if "predictions" in json_response:
if "bytesBase64Encoded" in json_response["predictions"][0]:
if json_contains(subscript(json_response["predictions"], 0), "bytesBase64Encoded"):
return True
return False

7
litellm/llms/vertex_ai/realtime/transformation.py
View file

@ -20,7 +20,10 @@ import json
from collections.abc import Awaitable, Callable
from typing import Final
from pydantic import TypeAdapter
from litellm import verbose_logger
from litellm.litellm_core_utils.core_helpers import decoded_json_object
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig
from litellm.llms.vertex_ai.audio_transcription.realtime_transformation import (
@ -30,6 +33,8 @@ from litellm.llms.vertex_ai.audio_transcription.realtime_transformation import (
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
_DECODED_JSON: Final = TypeAdapter(object)
class VertexAIRealtimeConfig(GeminiRealtimeConfig):
"""
@ -217,7 +222,7 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
``setup`` closes the connection with a 1007 policy error.
"""
json_message: Final = json.loads(message)
msg_type: Final = json_message.get("type")
msg_type: Final = decoded_json_object(_DECODED_JSON.validate_python(json_message)).get("type")
if msg_type == "session.update":
if session_configuration_request is None:

3
litellm/llms/vertex_ai/vertex_ai_partner_models/llama/transformation.py
View file

@ -131,7 +131,8 @@ class VertexAILlama3Config(OpenAIGPTConfig):
## RESPONSE OBJECT
try:
completion_response: Final = OpenAIChatCompletionResponse(**raw_response.json())
payload: Final[OpenAIChatCompletionResponse] = raw_response.json()
completion_response: Final = OpenAIChatCompletionResponse(**payload)
except Exception as e:
response_headers: Final = getattr(raw_response, "headers", None)
raise VertexAIError(

14
litellm/llms/volcengine/embedding/transformation.py
View file

@ -8,7 +8,10 @@ from collections.abc import Mapping
from typing import Final
import httpx
from pydantic import TypeAdapter
from typing_extensions import TypedDict
from litellm.litellm_core_utils.core_helpers import decoded_json_object
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@ -18,6 +21,12 @@ from litellm.types.utils import EmbeddingResponse
from ..common_utils import get_volcengine_base_url, get_volcengine_headers
_DECODED_JSON: Final = TypeAdapter(object)
class _VolcengineEmbeddingFields(TypedDict, total=False, extra_items=object):
pass
class VolcEngineEmbeddingConfig(BaseEmbeddingConfig):
"""
@ -159,9 +168,10 @@ class VolcEngineEmbeddingConfig(BaseEmbeddingConfig):
) -> EmbeddingResponse:
"""Transform Volcengine response to EmbeddingResponse"""
try:
response_json: Final = raw_response.json()
decoded_response: Final = _DECODED_JSON.validate_python(raw_response.json())
except Exception as e:
raise ValueError(f"Failed to parse Volcengine response as JSON: {e}")
response_json: Final = decoded_json_object(decoded_response)
# Volcengine response format matches OpenAI format closely
# Just need to ensure all required fields are present
@ -177,7 +187,7 @@ class VolcEngineEmbeddingConfig(BaseEmbeddingConfig):
transformed_response["id"] = response_json["id"]
# Create EmbeddingResponse from transformed data
return EmbeddingResponse(**transformed_response)
return EmbeddingResponse(**_VolcengineEmbeddingFields(**transformed_response))
def validate_environment(
self,

2
litellm/llms/volcengine/videos/transformation.py
View file

@ -874,7 +874,7 @@ class VolcEngineVideoConfig(BaseVideoConfig):
**(extra_query or {}),
}
def transform_video_list_response( # pyright: ignore[reportIncompatibleMethodOverride] # base declares dict[str, str] but the payload is a heterogeneous list body
def transform_video_list_response(
self,
raw_response: httpx.Response,
logging_obj: "Logging",

17
litellm/llms/watsonx/common_utils.py
View file

@ -1,10 +1,12 @@
from typing import Final, cast
import httpx
from pydantic import TypeAdapter
import litellm
from litellm import verbose_logger
from litellm.caching import InMemoryCache
from litellm.litellm_core_utils.core_helpers import subscript
from litellm.litellm_core_utils.prompt_templates import factory as ptf
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.secret_managers.main import get_secret_str
@ -23,6 +25,13 @@ class WatsonXAIError(BaseLLMException):
iam_token_cache: Final = InMemoryCache()
_IAM_TOKEN_RESPONSE: Final = TypeAdapter(object)
def _ttl_with_buffer(expires_in: object) -> float:
if isinstance(expires_in, (int, float)):
return expires_in - 10
raise TypeError(f"unsupported operand type(s) for -: '{type(expires_in).__name__}' and 'int'")
def get_watsonx_iam_url():
@ -30,7 +39,7 @@ def get_watsonx_iam_url():
def generate_iam_token(api_key=None, **params) -> str:
result: str | None = iam_token_cache.get_cache(api_key)
result: object = iam_token_cache.get_cache(api_key)
if result is None:
headers: Final = {}
@ -58,13 +67,13 @@ def generate_iam_token(api_key=None, **params) -> str:
)
response: Final = litellm.module_level_client.post(url=iam_token_url, data=data, headers=headers)
response.raise_for_status()
json_data: Final = response.json()
json_data: Final = _IAM_TOKEN_RESPONSE.validate_python(response.json())
result = json_data["access_token"]
result = subscript(json_data, "access_token")
iam_token_cache.set_cache(
key=api_key,
value=result,
ttl=json_data["expires_in"] - 10, # leave some buffer
ttl=_ttl_with_buffer(subscript(json_data, "expires_in")),
)
return cast(str, result)

14
litellm/llms/watsonx/completion/transformation.py
View file

@ -4,7 +4,9 @@ from datetime import datetime
from typing import TYPE_CHECKING, Any, Final
import httpx
from pydantic import TypeAdapter
from litellm.litellm_core_utils.core_helpers import decoded_json_object
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.types.llms.openai import AllMessageValues, ChatCompletionUsageBlock
@ -30,6 +32,9 @@ else:
LiteLLMLoggingObj = Any
_DECODED_JSON: Final = TypeAdapter(object)
class IBMWatsonXAIConfig(IBMWatsonXMixin, BaseConfig):
"""
Reference: https://cloud.ibm.com/apidocs/watsonx-ai#text-generation
@ -305,12 +310,15 @@ class IBMWatsonXAIConfig(IBMWatsonXMixin, BaseConfig):
completion_tokens: Final = json_resp["results"][0]["generated_token_count"]
model_response.choices[0].message.content = generated_text
model_response.choices[0].finish_reason = map_finish_reason(json_resp["results"][0]["stop_reason"])
if json_resp.get("created_at"):
created_at: Final = decoded_json_object(_DECODED_JSON.validate_python(json_resp)).get("created_at")
if created_at:
if not isinstance(created_at, str):
raise TypeError("fromisoformat: argument must be str")
try:
created_datetime = datetime.fromisoformat(json_resp["created_at"])
created_datetime = datetime.fromisoformat(created_at)
except ValueError:
# datetime.fromisoformat cannot handle 'Z' in Python 3.10
created_datetime = datetime.fromisoformat(f"{json_resp['created_at'].rstrip('Z')}+00:00")
created_datetime = datetime.fromisoformat(f"{created_at.rstrip('Z')}+00:00")
model_response.created = int(created_datetime.timestamp())
else:
model_response.created = int(time.time())

54
litellm/llms/xai/chat/transformation.py
View file

@ -1,15 +1,17 @@
import copy
from collections.abc import AsyncIterator, Iterator, Mapping
from types import MappingProxyType
from typing import Any, Final
from typing import Final, Protocol, SupportsIndex, SupportsInt, runtime_checkable
import httpx
from pydantic import ConfigDict, TypeAdapter
from typing_extensions import Buffer
import litellm
from litellm._logging import verbose_logger
from litellm.constants import XAI_API_BASE
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none
from litellm.litellm_core_utils.prompt_templates.common_utils import (
filter_value_from_dict,
strip_name_from_messages,
@ -36,6 +38,28 @@ _RESPONSE_BODY: Final = TypeAdapter(dict[object, object], config=ConfigDict(hide
_FILTERED_TOOL: Final = TypeAdapter(object)
@runtime_checkable
class _SupportsTrunc(Protocol):
def __trunc__(self) -> int: ...
def _token_count(value: object) -> int:
if isinstance(value, (str, Buffer, SupportsInt, SupportsIndex, _SupportsTrunc)):
return int(value)
raise TypeError(
f"int() argument must be a string, a bytes-like object or a real number, not '{type(value).__name__}'"
)
def _reasoning_tokens(details: object) -> int:
if not details:
return 0
details_fields: Final = as_str_mapping(details)
if details_fields is not None and isinstance(details, dict):
return _token_count(details_fields.get("reasoning_tokens") or 0)
return _token_count(attribute_or_none(details, "reasoning_tokens") or 0)
def _usage_restated_from_xai_ticks(usage: Usage | None) -> Usage | None:
reported_cost: Final = xai_reported_cost_in_usd(getattr(usage, "cost_in_usd_ticks", None))
if usage is None or reported_cost is None:
@ -324,7 +348,7 @@ class XAIChatConfig(OpenAIGPTConfig):
@staticmethod
def fold_reasoning_tokens_into_completion(
target: ModelResponse | Usage | dict[str, Any] | None,
target: ModelResponse | Usage | dict[str, object] | None,
) -> None:
"""Reconcile xAI Usage to the OpenAI invariant.
@ -341,24 +365,20 @@ class XAIChatConfig(OpenAIGPTConfig):
return
if isinstance(target, ModelResponse):
usage: Usage | dict[str, Any] | None = getattr(target, "usage", None)
usage: Usage | dict[str, object] | None = getattr(target, "usage", None)
else:
usage = target
if usage is None:
return
if isinstance(usage, dict):
details = usage.get("completion_tokens_details") or {}
if isinstance(details, dict):
reasoning_tokens = int(details.get("reasoning_tokens") or 0)
else:
reasoning_tokens = int(getattr(details, "reasoning_tokens", 0) or 0)
reasoning_tokens = _reasoning_tokens(usage.get("completion_tokens_details"))
if reasoning_tokens <= 0:
return
prompt_tokens = int(usage.get("prompt_tokens") or 0)
completion_tokens = int(usage.get("completion_tokens") or 0)
total_tokens = int(usage.get("total_tokens") or 0)
prompt_tokens = _token_count(usage.get("prompt_tokens") or 0)
completion_tokens = _token_count(usage.get("completion_tokens") or 0)
total_tokens = _token_count(usage.get("total_tokens") or 0)
if total_tokens == prompt_tokens + completion_tokens:
return
@ -370,8 +390,8 @@ class XAIChatConfig(OpenAIGPTConfig):
usage["completion_tokens"] = completion_tokens + reasoning_tokens
return
details = getattr(usage, "completion_tokens_details", None)
reasoning_tokens = int(getattr(details, "reasoning_tokens", 0) or 0) if details else 0
details = attribute_or_none(usage, "completion_tokens_details")
reasoning_tokens = _token_count(attribute_or_none(details, "reasoning_tokens") or 0) if details else 0
if reasoning_tokens <= 0:
return
@ -407,15 +427,15 @@ class XAIChatConfig(OpenAIGPTConfig):
@staticmethod
def _normalize_openai_compatible_usage_totals(
usage: Usage | dict[str, Any] | None,
usage: Usage | dict[str, object] | None,
) -> None:
if usage is None:
return
if isinstance(usage, dict):
prompt_tokens = int(usage.get("prompt_tokens") or 0)
completion_tokens = int(usage.get("completion_tokens") or 0)
prompt_tokens = _token_count(usage.get("prompt_tokens") or 0)
completion_tokens = _token_count(usage.get("completion_tokens") or 0)
expected_total = prompt_tokens + completion_tokens
if int(usage.get("total_tokens") or 0) < expected_total:
if _token_count(usage.get("total_tokens") or 0) < expected_total:
usage["total_tokens"] = expected_total
return
prompt_tokens = int(usage.prompt_tokens or 0)

27
litellm/llms/you_com/search/transformation.py
View file

@ -8,7 +8,9 @@ OpenAPI spec: https://you.com/specs/openapi_search_v1.yaml
from typing import Final, TypedDict
import httpx
from pydantic import TypeAdapter
from litellm.litellm_core_utils.core_helpers import decoded_json_object, iter_items, subscript
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.search.transformation import (
@ -18,6 +20,8 @@ from litellm.llms.base_llm.search.transformation import (
)
from litellm.secret_managers.main import get_secret_str
_DECODED_JSON: Final = TypeAdapter(object)
class _YouComSearchRequestRequired(TypedDict):
"""Required fields for You.com Search API request."""
@ -169,23 +173,26 @@ class YouComSearchConfig(BaseSearchConfig):
- snippets[0] → SearchResult.snippet (falls back to `description`)
- page_age → SearchResult.date
"""
response_json: Final = raw_response.json()
raw_results: Final = response_json.get("results") or {}
response_json: Final = decoded_json_object(_DECODED_JSON.validate_python(raw_response.json()))
raw_results: Final = decoded_json_object(response_json.get("results") or {})
web_results: Final = raw_results.get("web") or []
news_results: Final = raw_results.get("news") or []
results: Final[list[SearchResult]] = []
for item in list(web_results) + list(news_results):
for entry in list(iter_items(web_results)) + list(iter_items(news_results)):
item = decoded_json_object(entry)
snippets = item.get("snippets") or []
snippet = snippets[0] if snippets else item.get("description", "")
snippet = subscript(snippets, 0) if snippets else item.get("description", "")
results.append(
SearchResult(
title=item.get("title", ""),
url=item.get("url", ""),
snippet=snippet,
date=item.get("page_age"),
last_updated=None,
SearchResult.model_validate(
{
"title": item.get("title", ""),
"url": item.get("url", ""),
"snippet": snippet,
"date": item.get("page_age"),
"last_updated": None,
}
)
)

18
litellm/main.py
View file

@ -3657,8 +3657,10 @@ def _complete_openrouter(ctx: CompletionDispatchContext) -> _CompletionDispatchR
or get_secret_str("OR_API_KEY")
)
openrouter_site_url: Final = get_secret("OR_SITE_URL") or "https://litellm.ai"
openrouter_app_name: Final = get_secret("OR_APP_NAME") or "liteLLM"
openrouter_site_url: Final = _OPAQUE_VALUE_ADAPTER.validate_python(
get_secret("OR_SITE_URL") or "https://litellm.ai"
)
openrouter_app_name: Final = _OPAQUE_VALUE_ADAPTER.validate_python(get_secret("OR_APP_NAME") or "liteLLM")
openrouter_headers: Final = {
"HTTP-Referer": openrouter_site_url,
@ -3762,8 +3764,10 @@ def _complete_vercel_ai_gateway(
api_key = api_key or litellm.api_key or get_secret("VERCEL_AI_GATEWAY_API_KEY")
vercel_site_url: Final = get_secret("VERCEL_SITE_URL") or "https://litellm.ai"
vercel_app_name: Final = get_secret("VERCEL_APP_NAME") or "liteLLM"
vercel_site_url: Final = _OPAQUE_VALUE_ADAPTER.validate_python(
get_secret("VERCEL_SITE_URL") or "https://litellm.ai"
)
vercel_app_name: Final = _OPAQUE_VALUE_ADAPTER.validate_python(get_secret("VERCEL_APP_NAME") or "liteLLM")
vercel_headers: Final = {
"http-referer": vercel_site_url,
@ -7349,8 +7353,10 @@ def embedding(
or get_secret_str("OR_API_KEY")
)
openrouter_site_url: Final = get_secret("OR_SITE_URL") or "https://litellm.ai"
openrouter_app_name: Final = get_secret("OR_APP_NAME") or "liteLLM"
openrouter_site_url: Final = _OPAQUE_VALUE_ADAPTER.validate_python(
get_secret("OR_SITE_URL") or "https://litellm.ai"
)
openrouter_app_name: Final = _OPAQUE_VALUE_ADAPTER.validate_python(get_secret("OR_APP_NAME") or "liteLLM")
openrouter_headers: Final = {
"HTTP-Referer": openrouter_site_url,

39
litellm/models/base.py
View file

@ -2,14 +2,34 @@
Base model class for domain models.
"""
from collections.abc import Iterable, Mapping
from datetime import datetime
from typing import Any
from typing import Final, Protocol
from pydantic import ConfigDict
from typing_extensions import TypedDict
from litellm.litellm_core_utils.core_helpers import as_str_mapping, has_attributes
from litellm.types.llms.base import LiteLLMBaseModel
class _RecordFields(TypedDict, total=False, extra_items=object):
pass
class _ModelDumpRecord(Protocol):
def model_dump(self) -> Mapping[str, object]: ...
class _DictRecord(Protocol):
def dict(self) -> Mapping[str, object]: ...
def _as_dict(record: object) -> Mapping[str, object] | None:
fields: Final = as_str_mapping(record)
return fields if isinstance(record, dict) else None
class DomainModel(LiteLLMBaseModel):
"""Base class for all domain models."""
@ -23,17 +43,18 @@ class DomainModel(LiteLLMBaseModel):
updated_at: datetime | None = None
@classmethod
def from_db_record(cls, record: Any) -> "DomainModel":
def from_db_record(cls, record: Mapping[str, object] | Iterable[tuple[str, object]] | None) -> "DomainModel":
"""Create a domain model from a database record."""
if record is None:
raise ValueError("Cannot create domain model from None record")
if isinstance(record, dict):
return cls(**record)
if hasattr(record, "model_dump") and callable(record.model_dump):
return cls(**record.model_dump())
if hasattr(record, "dict") and callable(record.dict):
return cls(**record.dict())
return cls(**dict(record))
fields: Final = _as_dict(record)
if fields is not None:
return cls(**_RecordFields(**fields))
if has_attributes(record, _ModelDumpRecord, "model_dump") and callable(record.model_dump):
return cls(**_RecordFields(**record.model_dump()))
if has_attributes(record, _DictRecord, "dict") and callable(record.dict):
return cls(**_RecordFields(**record.dict()))
return cls(**_RecordFields(**dict(record)))
def to_db_dict(self, exclude_unset: bool = False) -> dict[str, object]:
"""Convert domain model to a dictionary for database operations."""

4
litellm/proxy/_lazy_openapi_snapshot.py
View file

@ -13,7 +13,7 @@ the JSON, then run `npm run gen:api` in ui/litellm-dashboard and commit schema.d
import json
import re
import sys
from collections.abc import Callable, Mapping
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from types import MappingProxyType
@ -43,7 +43,7 @@ def _stabilize_multi_method_route_ids(routes) -> None:
"""FastAPI derives route IDs from a set of methods; make snapshots stable."""
for route in routes:
methods = sorted(getattr(route, "methods", None) or [])
methods: Sequence[str] = sorted(getattr(route, "methods", None) or [])
if len(methods) <= 1 or not getattr(route, "path_format", None):
continue

11
litellm/proxy/agent_endpoints/a2a_endpoints.py
View file

@ -23,7 +23,7 @@ from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.core_helpers import as_str_mapping, keyword_arguments
from litellm.litellm_core_utils.url_utils import SSRFError, validate_url
from litellm.llms.a2a.common_utils import resolve_a2a_hop_auth_header
from litellm.llms.azure.common_utils import caused_by_entra_rejection
@ -145,19 +145,20 @@ def _to_jsonrpc_object(chunk: object) -> object:
return chunk
def _build_message_send_params(params: dict[str, Any]) -> "MessageSendParams":
def _build_message_send_params(params: object) -> "MessageSendParams":
"""Build MessageSendParams from wire (0.3) or A2A 1.0 JSON-RPC params."""
from a2a.compat.v0_3.types import MessageSendParams
fields: Final = dict(**keyword_arguments(params, MessageSendParams))
try:
return MessageSendParams(**params)
return MessageSendParams.model_validate(fields)
except ValidationError:
from a2a.compat.v0_3.conversions import pb2_v10, to_compat_send_message_request
from google.protobuf.json_format import ParseDict, ParseError
pb: Final = pb2_v10.SendMessageRequest()
try:
ParseDict(params, pb, ignore_unknown_fields=True)
ParseDict(fields, pb, ignore_unknown_fields=True)
except ParseError as e:
raise ValueError(f"Invalid message/send params: {e}") from e
return to_compat_send_message_request(pb, "").params
@ -1027,7 +1028,7 @@ async def invoke_agent_a2a(
from a2a.compat.v0_3.types import SendMessageRequest
try:
message_send_params: Final = _build_message_send_params(params)
message_send_params: Final = _build_message_send_params(_DECODED_JSON.validate_python(params))
except (ValidationError, ValueError) as e:
return _jsonrpc_error(request_id, -32602, f"Invalid params: {e}")

8
litellm/proxy/alerting_endpoints/model_deprecation_alerts.py
View file

@ -68,7 +68,13 @@ async def update_deprecation_alert_state(
) -> DeprecationAlertState:
_require_admin(user, write=True)
state: Final = DeprecationAlertState(
**data.model_dump(), updated_at=datetime.now(timezone.utc), updated_by=user.user_id
model=data.model,
deprecation_date=data.deprecation_date,
status=data.status,
last_seen_date=data.last_seen_date,
last_seen_requests=data.last_seen_requests,
updated_at=datetime.now(timezone.utc),
updated_by=user.user_id,
)
await repository.set_param(_state_key(data.model, data.deprecation_date), state.model_dump(mode="json"))
return state

13
litellm/proxy/auth/handle_jwt.py
View file

@ -30,6 +30,7 @@ from typing_extensions import ReadOnly, TypedDict
from litellm._internal_context import with_service_target
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import json_contains, subscript
from litellm.litellm_core_utils.dot_notation_indexing import get_nested_value
from litellm.llms.custom_httpx.httpx_handler import HTTPHandler
from litellm.proxy._types import (
@ -164,6 +165,10 @@ class _UserInfoResponse(Protocol):
def json(self) -> dict[str, object]: ...
class _JWKSResponse(Protocol):
def json(self) -> object: ...
@dataclass(frozen=True, slots=True)
class JWTIdentity:
user_id: str | None
@ -218,6 +223,10 @@ def _userinfo_document(response: _UserInfoResponse) -> dict[str, object]:
return response.json()
def _jwks_document(response: _JWKSResponse) -> object:
return response.json()
def jwks_unavailable_exception(error: JWKSUnreachableError) -> ProxyException:
return ProxyException(
message=(
@ -960,12 +969,12 @@ class JWTHandler:
)
try:
response_json: Final = response.json()
response_json: Final = _jwks_document(response)
except Exception as e:
verbose_proxy_logger.error("Error parsing response: %s. Original Response: %s", e, response.text)
raise Exception(f"Error parsing response: {e}. Check server logs for original response.")
keys: Final = response_json["keys"] if "keys" in response_json else response_json
keys: Final = subscript(response_json, "keys") if json_contains(response_json, "keys") else response_json
return cast(JWKKeyValue, keys) # cast-ok: JWTKeyItem declares only `kid`, validating would drop key material
async def _get_public_key_from_jwks_url(self, jwks_url: str, kid: str | None) -> JWTKeyItem:

26
litellm/proxy/auth/litellm_license.py
View file

@ -3,14 +3,17 @@
import base64
import json
import os
from collections.abc import Mapping
from datetime import datetime
from typing import TYPE_CHECKING, Final
import httpx
from pydantic import TypeAdapter
from typing_extensions import TypedDict
from litellm._logging import verbose_proxy_logger
from litellm.constants import NON_LLM_CONNECTION_TIMEOUT
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.llms.custom_httpx.http_handler import HTTPHandler
if TYPE_CHECKING:
@ -23,6 +26,23 @@ AUTO_ROUTER_LICENSE_REMEDY: Final = "A LiteLLM license with the 'auto_router' fe
_LICENSE_VERDICT: Final = TypeAdapter(object)
class _LicenseFields(TypedDict, total=False, extra_items=object):
pass
def _license_fields(license_data: object) -> Mapping[str, object]:
fields: Final = as_str_mapping(license_data)
if fields is None:
raise TypeError(f"'{type(license_data).__name__}' object is not subscriptable")
return fields
def _expiration_text(expiration_date: object) -> str:
if isinstance(expiration_date, str):
return expiration_date
raise TypeError(f"strptime() argument 1 must be str, not {type(expiration_date).__name__}")
class LicenseCheck:
"""
- Check if license in env
@ -201,15 +221,15 @@ class LicenseCheck:
)
# Decode and parse the data
license_data: Final = json.loads(message.decode())
license_data: Final = _license_fields(_LICENSE_VERDICT.validate_python(json.loads(message.decode())))
# Check expiration date
expiration_date: Final = datetime.strptime(license_data["expiration_date"], "%Y-%m-%d")
expiration_date: Final = datetime.strptime(_expiration_text(license_data["expiration_date"]), "%Y-%m-%d")
if expiration_date < datetime.now():
self.airgapped_license_data = None
return False, "License has expired"
self.airgapped_license_data = EnterpriseLicenseData(**license_data)
self.airgapped_license_data = EnterpriseLicenseData(**_LicenseFields(**license_data))
return True

Some files were not shown because too many files have changed in this diff Show more