scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-08 22:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

Merge branch 'litellm_internal_staging' into litellm_guardrail_usage_cost_ui

Browse source
This commit is contained in:
ryan-crabbe-berri 2026-09-05 13:09:45 -07:00
commit 0be8bb98b0
419 changed files with 18094 additions and 2968 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.github/workflows/report-rust-release-wheel.yml vendored
View file

@ -24,8 +24,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
issues: write # PR comments use the issues API
pull-requests: read # Current-head validation rejects stale workflow runs
pull-requests: write
steps:
- name: Link release wheel report on PR

31
.github/workflows/test-terraform-modules.yml vendored
View file

@ -4,6 +4,7 @@ on:
push:
paths:
- "terraform/litellm/aws/**"
- "terraform/litellm/gcp/**"
- ".github/workflows/test-terraform-modules.yml"
pull_request:
branches:
@ -13,6 +14,7 @@ on:
- "litellm_**"
paths:
- "terraform/litellm/aws/**"
- "terraform/litellm/gcp/**"
- ".github/workflows/test-terraform-modules.yml"
permissions:
@ -52,3 +54,32 @@ jobs:
# Plan-only, mock_provider-backed: no AWS credentials, no API calls.
- name: test
run: terraform test
gcp-module:
name: fmt, validate, test (gcp)
runs-on: ubuntu-latest
timeout-minutes: 15
defaults:
run:
working-directory: terraform/litellm/gcp
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2
with:
terraform_version: 1.13.3
terraform_wrapper: false
- name: fmt
run: terraform fmt -recursive -check -diff
- name: init
run: terraform init -backend=false -input=false
- name: validate
run: terraform validate
- name: test
run: terraform test

24
basedpyright-code-budget.json
View file

@ -1,9 +1,9 @@
{
"reportAny": {
"limit": 14074
"limit": 13429
},
"reportArgumentType": {
"limit": 2206
"limit": 2198
},
"reportAssignmentType": {
"limit": 319
@ -24,7 +24,7 @@
"limit": 19
},
"reportExplicitAny": {
"limit": 4124
"limit": 3369
},
"reportFunctionMemberAccess": {
"limit": 7
@ -48,16 +48,16 @@
"limit": 30
},
"reportInvalidTypeVarUse": {
"limit": 2
"limit": 1
},
"reportMatchNotExhaustive": {
"limit": 0
},
"reportMissingParameterType": {
"limit": 5601
"limit": 5570
},
"reportMissingTypeArgument": {
"limit": 15284
"limit": 15281
},
"reportMissingTypeStubs": {
"limit": 40
@ -90,7 +90,7 @@
"limit": 8
},
"reportReturnType": {
"limit": 181
"limit": 180
},
"reportTypedDictNotRequiredAccess": {
"limit": 22
@ -105,16 +105,16 @@
"limit": 109
},
"reportUnknownMemberType": {
"limit": 38309
"limit": 38283
},
"reportUnknownParameterType": {
"limit": 19621
"limit": 19584
},
"reportUnknownVariableType": {
"limit": 29844
"limit": 29829
},
"reportUnnecessaryCast": {
"limit": 111
"limit": 110
},
"reportUnnecessaryComparison": {
"limit": 687
@ -123,7 +123,7 @@
"limit": 4
},
"reportUnnecessaryIsInstance": {
"limit": 819
"limit": 816
},
"reportUntypedBaseClass": {
"limit": 0

27
enterprise/litellm_enterprise/enterprise_callbacks/secret_detection.py
View file

@ -433,9 +433,9 @@ _default_detect_secrets_config = {
"name": "ZendeskSecretKeyDetector",
"path": _custom_plugins_path + "/zendesk_secret_key.py",
},
{"name": "Base64HighEntropyString", "limit": 3.0},
{"name": "Base64HighEntropyString", "limit": 4.5},
{"name": "HexHighEntropyString", "limit": 3.0},
]
],
}
@ -466,16 +466,19 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
os.remove(temp_file.name)
detected_secrets = []
for file in secrets.files:
for found_secret in secrets[file]:
if found_secret.secret_value is None:
continue
detected_secrets.append(
{"type": found_secret.type, "value": found_secret.secret_value}
)
return detected_secrets
return [
{"type": found_secret.type, "value": found_secret.secret_value}
for file in sorted(secrets.files)
for found_secret in sorted(
secrets[file],
key=lambda secret: (
-len(secret.secret_value or ""),
secret.type,
secret.secret_value or "",
),
)
if found_secret.secret_value is not None
]
def redact_text(self, text: str, source: str = "message") -> str:
"""Replace every detected secret in ``text`` with ``[REDACTED]`` and

15
enterprise/litellm_enterprise/enterprise_callbacks/secrets_plugins/openai_api_key.py
View file

@ -3,6 +3,7 @@ This plugin searches for OpenAI API Keys.
"""
import re
from collections.abc import Generator
from detect_secrets.plugins.base import RegexBasedDetector
@ -16,4 +17,16 @@ class OpenAIApiKeyDetector(RegexBasedDetector):
@property
def denylist(self) -> list[re.Pattern]:
return [re.compile(r"""(sk-[a-zA-Z0-9]{5,})""")]
return [
re.compile(
r"((?:(?<![a-zA-Z0-9])|(?<=%[0-9A-Fa-f]{2}))"
r"sk[-_]"
r"[a-zA-Z0-9_-]{5,}"
r"(?![a-zA-Z0-9_-]))"
)
]
def analyze_string(self, string: str) -> Generator[str, None, None]:
# the digit check lives outside the regex: a lookahead re-scans the token
# from every `sk` inside it, which is quadratic on `-sk-sk-sk-...` input
yield from (match for match in super().analyze_string(string) if re.search(r"[0-9]", match))

8
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -46,6 +46,7 @@ from litellm.proxy._types import (
UserAPIKeyAuth,
)
from litellm.proxy.openai_files_endpoints.common_utils import (
BATCH_CREATE_HIDDEN_PARAM,
FILE_LIST_CONTINUATION_CHUNK_SIZE,
MAX_FILE_LIST_LIMIT,
_is_base64_encoded_unified_file_id,
@ -1321,7 +1322,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
## Check if unified_file_id is in the response
unified_file_id = response._hidden_params.get("unified_file_id") # managed file id
unified_batch_id = response._hidden_params.get("unified_batch_id") # managed batch id
is_batch_create: Final = unified_file_id is not None
is_batch_create: Final = response._hidden_params.get(BATCH_CREATE_HIDDEN_PARAM) is True
model_id = cast(Optional[str], response._hidden_params.get("model_id"))
model_name = cast(Optional[str], response._hidden_params.get("model_name"))
@ -1410,10 +1411,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
)
# Only record batch creation metric on actual create (not retrieve/cancel).
# unified_file_id in _hidden_params is only set by the create_batch endpoint.
original_unified_file_id = response._hidden_params.get("unified_file_id")
if original_unified_file_id:
if is_batch_create:
prom_logger = self._get_prometheus_logger()
if prom_logger:
batch_provider = ""

12
enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py
View file

@ -16,6 +16,7 @@ from litellm.llms.base_llm.managed_resources.utils import (
is_base64_encoded_unified_id,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.utils import LLMResponseTypes
from litellm.types.vector_stores import (
VectorStoreCreateOptionalRequestParams,
VectorStoreCreateResponse,
@ -24,6 +25,7 @@ from litellm.types.vector_stores import (
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from litellm.caching.caching import DualCache
from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
from litellm.proxy.utils import PrismaClient as _PrismaClient
@ -156,7 +158,7 @@ class _PROXY_LiteLLMManagedVectorStores(
# Create vector store for each model
# Convert TypedDict to Dict[str, Any] for base class compatibility
request_data_dict: Dict[str, Any] = dict(create_request)
request_data_dict: Dict[str, object] = dict(create_request)
responses = await self.create_resource_for_each_model(
llm_router=llm_router,
request_data=request_data_dict,
@ -209,7 +211,7 @@ class _PROXY_LiteLLMManagedVectorStores(
limit: Optional[int] = None,
after: Optional[str] = None,
order: Optional[str] = None,
) -> Dict[str, Any]:
) -> Dict[str, object]:
"""
List vector stores created by a user.
@ -301,7 +303,7 @@ class _PROXY_LiteLLMManagedVectorStores(
async def async_pre_call_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
cache: Any,
cache: "DualCache",
data: Dict,
call_type: str,
) -> Union[Exception, str, Dict, None]:
@ -403,8 +405,8 @@ class _PROXY_LiteLLMManagedVectorStores(
self,
data: Dict,
user_api_key_dict: UserAPIKeyAuth,
response: Any,
) -> Any:
response: LLMResponseTypes,
) -> LLMResponseTypes:
"""
Post-call hook to transform responses.

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.64"
version = "0.1.65"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.64"
version = "0.1.65"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

12
helm/litellm/templates/migrations-job.yaml
View file

@ -77,4 +77,16 @@ spec:
volumes:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.migrationJob.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.migrationJob.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.migrationJob.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}

68
helm/litellm/tests/migration_job_tests.yaml
View file

@ -1,4 +1,4 @@
suite: test migrations Job ServiceAccount resolution and pod hardening
suite: test migrations Job ServiceAccount resolution, pod hardening, and scheduling
templates:
- migrations-job.yaml
values:
@ -188,3 +188,69 @@ tests:
asserts:
- notExists:
path: spec.activeDeadlineSeconds
- it: renders no scheduling fields by default
asserts:
- isNull:
path: spec.template.spec.nodeSelector
- isNull:
path: spec.template.spec.tolerations
- isNull:
path: spec.template.spec.affinity
- it: renders nodeSelector, tolerations, and affinity from the migrationJob values
set:
migrationJob.nodeSelector:
intent: no-csi-nodes
migrationJob.tolerations:
- key: intent
operator: Equal
value: no-csi-nodes
effect: NoSchedule
migrationJob.affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: intent
operator: In
values:
- no-csi-nodes
asserts:
- equal:
path: spec.template.spec.nodeSelector
value:
intent: no-csi-nodes
- equal:
path: spec.template.spec.tolerations
value:
- key: intent
operator: Equal
value: no-csi-nodes
effect: NoSchedule
- equal:
path: spec.template.spec.affinity
value:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: intent
operator: In
values:
- no-csi-nodes
- it: does not inherit the gateway's scheduling values
set:
gateway.nodeSelector:
intent: no-csi-nodes
gateway.tolerations:
- key: intent
operator: Equal
value: no-csi-nodes
effect: NoSchedule
asserts:
- isNull:
path: spec.template.spec.nodeSelector
- isNull:
path: spec.template.spec.tolerations

7
helm/litellm/values.yaml
View file

@ -152,6 +152,13 @@ migrationJob:
# the writable scratch space a read-only root filesystem needs.
volumes: []
volumeMounts: []
# Scheduling for the Job pod, same shape as gateway.nodeSelector /
# gateway.tolerations / gateway.affinity. The Job does not inherit the other
# components' scheduling values: a migration usually needs a larger node
# than the gateway, so pin it here explicitly.
nodeSelector: {}
tolerations: []
affinity: {}
image:
repository: ghcr.io/berriai/litellm-migrations
tag: "" # defaults to .Chart.AppVersion

1
litellm-proxy-extras/litellm_proxy_extras/migrations/20260904000000_shadow_eval_model_scope/migration.sql Normal file
View file

@ -0,0 +1 @@
ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN IF NOT EXISTS "models" TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[];

1
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -1536,6 +1536,7 @@ model LiteLLM_ShadowEvalJob {
target_id String // hashed virtual key, team_id, or user_id whose traffic this leg shadows
router_name String // first (often only) auto-router under evaluation; router_names is the full set
router_names String[] @default([]) // all routers this job runs as shadow arms; empty on legacy rows, whose set is (router_name)
models String[] @default([]) // model groups the sampled traffic is narrowed to; empty samples every model
direction String @default("forward") // forward | reverse
baseline_model String? // reverse only: the fixed model the router is judged against
judge_model String

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.93"
version = "0.4.94"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.93"
version = "0.4.94"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

410
litellm/__init__.py
View file

@ -13,6 +13,7 @@ warnings.filterwarnings("ignore", message=".*`ReadOnly` qualifier.*")
### INIT VARIABLES #########################
import threading
import os
import sys
# Load .env before any other litellm imports so env vars (e.g. LITELLM_UI_SESSION_DURATION) are available
import dotenv as _dotenv
@ -45,8 +46,6 @@ from typing import (
TYPE_CHECKING,
Union,
)
from litellm.types.integrations.datadog import DatadogInitParams
from litellm.types.integrations.newrelic import NewRelicInitParams
from litellm._logging import (
set_verbose,
_turn_on_debug,
@ -95,8 +94,7 @@ from litellm.constants import (
DEFAULT_SOFT_BUDGET,
DEFAULT_ALLOWED_FAILS,
)
import httpx
# httpx is lazy-loaded via __getattr__
# register_async_client_cleanup is lazy-loaded and called on first access
litellm_mode = os.getenv("LITELLM_MODE", "DEV") # "PRODUCTION", "DEV"
@ -364,8 +362,6 @@ guardrail_name_config_map: Dict[str, GuardrailItem] = {}
include_cost_in_streaming_usage: bool = False
reasoning_auto_summary: bool = False
### PROMPTS ####
from litellm.types.prompts.init_prompts import PromptSpec
prompt_name_config_map: Dict[str, PromptSpec] = {}
##################
@ -1271,206 +1267,203 @@ openai_video_generation_models = ["sora-2"]
# get_llm_provider is lazy-loaded via __getattr__
# remove_index_from_tool_calls is lazy-loaded via __getattr__
# Import KeyManagementSettings here (before utils import) because _key_management_settings
# is accessed during import time in secret_managers/main.py (via dd_tracing -> datadog -> _service_logger -> utils)
from litellm.types.secret_managers.main import KeyManagementSettings
# SDK symbols previously imported eagerly here are lazy-loaded via __getattr__
# (_SDK_SYMBOLS_IMPORT_MAP in _lazy_imports_registry.py); mirrored under TYPE_CHECKING
# so static type checkers still see them
if TYPE_CHECKING:
_key_management_settings: KeyManagementSettings
_key_management_settings: KeyManagementSettings = KeyManagementSettings()
from .utils import client
# client must be imported immediately as it's used as a decorator at function definition time
from .utils import client
from .llms.custom_llm import CustomLLM
from .llms.anthropic.common_utils import AnthropicModelInfo
from .llms.ai21.chat.transformation import AI21ChatConfig, AI21ChatConfig as AI21Config
from .llms.deprecated_providers.palm import (
PalmConfig,
) # here to prevent breaking changes
from .llms.deprecated_providers.aleph_alpha import AlephAlphaConfig
from .llms.gemini.common_utils import GeminiModelInfo
# Note: Most other utils imports are lazy-loaded via __getattr__ to avoid loading utils.py
# (which imports tiktoken) at import time
from .llms.vertex_ai.vertex_embeddings.transformation import (
VertexAITextEmbeddingConfig,
)
from .llms.custom_llm import CustomLLM
from .llms.anthropic.common_utils import AnthropicModelInfo
from .llms.ai21.chat.transformation import AI21ChatConfig, AI21ChatConfig as AI21Config
from .llms.deprecated_providers.palm import (
PalmConfig,
) # here to prevent breaking changes
from .llms.deprecated_providers.aleph_alpha import AlephAlphaConfig
from .llms.gemini.common_utils import GeminiModelInfo
vertexAITextEmbeddingConfig = VertexAITextEmbeddingConfig()
from .llms.bedrock.embed.amazon_titan_v2_transformation import (
AmazonTitanV2Config,
)
from .llms.topaz.common_utils import TopazModelInfo
from .llms.vertex_ai.vertex_embeddings.transformation import (
VertexAITextEmbeddingConfig,
)
# OpenAIOSeriesConfig is lazy loaded - openaiOSeriesConfig will be created on first access
# OpenAIGPTConfig, OpenAIGPT5Config, etc. are lazy loaded - instances will be created on first access
from .llms.xai.common_utils import XAIModelInfo
vertexAITextEmbeddingConfig = VertexAITextEmbeddingConfig()
# PublicAI now uses JSON-based configuration (see litellm/llms/openai_like/providers.json)
# All remaining configs are now lazy loaded - see _lazy_imports_registry.py
# Import LlmProviders here (before main import) because it's imported during import time
# in multiple places including openai.py (via main import)
from .llms.bedrock.embed.amazon_titan_v2_transformation import (
AmazonTitanV2Config,
)
from .llms.topaz.common_utils import TopazModelInfo
## Lazy loading this is not straightforward, will leave it here for now.
from .main import *
from .compression import compress
# OpenAIOSeriesConfig is lazy loaded - openaiOSeriesConfig will be created on first access
# OpenAIGPTConfig, OpenAIGPT5Config, etc. are lazy loaded - instances will be created on first access
from .llms.xai.common_utils import XAIModelInfo
# Skills API
from .skills.main import (
create_skill,
acreate_skill,
list_skills,
alist_skills,
get_skill,
aget_skill,
delete_skill,
adelete_skill,
)
from .evals.main import (
create_eval,
acreate_eval,
list_evals,
alist_evals,
get_eval,
aget_eval,
delete_eval,
adelete_eval,
cancel_eval,
acancel_eval,
create_run,
acreate_run,
list_runs,
alist_runs,
get_run,
aget_run,
delete_run,
adelete_run,
cancel_run,
acancel_run,
)
from .integrations import *
from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients
from .exceptions import (
AuthenticationError,
InvalidRequestError,
BadRequestError,
ImageFetchError,
NotFoundError,
PermissionDeniedError,
RateLimitError,
RateLimitErrorCategory,
RateLimitType,
ServiceUnavailableError,
BadGatewayError,
OpenAIError,
ContextWindowExceededError,
ContentPolicyViolationError,
BudgetExceededError,
APIError,
Timeout,
APIConnectionError,
UnsupportedParamsError,
APIResponseValidationError,
UnprocessableEntityError,
InternalServerError,
JSONSchemaValidationError,
LITELLM_EXCEPTION_TYPES,
MockException,
)
from .budget_manager import BudgetManager
from .proxy.proxy_cli import run_server
from .router import Router
from .assistants.main import *
from .batches.main import *
from .images.main import *
from .videos.main import *
from .batch_completion.main import *
from .rerank_api.main import *
from .llms.anthropic.experimental_pass_through.messages.handler import *
from .responses.main import *
# PublicAI now uses JSON-based configuration (see litellm/llms/openai_like/providers.json)
# All remaining configs are now lazy loaded - see _lazy_imports_registry.py
# Interactions API is available as litellm.interactions module
# Usage: litellm.interactions.create(), litellm.interactions.get(), etc.
from . import interactions
from .interactions.agents.main import (
acreate as acreate_agent,
create as create_agent,
alist as alist_agents,
list as list_agents,
aget as aget_agent,
get as get_agent,
adelete as adelete_agent,
delete as delete_agent,
alist_versions as alist_agent_versions,
list_versions as list_agent_versions,
)
from .skills.main import (
create_skill,
acreate_skill,
list_skills,
alist_skills,
get_skill,
aget_skill,
delete_skill,
adelete_skill,
)
from .containers.main import *
from .ocr.main import *
from .rust_bridge import rust
from .rag.main import *
from .sandbox.main import *
from .search.main import *
from .realtime_api.main import (
_arealtime,
acreate_realtime_client_secret,
acreate_realtime_transcription_session,
arealtime_calls,
)
from .responses.main import _aresponses_websocket
from .fine_tuning.main import *
from .files.main import *
from .vector_store_files.main import (
acreate as avector_store_file_create,
adelete as avector_store_file_delete,
alist as avector_store_file_list,
aretrieve as avector_store_file_retrieve,
aretrieve_content as avector_store_file_content,
aupdate as avector_store_file_update,
create as vector_store_file_create,
delete as vector_store_file_delete,
list as vector_store_file_list,
retrieve as vector_store_file_retrieve,
retrieve_content as vector_store_file_content,
update as vector_store_file_update,
)
from .scheduler import *
# Import LlmProviders here (before main import) because it's imported during import time
# in multiple places including openai.py (via main import)
from litellm.types.utils import LlmProviders
### ADAPTERS ###
import litellm.anthropic_interface as anthropic
## Lazy loading this is not straightforward, will leave it here for now.
from .main import *
from .compression import compress
### Vector Store Registry ###
# Skills API
from .skills.main import (
create_skill,
acreate_skill,
list_skills,
alist_skills,
get_skill,
aget_skill,
delete_skill,
adelete_skill,
)
from .evals.main import (
create_eval,
acreate_eval,
list_evals,
alist_evals,
get_eval,
aget_eval,
delete_eval,
adelete_eval,
cancel_eval,
acancel_eval,
create_run,
acreate_run,
list_runs,
alist_runs,
get_run,
aget_run,
delete_run,
adelete_run,
cancel_run,
acancel_run,
)
from .integrations import *
from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients
from .exceptions import (
AuthenticationError,
InvalidRequestError,
BadRequestError,
ImageFetchError,
NotFoundError,
PermissionDeniedError,
RateLimitError,
RateLimitErrorCategory,
RateLimitType,
ServiceUnavailableError,
BadGatewayError,
OpenAIError,
ContextWindowExceededError,
ContentPolicyViolationError,
BudgetExceededError,
APIError,
Timeout,
APIConnectionError,
UnsupportedParamsError,
APIResponseValidationError,
UnprocessableEntityError,
InternalServerError,
JSONSchemaValidationError,
LITELLM_EXCEPTION_TYPES,
MockException,
)
from .budget_manager import BudgetManager
from .proxy.proxy_cli import run_server
from .router import Router
from .assistants.main import *
from .batches.main import *
from .images.main import *
from .videos.main import *
from .batch_completion.main import *
from .rerank_api.main import *
from .llms.anthropic.experimental_pass_through.messages.handler import *
from .responses.main import *
### RAG ###
from . import rag
# Interactions API is available as litellm.interactions module
# Usage: litellm.interactions.create(), litellm.interactions.get(), etc.
from . import interactions
from .interactions.agents.main import (
acreate as acreate_agent,
create as create_agent,
alist as alist_agents,
list as list_agents,
aget as aget_agent,
get as get_agent,
adelete as adelete_agent,
delete as delete_agent,
alist_versions as alist_agent_versions,
list_versions as list_agent_versions,
)
from .skills.main import (
create_skill,
acreate_skill,
list_skills,
alist_skills,
get_skill,
aget_skill,
delete_skill,
adelete_skill,
)
from .containers.main import *
from .ocr.main import *
from .rust_bridge import rust
from .rag.main import *
from .sandbox.main import *
from .search.main import *
from .realtime_api.main import (
_arealtime,
acreate_realtime_client_secret,
acreate_realtime_transcription_session,
arealtime_calls,
)
from .responses.main import _aresponses_websocket
from .fine_tuning.main import *
from .files.main import *
from .vector_store_files.main import (
acreate as avector_store_file_create,
adelete as avector_store_file_delete,
alist as avector_store_file_list,
aretrieve as avector_store_file_retrieve,
aretrieve_content as avector_store_file_content,
aupdate as avector_store_file_update,
create as vector_store_file_create,
delete as vector_store_file_delete,
list as vector_store_file_list,
retrieve as vector_store_file_retrieve,
retrieve_content as vector_store_file_content,
update as vector_store_file_update,
)
from .scheduler import *
### CUSTOM LLMs ###
### CLI UTILITIES ###
from litellm.litellm_core_utils.cli_token_utils import get_litellm_gateway_api_key
### PASSTHROUGH ###
from .passthrough import allm_passthrough_route, llm_passthrough_route
from .google_genai import agenerate_content
### ADAPTERS ###
from .types.adapter import AdapterItem
import litellm.anthropic_interface as anthropic
adapters: List[AdapterItem] = []
### Vector Store Registry ###
from .vector_stores.vector_store_registry import (
VectorStoreRegistry,
VectorStoreIndexRegistry,
)
vector_store_registry: Optional[VectorStoreRegistry] = None
vector_store_index_registry: Optional[VectorStoreIndexRegistry] = None
### RAG ###
from . import rag
### CUSTOM LLMs ###
from .types.llms.custom_llm import CustomLLMItem
custom_provider_map: List[CustomLLMItem] = []
_custom_providers: List[str] = [] # internal helper util, used to track names of custom providers
disable_hf_tokenizer_download: Optional[bool] = (
@ -1478,13 +1471,6 @@ disable_hf_tokenizer_download: Optional[bool] = (
)
global_disable_no_log_param: bool = False
### CLI UTILITIES ###
from litellm.litellm_core_utils.cli_token_utils import get_litellm_gateway_api_key
### PASSTHROUGH ###
from .passthrough import allm_passthrough_route, llm_passthrough_route
from .google_genai import agenerate_content
### GLOBAL CONFIG ###
global_bitbucket_config: Optional[Dict[str, Any]] = None
@ -1508,10 +1494,21 @@ def set_global_gitlab_config(config: Dict[str, Any]) -> None:
# Lazy loading system for heavy modules to reduce initial import time and memory usage
if TYPE_CHECKING:
import httpx
from litellm.types.utils import ModelInfo as _ModelInfoType
from litellm.types.utils import PriorityReservationSettings
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.caching.caching import Cache
from litellm.types.adapter import AdapterItem
from litellm.types.integrations.datadog import DatadogInitParams
from litellm.types.integrations.newrelic import NewRelicInitParams
from litellm.types.llms.custom_llm import CustomLLMItem
from litellm.types.prompts.init_prompts import PromptSpec
from litellm.vector_stores.vector_store_registry import (
VectorStoreIndexRegistry,
VectorStoreRegistry,
)
# Type stubs for lazy-loaded configs to help mypy
from .llms.bedrock.chat.converse_transformation import (
@ -2187,16 +2184,6 @@ if TYPE_CHECKING:
# Track if async client cleanup has been registered (for lazy loading)
_async_client_cleanup_registered = False
# Eager loading for backwards compatibility with VCR and other HTTP recording tools
# When LITELLM_DISABLE_LAZY_LOADING is set, lazy-loaded attributes are loaded at import time
# For now, this only affects encoding (tiktoken) as it was the only reported issue
# See: https://github.com/BerriAI/litellm/issues/18659
# This ensures encoding is initialized before VCR starts recording HTTP requests
if os.getenv("LITELLM_DISABLE_LAZY_LOADING", "").lower() in ("1", "true", "yes", "on"):
# Load encoding at import time (pre-#18070 behavior)
# This ensures encoding is initialized before VCR starts recording
from .main import encoding
def __getattr__(name: str) -> Any:
"""Lazy import handler with cached registry for improved performance."""
@ -2276,6 +2263,8 @@ def __getattr__(name: str) -> Any:
"openAIGPT5Config": "OpenAIGPT5Config",
"nvidiaNimConfig": "NvidiaNimConfig",
"nvidiaNimEmbeddingConfig": "NvidiaNimEmbeddingConfig",
"vertexAITextEmbeddingConfig": "VertexAITextEmbeddingConfig",
"_key_management_settings": "KeyManagementSettings",
}
if name in _config_instances:
from ._lazy_imports import get_litellm_globals
@ -2393,7 +2382,30 @@ def __getattr__(name: str) -> Any:
return locals()[name]
from ._lazy_imports import lazy_import_litellm_submodule
submodule: Final = lazy_import_litellm_submodule(name)
if submodule is not None:
return submodule
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
from ._lazy_imports import LiteLLMModule
from ._lazy_imports_registry import STAR_IMPORT_PUBLIC_NAMES
sys.modules[__name__].__class__ = LiteLLMModule
__all__ = list(STAR_IMPORT_PUBLIC_NAMES) # mutable-ok: star imports require __all__ to be a list of str
# ALL_LITELLM_RESPONSE_TYPES is lazy-loaded via __getattr__ to avoid loading utils at import time
# Eager loading for backwards compatibility with VCR and other HTTP recording tools
# When LITELLM_DISABLE_LAZY_LOADING is set, lazy-loaded attributes are loaded at import time
# For now, this only affects encoding (tiktoken) as it was the only reported issue
# See: https://github.com/BerriAI/litellm/issues/18659
# This ensures encoding is initialized before VCR starts recording HTTP requests
# This block stays at the bottom so __getattr__ can resolve attributes main.py needs during its import
if os.getenv("LITELLM_DISABLE_LAZY_LOADING", "").lower() in ("1", "true", "yes", "on"):
from .main import encoding

96
litellm/_lazy_imports.py
View file

@ -16,9 +16,10 @@ until they're actually needed.
"""
import importlib
import importlib.util
import sys
from collections.abc import Callable, Mapping
from types import ModuleType
from types import MappingProxyType, ModuleType
from typing import TYPE_CHECKING, Any, Final, cast
from typing_extensions import ReadOnly, TypedDict
@ -34,6 +35,8 @@ from ._lazy_imports_registry import (
_LITELLM_LOGGING_IMPORT_MAP,
_LLM_CONFIGS_IMPORT_MAP,
_LLM_PROVIDER_LOGIC_IMPORT_MAP,
_SDK_MODULE_ALIASES,
_SDK_SYMBOLS_IMPORT_MAP,
_TOKEN_COUNTER_IMPORT_MAP,
_TYPES_IMPORT_MAP,
_TYPES_UTILS_IMPORT_MAP,
@ -78,7 +81,10 @@ def _get_utils_globals() -> dict[str, object]:
This is where we cache imported attributes so we don't import them twice.
When you do `litellm.utils.some_function`, it gets stored in this dictionary.
"""
return sys.modules["litellm.utils"].__dict__
cached: Final = sys.modules.get("litellm.utils")
if cached is not None:
return cached.__dict__
return importlib.import_module("litellm.utils").__dict__
def _get_module_level_client_timeout(litellm_globals: Mapping[str, Any]) -> "float | httpx.Timeout | None":
@ -214,6 +220,10 @@ def _get_lazy_import_registry() -> dict[str, Callable[[str], object]]:
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_llm_provider_logic
for name in UTILS_MODULE_NAMES:
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_utils_module
for name in _SDK_SYMBOLS_IMPORT_MAP:
_LAZY_IMPORT_REGISTRY.setdefault(name, _lazy_import_sdk_symbols)
for name in _SDK_MODULE_ALIASES:
_LAZY_IMPORT_REGISTRY.setdefault(name, _lazy_import_sdk_module_alias)
return _LAZY_IMPORT_REGISTRY
@ -229,7 +239,7 @@ def _module_attribute(module: ModuleType, attr_name: str) -> object:
return attribute["value"]
def _generic_lazy_import(name: str, import_map: dict[str, tuple[str, str]], category: str) -> object:
def _generic_lazy_import(name: str, import_map: Mapping[str, tuple[str, str]], category: str) -> object:
"""
Generic function that handles lazy importing for most attributes.
@ -350,6 +360,86 @@ def _lazy_import_llm_provider_logic(name: str) -> object:
return _generic_lazy_import(name, _LLM_PROVIDER_LOGIC_IMPORT_MAP, "LLM provider logic")
def _lazy_import_sdk_symbols(name: str) -> object:
"""Handler for SDK symbols previously imported eagerly at the bottom of litellm/__init__.py"""
return _generic_lazy_import(name, _SDK_SYMBOLS_IMPORT_MAP, "SDK symbols")
def _lazy_import_sdk_module_alias(name: str) -> object:
"""Handler for litellm attributes that bind a module (e.g. litellm.anthropic)"""
_globals: Final = get_litellm_globals()
if name in _globals:
return _globals[name]
module: Final = importlib.import_module(_SDK_MODULE_ALIASES[name])
_globals[name] = module # rebind-ok: caches the resolved module alias on the package
return module
_SHADOWABLE_SDK_FUNCTIONS: Final = MappingProxyType(
{
"batch_completion": ("litellm.batch_completion.main", "batch_completion"),
"ocr": ("litellm.ocr.main", "ocr"),
"responses": ("litellm.responses.main", "responses"),
"search": ("litellm.search.main", "search"),
}
)
def _shadowable_function_property(name: str) -> property:
"""Property keeping litellm.<name> bound to the SDK function even after the import
machinery binds the identically named litellm.<name> subpackage onto the litellm module."""
module_path, attr_name = _SHADOWABLE_SDK_FUNCTIONS[name]
def _get(module: ModuleType) -> object:
stored: Final = module.__dict__.get(name)
if stored is not None and not (isinstance(stored, ModuleType) and stored.__name__ == f"litellm.{name}"):
return stored
value: Final = _module_attribute(importlib.import_module(module_path), attr_name)
module.__dict__[name] = value # rebind-ok: caches the resolved function on the litellm module
return value
def _set(module: ModuleType, value: object) -> None:
module.__dict__[name] = value # rebind-ok: property setter must store assignments on the module
return property(_get, _set)
class LiteLLMModule(ModuleType):
"""Module type installed on the litellm package so function names shadowed by
same-named subpackages (litellm.responses, ...) keep resolving to the functions."""
batch_completion = _shadowable_function_property("batch_completion")
ocr = _shadowable_function_property("ocr")
responses = _shadowable_function_property("responses")
search = _shadowable_function_property("search")
def lazy_import_submodule(package: str, name: str) -> "ModuleType | None":
"""Resolve <package>.<name> as a submodule (e.g. litellm.utils) when no other handler matches"""
if name.startswith("__") or not name.isidentifier():
return None
qualified_name: Final = f"{package}.{name}"
try:
spec: Final = importlib.util.find_spec(qualified_name)
except ModuleNotFoundError:
return None
if spec is None:
return None
try:
module: Final = importlib.import_module(qualified_name)
except ModuleNotFoundError as exc:
if exc.name == qualified_name:
return None
raise
sys.modules[package].__dict__[name] = module # rebind-ok: caches the resolved submodule on the package
return module
def lazy_import_litellm_submodule(name: str) -> "ModuleType | None":
"""Resolve litellm.<name> as a submodule (e.g. litellm.utils) when no other handler matches"""
return lazy_import_submodule("litellm", name)
def _lazy_import_utils_module(name: str) -> object:
"""
Handler for utils module lazy imports.

2693
litellm/_lazy_imports_registry.py
View file

File diff suppressed because it is too large Load diff

17
litellm/_redis_credential_provider.py
View file

@ -1,7 +1,7 @@
import asyncio
import threading
import time
from typing import Any, Final
from typing import Final, Protocol
from redis.credentials import CredentialProvider
@ -18,6 +18,19 @@ _token_cache: Final[dict[str, tuple[str, float]]] = {}
_token_cache_lock: Final = threading.Lock()
class AzureAccessToken(Protocol):
"""The ``azure.core.credentials.AccessToken`` shape this module reads."""
@property
def token(self) -> str: ...
class AzureCredential(Protocol):
"""The ``azure-identity`` credential surface this module calls."""
def get_token(self, *scopes: str) -> AzureAccessToken: ...
def _generate_gcp_iam_access_token(service_account: str) -> str:
"""
Generate GCP IAM access token for Redis authentication.
@ -115,7 +128,7 @@ class AzureADCredentialProvider(CredentialProvider):
fail authentication after the initial token expired (~1 hour TTL).
"""
def __init__(self, credential: Any, username: str | None = None) -> None:
def __init__(self, credential: AzureCredential, username: str | None = None) -> None:
self._credential = credential
self._username = username

46
litellm/_service_logger.py
View file

@ -1,7 +1,7 @@
import asyncio
from collections.abc import Callable, Coroutine
from datetime import datetime, timedelta
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Any, Final, Protocol
import litellm
from litellm._logging import verbose_logger
@ -25,7 +25,30 @@ else:
UserAPIKeyAuth = Any
def _get_otel_v2_class() -> type | None:
class _ServiceSpanLogger(Protocol):
"""The OTel logger surface this module drives: the two service-span hooks it calls."""
async def async_service_success_hook(
self,
payload: ServiceLoggerPayload,
parent_otel_span: Span | None = None,
start_time: datetime | float | None = None,
end_time: datetime | float | None = None,
event_metadata: dict | None = None,
) -> None: ...
async def async_service_failure_hook(
self,
payload: ServiceLoggerPayload,
error: str | None = "",
parent_otel_span: Span | None = None,
start_time: datetime | float | None = None,
end_time: datetime | float | None = None,
event_metadata: dict | None = None,
) -> None: ...
def _get_otel_v2_class() -> type[_ServiceSpanLogger] | None:
"""Return the ``OpenTelemetryV2`` class, or ``None`` if the OTel SDK is absent.
Imported lazily: ``litellm.integrations.otel.logger`` imports the OpenTelemetry
@ -55,7 +78,7 @@ class ServiceLogging(CustomLogger):
if "prometheus_system" in litellm.service_callback:
self.prometheusServicesLogger = PrometheusServicesLogger()
def _resolve_otel_service_logger(self, callback: Any) -> Any | None:
def _resolve_otel_service_logger(self, callback: object) -> _ServiceSpanLogger | None:
"""Resolve the OTel logger (legacy or V2) to emit a service span on.
Returns the logger instance whose ``async_service_*_hook`` should fire for
@ -70,18 +93,21 @@ class ServiceLogging(CustomLogger):
"""
otel_v2_cls: Final = _get_otel_v2_class()
def _is_otel_logger(obj: Any) -> bool:
def _as_otel_logger(obj: object) -> _ServiceSpanLogger | None:
if isinstance(obj, OpenTelemetry):
return True
return otel_v2_cls is not None and isinstance(obj, otel_v2_cls)
return obj
if otel_v2_cls is not None and isinstance(obj, otel_v2_cls):
return obj
return None
if _is_otel_logger(callback):
return callback
resolved_callback: Final = _as_otel_logger(callback)
if resolved_callback is not None:
return resolved_callback
if callback == "otel":
from litellm.proxy.proxy_server import open_telemetry_logger
if open_telemetry_logger is not None and _is_otel_logger(open_telemetry_logger):
return open_telemetry_logger
if open_telemetry_logger is not None:
return _as_otel_logger(open_telemetry_logger)
return None
@staticmethod

3
litellm/a2a_protocol/card_resolver.py
View file

@ -4,6 +4,7 @@ Custom A2A Card Resolver for LiteLLM.
Extends the A2A SDK's card resolver to support multiple well-known paths.
"""
from collections.abc import Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final
@ -152,7 +153,7 @@ class LiteLLMA2ACardResolver(_A2ACardResolver):
async def get_agent_card(
self,
relative_card_path: str | None = None,
http_kwargs: dict[str, Any] | None = None,
http_kwargs: Mapping[str, object] | None = None,
) -> "AgentCard":
"""
Fetch the agent card, trying multiple well-known paths.

12
litellm/a2a_protocol/providers/bedrock_agentcore/handler.py
View file

@ -6,8 +6,8 @@ completion bridge that would otherwise strip the envelope.
"""
import json
from collections.abc import AsyncIterator
from typing import Any, Final, cast
from collections.abc import AsyncIterator, Mapping
from typing import Any, Final
from litellm._logging import verbose_logger
from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import (
@ -28,7 +28,7 @@ class BedrockAgentCoreA2AHandler:
@staticmethod
async def handle_non_streaming(
request_id: str,
params: dict[str, Any],
params: Mapping[str, object],
litellm_params: dict[str, Any],
agent_extra_headers: dict[str, str] | None = None,
) -> dict[str, Any]:
@ -56,7 +56,7 @@ class BedrockAgentCoreA2AHandler:
verbose_logger.info("BedrockAgentCore A2A: Sending non-streaming request to %s", url)
client: Final = get_async_httpx_client(
llm_provider=cast(Any, httpxSpecialProvider.A2AProvider),
llm_provider=httpxSpecialProvider.A2AProvider,
)
response: Final = await client.post(
url,
@ -74,7 +74,7 @@ class BedrockAgentCoreA2AHandler:
@staticmethod
async def handle_streaming(
request_id: str,
params: dict[str, Any],
params: Mapping[str, object],
litellm_params: dict[str, Any],
agent_extra_headers: dict[str, str] | None = None,
) -> AsyncIterator[dict[str, Any]]:
@ -103,7 +103,7 @@ class BedrockAgentCoreA2AHandler:
verbose_logger.info("BedrockAgentCore A2A: Sending streaming request to %s", url)
client: Final = get_async_httpx_client(
llm_provider=cast(Any, httpxSpecialProvider.A2AProvider),
llm_provider=httpxSpecialProvider.A2AProvider,
)
response: Final = await client.post(
url,

12
litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py
View file

@ -7,7 +7,7 @@ and signs requests via AmazonAgentCoreConfig (SigV4 or JWT).
import json
from collections.abc import AsyncIterator, Mapping
from typing import Any, Final
from typing import Any, Final, Protocol
from litellm._logging import verbose_logger
from litellm.a2a_protocol.litellm_completion_bridge.handler import (
@ -47,6 +47,12 @@ _RESERVED_PREFIX_HEADERS: Final[tuple[str, ...]] = (
)
class _SSELineSource(Protocol):
"""Minimal streaming-response surface used to read SSE lines."""
def aiter_lines(self) -> AsyncIterator[str]: ...
def _filter_reserved_headers(
agent_extra_headers: Mapping[str, str] | None,
) -> dict[str, str] | None:
@ -114,7 +120,7 @@ class BedrockAgentCoreA2ATransformation:
@staticmethod
def get_url_and_signed_request(
request_id: str,
params: dict[str, Any],
params: Mapping[str, object],
litellm_params: dict[str, Any],
method: str = "message/send",
stream: bool = False,
@ -213,7 +219,7 @@ class BedrockAgentCoreA2ATransformation:
return url, signed_headers, signed_body
@staticmethod
async def parse_sse_events(response: Any) -> AsyncIterator[dict[str, Any]]:
async def parse_sse_events(response: _SSELineSource) -> AsyncIterator[dict[str, Any]]:
"""
Parse SSE events from an httpx streaming response.

14
litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py
View file

@ -8,7 +8,7 @@ WXO uses a REST API (not A2A/JSON-RPC) with an async-poll execution model:
"""
import asyncio
from collections.abc import AsyncIterator
from collections.abc import AsyncIterator, Mapping
from typing import Any, Final
from uuid import uuid4
@ -51,9 +51,9 @@ class WatsonxOrchestrateTransformation:
wxo_agent_id: str,
text: str,
thread_id: str | None = None,
) -> dict[str, Any]:
) -> dict[str, object]:
"""Build the WXO POST /v1/orchestrate/runs request body."""
body: Final[dict[str, Any]] = {
body: Final[dict[str, object]] = {
"agent_id": wxo_agent_id,
"message": {
"role": "user",
@ -70,7 +70,7 @@ class WatsonxOrchestrateTransformation:
return body
@staticmethod
def extract_text_from_wxo_result(result: Any) -> str:
def extract_text_from_wxo_result(result: object) -> str:
"""
Extract response text from a WXO run result.
@ -103,7 +103,7 @@ class WatsonxOrchestrateTransformation:
return ""
@staticmethod
def extract_text_from_a2a_message_response(a2a_response: dict[str, Any]) -> str:
def extract_text_from_a2a_message_response(a2a_response: Mapping[str, object]) -> str:
result: Final = a2a_response.get("result")
if not isinstance(result, dict):
verbose_logger.warning("WXO: A2A response missing result object")
@ -119,7 +119,7 @@ class WatsonxOrchestrateTransformation:
return ""
@staticmethod
def build_a2a_message_response(request_id: str, text: str) -> dict[str, Any]:
def build_a2a_message_response(request_id: str, text: str) -> dict[str, object]:
"""
Build a standard A2A non-streaming SendMessageResponse (kind=message).
"""
@ -140,7 +140,7 @@ class WatsonxOrchestrateTransformation:
request_id: str,
chunk_size: int = 50,
delay_ms: int = 10,
) -> AsyncIterator[dict[str, Any]]:
) -> AsyncIterator[dict[str, object]]:
"""
Emit standard A2A streaming events from a completed text response.

4
litellm/a2a_protocol/streaming_iterator.py
View file

@ -148,9 +148,9 @@ class A2AStreamingIterator:
except Exception as e:
verbose_logger.debug("Error in A2A streaming completion handler: %s", e)
def _build_logging_result(self, usage: litellm.Usage) -> dict[str, Any]:
def _build_logging_result(self, usage: litellm.Usage) -> dict[str, object]:
"""Build a result dict for logging."""
result: Final[dict[str, Any]] = {
result: Final[dict[str, object]] = {
"id": getattr(self.request, "id", "unknown"),
"jsonrpc": "2.0",
"usage": (usage.model_dump() if hasattr(usage, "model_dump") else dict(usage)),

6
litellm/a2a_protocol/utils.py
View file

@ -48,7 +48,7 @@ class A2ARequestUtils:
return " ".join(text_parts)
@staticmethod
def extract_text_from_response(response_dict: dict[str, Any]) -> str:
def extract_text_from_response(response_dict: Mapping[str, object]) -> str:
"""
Extract text content from A2A response result.
@ -111,7 +111,7 @@ class A2ARequestUtils:
@staticmethod
def calculate_usage_from_request_response(
request: "SendMessageRequest | SendStreamingMessageRequest",
response_dict: dict[str, Any],
response_dict: Mapping[str, object],
) -> tuple[int, int, int]:
"""
Calculate token usage from A2A request and response.
@ -170,5 +170,5 @@ def extract_text_from_a2a_message(message: Any) -> str:
return A2ARequestUtils.extract_text_from_message(message)
def extract_text_from_a2a_response(response_dict: dict[str, Any]) -> str:
def extract_text_from_a2a_response(response_dict: Mapping[str, object]) -> str:
return A2ARequestUtils.extract_text_from_response(response_dict)

45
litellm/assistants/utils.py
View file

@ -1,3 +1,4 @@
from collections.abc import Mapping, Sequence
from typing import Final
import litellm
@ -10,20 +11,22 @@ def get_optional_params_add_message(
role: str | None,
content: str | list[MessageContentTextObject | MessageContentImageFileObject | MessageContentImageURLObject] | None,
attachments: list[Attachment] | None,
metadata: dict | None,
metadata: Mapping[str, object] | None,
custom_llm_provider: str,
**kwargs,
):
**kwargs: object,
) -> dict[str, object]:
"""
Azure doesn't support 'attachments' for creating a message
Reference - https://learn.microsoft.com/en-us/azure/ai-services/openai/assistants-reference-messages?tabs=python#create-message
"""
passed_params: Final = locals()
custom_llm_provider = passed_params.pop("custom_llm_provider")
special_params: Final = passed_params.pop("kwargs")
for k, v in special_params.items():
passed_params[k] = v
passed_params: Final[Mapping[str, object]] = {
"role": role,
"content": content,
"attachments": attachments,
"metadata": metadata,
**kwargs,
}
default_params: Final = {
"role": None,
@ -33,10 +36,10 @@ def get_optional_params_add_message(
}
non_default_params = {k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k])}
optional_params = {}
optional_params: dict[str, object] = {}
## raise exception if non-default value passed for non-openai/azure embedding calls
def _check_valid_arg(supported_params):
def _check_valid_arg(supported_params: Sequence[str]) -> Mapping[str, object] | None:
if len(non_default_params.keys()) > 0:
keys: Final = list(non_default_params.keys())
for k in keys:
@ -71,14 +74,18 @@ def get_optional_params_image_gen(
style: str | None = None,
user: str | None = None,
custom_llm_provider: str | None = None,
**kwargs,
):
**kwargs: object,
) -> dict[str, object]:
# retrieve all parameters passed to the function
passed_params: Final = locals()
custom_llm_provider = passed_params.pop("custom_llm_provider")
special_params: Final = passed_params.pop("kwargs")
for k, v in special_params.items():
passed_params[k] = v
passed_params: Final[Mapping[str, object]] = {
"n": n,
"quality": quality,
"response_format": response_format,
"size": size,
"style": style,
"user": user,
**kwargs,
}
default_params: Final = {
"n": None,
@ -90,10 +97,10 @@ def get_optional_params_image_gen(
}
non_default_params = {k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k])}
optional_params = {}
optional_params: dict[str, object] = {}
## raise exception if non-default value passed for non-openai/azure embedding calls
def _check_valid_arg(supported_params):
def _check_valid_arg(supported_params: Sequence[str]) -> Mapping[str, object] | None:
if len(non_default_params.keys()) > 0:
keys: Final = list(non_default_params.keys())
for k in keys:

12
litellm/batches/batch_utils.py
View file

@ -160,7 +160,7 @@ def _classify_output_line_stats(
def _safe_output_line_stats(
entry: Mapping[str, Any],
entry: Mapping[str, object],
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"],
model_name: str | None,
model_info: ModelInfo | None,
@ -182,7 +182,7 @@ def _safe_output_line_stats(
def _compute_output_line_stats(
entry: Mapping[str, Any],
entry: Mapping[str, object],
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"],
model_name: str | None,
model_info: ModelInfo | None,
@ -213,7 +213,7 @@ def _compute_output_line_stats(
def _output_line_cost(
response_body: Mapping[str, Any],
response_body: Mapping[str, object],
usage: Usage,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"],
model_name: str | None,
@ -556,7 +556,7 @@ def _iter_batch_output_entries(file_content: bytes) -> Iterator[dict]:
def _parse_batch_output_line(line: bytes) -> dict | None:
try:
parsed: Final = json.loads(line)
parsed: Final[object] = json.loads(line)
except ValueError as e:
verbose_logger.warning("skipping malformed batch output line: %s", str(e))
return None
@ -601,7 +601,7 @@ def _count_entry_tokens(
return 0
def _count_prompt_or_input_tokens(model: str, value: Any) -> int:
def _count_prompt_or_input_tokens(model: str, value: object) -> int:
"""Token-count a ``prompt`` / ``input`` field that the OpenAI batch
schema allows in four shapes:
@ -680,7 +680,7 @@ def _get_anthropic_result_from_batch_results_line(batch_results_line: Mapping[st
def _get_response_from_batch_job_output_file(
batch_job_output_file: Mapping[str, Any], custom_llm_provider: str = "openai"
) -> Mapping[str, Any]:
) -> Mapping[str, object]:
"""
Get the response from the batch job output file
"""

4
litellm/caching/caching_handler.py
View file

@ -672,7 +672,7 @@ class LLMCachingHandler:
def _async_log_cache_hit_on_callbacks(
self,
logging_obj: LiteLLMLoggingObj,
cached_result: Any,
cached_result: object,
start_time: datetime.datetime,
end_time: datetime.datetime,
cache_hit: bool,
@ -1184,7 +1184,7 @@ class LLMCachingHandler:
logging_obj: LiteLLMLoggingObj,
model: str,
kwargs: dict[str, Any],
cached_result: Any,
cached_result: object,
is_async: bool,
is_embedding: bool = False,
custom_llm_provider: str | None = None,

2
litellm/caching/dual_cache.py
View file

@ -257,7 +257,7 @@ class DualCache(BaseCache):
self,
current_time: float,
keys: list[str],
result: Sequence[Any],
result: Sequence[object],
) -> tuple[list[str], dict[str, float | None]]:
"""
Atomically choose keys to fetch from Redis and reserve their access time.

14
litellm/caching/redis_semantic_cache.py
View file

@ -116,7 +116,7 @@ class RedisSemanticCache(BaseCache):
password = password or os.environ["REDIS_PASSWORD"]
except KeyError as e:
# Raise a more informative exception if any of the required keys are missing
missing_var: Final = e.args[0]
missing_var: Final[object] = e.args[0]
raise ValueError(
f"Missing required Redis configuration: {missing_var}. Provide {missing_var} or redis_url."
) from e
@ -273,7 +273,7 @@ class RedisSemanticCache(BaseCache):
return prompt or None
@classmethod
def _collect_responses_input_text(cls, value: Any, prompt_parts: list[str]) -> None:
def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) -> None:
value = cls._coerce_response_input_value(value)
if value is None:
return
@ -334,7 +334,7 @@ class RedisSemanticCache(BaseCache):
resolve_embedding_max_input_tokens(self.embedding_max_input_tokens, self.embedding_model, router),
)
def _get_embedding(self, prompt: str, metadata: dict[str, Any] | None = None) -> list[float]:
def _get_embedding(self, prompt: str, metadata: dict[str, object] | None = None) -> list[float]:
"""
Routes through the proxy Router when the embedding model is a Router
deployment so per-deployment auth (e.g. Bedrock aws_role_name) applies,
@ -425,7 +425,7 @@ class RedisSemanticCache(BaseCache):
prompt_embedding: Final = self._get_embedding(prompt, metadata=kwargs.get("metadata"))
store_kwargs: Final[dict[str, Any]] = {
store_kwargs: Final[dict[str, object]] = {
"vector": prompt_embedding,
"filters": self._get_cache_filters(key),
}
@ -504,7 +504,7 @@ class RedisSemanticCache(BaseCache):
print_verbose(f"Error retrieving from Redis semantic cache: {e}")
kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
async def _get_async_embedding(self, prompt: str, metadata: dict[str, Any] | None = None) -> list[float]:
async def _get_async_embedding(self, prompt: str, metadata: dict[str, object] | None = None) -> list[float]:
"""
Asynchronously generate an embedding for the given prompt.
@ -571,7 +571,7 @@ class RedisSemanticCache(BaseCache):
# Generate embedding for the value (response) to cache
prompt_embedding: Final = await self._get_async_embedding(prompt, metadata=kwargs.get("metadata"))
store_kwargs: Final[dict[str, Any]] = {
store_kwargs: Final[dict[str, object]] = {
"vector": prompt_embedding,
"filters": self._get_cache_filters(key),
}
@ -665,7 +665,7 @@ class RedisSemanticCache(BaseCache):
aindex: Final = await self.llmcache._get_async_index()
return await aindex.info()
async def async_set_cache_pipeline(self, cache_list: list[tuple[str, Any]], **kwargs: object) -> None:
async def async_set_cache_pipeline(self, cache_list: list[tuple[str, object]], **kwargs: object) -> None:
"""
Asynchronously store multiple values in the semantic cache.

12
litellm/compression/compress.py
View file

@ -66,7 +66,7 @@ def _build_retrieval_tools(keys: list[str], call_type: str) -> list[dict]:
return cast(list[dict], anthropic_tools)
def _content_to_text(content: Any) -> str:
def _content_to_text(content: object) -> str:
"""
Convert OpenAI/Anthropic message content blocks to plain text.
@ -78,7 +78,7 @@ def _content_to_text(content: Any) -> str:
Implemented iteratively (stack-based) to avoid unbounded recursion.
"""
parts: Final[list[str]] = []
stack: Final[list[Any]] = [content]
stack: Final[list[object]] = [content]
while stack:
item = stack.pop()
if isinstance(item, str):
@ -111,7 +111,7 @@ def _normalize_messages_for_compression(
f"Unsupported call_type={call_type!r} for compression. Expected one of: {sorted(_SUPPORTED_CALL_TYPES)}."
)
original_messages: Final[list[dict[str, Any]]] = [dict(m) for m in messages]
original_messages: Final[list[dict[str, object]]] = [dict(m) for m in messages]
normalized_messages: Final[list[dict]] = []
for msg in original_messages:
@ -132,7 +132,7 @@ def _extract_last_user_message(messages: list[dict]) -> str:
return ""
def _extract_tool_use_ids(content: Any) -> list[str]:
def _extract_tool_use_ids(content: object) -> list[str]:
if not isinstance(content, list):
return []
tool_use_ids: Final[list[str]] = []
@ -147,7 +147,7 @@ def _extract_tool_use_ids(content: Any) -> list[str]:
return tool_use_ids
def _extract_tool_result_ids(content: Any) -> set[str]:
def _extract_tool_result_ids(content: object) -> set[str]:
if not isinstance(content, list):
return set()
tool_result_ids: Final[set[str]] = set()
@ -337,7 +337,7 @@ def compress(
compression_trigger: int = 200_000,
compression_target: int | None = None,
embedding_model: str | None = None,
embedding_model_params: dict[str, Any] | None = None,
embedding_model_params: Mapping[str, object] | None = None,
compression_cache: DualCache | None = None,
) -> CompressedResult:
"""

3
litellm/compression/scoring/embedding_scorer.py
View file

@ -5,6 +5,7 @@ Computes cosine similarity between the query embedding and each message embeddin
"""
import math
from collections.abc import Mapping
from typing import Any, Final
from litellm.caching.dual_cache import DualCache
@ -49,7 +50,7 @@ def embedding_score_messages(
messages: list[dict],
model: str,
cache: DualCache | None = None,
embedding_model_params: dict[str, Any] | None = None,
embedding_model_params: Mapping[str, object] | None = None,
) -> list[float]:
"""
Score each message's semantic similarity to the query using embeddings.

6
litellm/constants.py
View file

@ -40,7 +40,7 @@ ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG: Final[frozenset[str]] = frozenset(
"router_general_settings",
"ignore_invalid_deployments",
"fallback_access_check",
"heuristic_v2_router_limit",
"auto_router_capability_limit",
}
)
DEFAULT_BATCH_SIZE: Final = int(os.getenv("DEFAULT_BATCH_SIZE", 512))
@ -89,6 +89,7 @@ LITELLM_MAX_STREAMING_DURATION_SECONDS: Final = (
# Data URIs exceeding this are replaced with a size placeholder.
# Set to 0 to disable truncation.
MAX_BASE64_LENGTH_FOR_LOGGING: Final = int(os.getenv("MAX_BASE64_LENGTH_FOR_LOGGING", 64))
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS: Final = 256 * 1024
REDACTED_BY_LITELLM: Final = "redacted-by-litellm"
# in-memory stand-in handed to provider converters for redacted arguments; never stored
REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER: Final = "{}"
@ -215,6 +216,9 @@ MAX_CALLBACKS: Final = get_env_int("LITELLM_MAX_CALLBACKS", 100)
# so the deployment-level hook does not re-run them for the same request
PRE_CALL_EXECUTED_GUARDRAILS_KEY: Final = "_pre_call_executed_guardrails"
# Attribute stamped on log_guardrail_information wrappers so __init_subclass__ does not wrap them again
LOGS_GUARDRAIL_INFORMATION_MARKER: Final = "_litellm_logs_guardrail_information"
# Generic fallback for unknown models
DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET: Final = int(
os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET", 128)

14
litellm/containers/endpoint_factory.py
View file

@ -11,7 +11,7 @@ import json
from collections.abc import Callable
from functools import partial
from pathlib import Path
from typing import Any, Final, Literal
from typing import Final, Literal
import litellm
from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT
@ -56,9 +56,9 @@ def create_sync_endpoint_function(endpoint_config: dict) -> Callable:
def endpoint_func(
timeout: int = 600,
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
extra_headers: dict[str, Any] | None = None,
extra_query: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
extra_headers: dict[str, object] | None = None,
extra_query: dict[str, object] | None = None,
extra_body: dict[str, object] | None = None,
**kwargs,
):
local_vars: Final = locals()
@ -145,9 +145,9 @@ def create_async_endpoint_function(
async def async_endpoint_func(
timeout: int = 600,
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
extra_headers: dict[str, Any] | None = None,
extra_query: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
extra_headers: dict[str, object] | None = None,
extra_query: dict[str, object] | None = None,
extra_body: dict[str, object] | None = None,
**kwargs,
):
local_vars: Final = locals()

4
litellm/exceptions.py
View file

@ -85,7 +85,7 @@ _RATE_LIMIT_CATEGORY_VALUES: Final = frozenset(c.value for c in RateLimitErrorCa
_RATE_LIMIT_TYPE_VALUES: Final = frozenset(t.value for t in RateLimitType)
def validate_rate_limit_category(value: Any) -> str | None:
def validate_rate_limit_category(value: object) -> str | None:
"""Return ``value`` only if it matches a known :class:`RateLimitErrorCategory`.
Used at duck-typed read sites (StandardLoggingPayload extraction, Prometheus
@ -100,7 +100,7 @@ def validate_rate_limit_category(value: Any) -> str | None:
return None
def validate_rate_limit_type(value: Any) -> str | None:
def validate_rate_limit_type(value: object) -> str | None:
"""Return ``value`` only if it matches a known :class:`RateLimitType`.
See :func:`validate_rate_limit_category` for the rationale.

30
litellm/experimental_mcp_client/client.py
View file

@ -6,17 +6,35 @@ import asyncio
import base64
import os
from collections.abc import Awaitable, Callable, Generator
from contextlib import AbstractAsyncContextManager
from datetime import timedelta
from functools import partial
from importlib import metadata
from typing import Any, Final, TypeVar
from typing import Any, Final, Protocol, TypeAlias, TypeVar
import httpx
from anyio.streams.memory import MemoryObjectReceiveStream, MemoryObjectSendStream
from mcp import ClientSession, McpError, ReadResourceResult, Resource, StdioServerParameters
from mcp.client.sse import sse_client
from mcp.client.stdio import stdio_client
from mcp.shared.message import SessionMessage
from typing_extensions import Unpack
streamable_http_client: Any | None = None
_TransportStreams: TypeAlias = tuple[
MemoryObjectReceiveStream[SessionMessage | Exception],
MemoryObjectSendStream[SessionMessage],
Unpack[tuple[object, ...]],
]
_TransportContext: TypeAlias = AbstractAsyncContextManager[_TransportStreams]
class _StreamableHttpClientFactory(Protocol):
"""The ``streamable_http_client`` entry point this module calls on the installed MCP SDK."""
def __call__(self, *, url: str, http_client: httpx.AsyncClient | None) -> _TransportContext: ...
streamable_http_client: _StreamableHttpClientFactory | None = None
try:
import mcp.client.streamable_http as streamable_http_module
@ -216,10 +234,12 @@ class MCPSigV4Auth(httpx.Auth):
aws_region_name: str,
):
"""Call STS AssumeRole and return temporary credentials."""
import time
import boto3
from botocore.credentials import Credentials
session_name: Final = aws_session_name or f"litellm-mcp-{int(__import__('time').time())}"
session_name: Final = aws_session_name or f"litellm-mcp-{int(time.time())}"
sts_kwargs: Final[dict] = {"region_name": aws_region_name}
if aws_access_key_id and aws_secret_access_key:
sts_kwargs["aws_access_key_id"] = aws_access_key_id
@ -315,7 +335,7 @@ class MCPClient:
def _create_transport_context(
self,
) -> tuple[Any, httpx.AsyncClient | None]:
) -> tuple[_TransportContext, httpx.AsyncClient | None]:
"""
Create the appropriate transport context based on transport type.
Returns:
@ -408,7 +428,7 @@ class MCPClient:
async def _execute_session_operation(
self,
transport_ctx: Any,
transport_ctx: _TransportContext,
operation: Callable[[ClientSession], Awaitable[TSessionResult]],
) -> TSessionResult:
"""

11
litellm/files/main.py
View file

@ -14,6 +14,7 @@ from functools import partial
from typing import Any, Final, Literal, cast
import httpx
from openai import AsyncOpenAI, OpenAI
# Type aliases for provider parameters
FileCreateProvider = Literal[
@ -431,7 +432,7 @@ async def afile_delete(
extra_headers: dict[str, str] | None = None,
extra_body: dict[str, str] | None = None,
**kwargs,
) -> Coroutine[Any, Any, FileObject]:
) -> Coroutine[object, object, FileObject]:
"""
Async: Delete file
@ -1002,8 +1003,8 @@ def file_content_streaming(
timeout: float | httpx.Timeout,
logging_obj: LiteLLMLoggingObj | None,
_is_async: bool,
client: Any | None,
) -> FileContentStreamingResult | Coroutine[Any, Any, FileContentStreamingResult]:
client: OpenAI | AsyncOpenAI | None,
) -> FileContentStreamingResult | Coroutine[object, object, FileContentStreamingResult]:
if logging_obj is not None:
logging_obj.model = model or ""
logging_obj.model_call_details["model"] = model or ""
@ -1028,8 +1029,8 @@ def file_content_streaming(
headers=response.headers,
)
response: FileContentStreamingResult | Coroutine[Any, Any, FileContentStreamingResult] = FileContentStreamingResult(
stream_iterator=iter(()), headers={}
response: FileContentStreamingResult | Coroutine[object, object, FileContentStreamingResult] = (
FileContentStreamingResult(stream_iterator=iter(()), headers={})
)
if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
openai_creds: Final = get_openai_credentials(

14
litellm/fine_tuning/main.py
View file

@ -11,7 +11,7 @@ https://platform.openai.com/docs/api-reference/fine-tuning
import asyncio
import contextvars
import os
from collections.abc import Coroutine
from collections.abc import Coroutine, Mapping
from functools import partial
from typing import Any, Final, Literal
@ -37,8 +37,8 @@ vertex_fine_tuning_apis_instance: Final = VertexFineTuningAPI()
def _prepare_azure_extra_body(
extra_body: dict[str, Any] | None,
kwargs: dict[str, Any],
azure_specific_hyperparams: dict[str, Any],
kwargs: Mapping[str, object],
azure_specific_hyperparams: Mapping[str, object],
) -> dict[str, Any]:
"""
Prepare extra_body for Azure fine-tuning API by combining Azure-specific parameters.
@ -138,7 +138,7 @@ def _build_fine_tuning_job_data(model, training_file, hyperparameters, suffix, v
def _resolve_fine_tuning_timeout(
timeout: Any,
timeout: float | str | httpx.Timeout | None,
custom_llm_provider: str,
) -> float | httpx.Timeout:
"""Normalise a raw timeout value to a float (seconds) or httpx.Timeout for fine-tuning calls."""
@ -163,7 +163,7 @@ def create_fine_tuning_job(
extra_headers: dict[str, str] | None = None,
extra_body: dict[str, str] | None = None,
**kwargs,
) -> LiteLLMFineTuningJob | Coroutine[Any, Any, LiteLLMFineTuningJob]:
) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
"""
Creates a fine-tuning job which begins the process of creating a new model from a given dataset.
@ -375,7 +375,7 @@ def cancel_fine_tuning_job(
extra_headers: dict[str, str] | None = None,
extra_body: dict[str, str] | None = None,
**kwargs,
) -> LiteLLMFineTuningJob | Coroutine[Any, Any, LiteLLMFineTuningJob]:
) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
"""
Immediately cancel a fine-tune job.
@ -682,7 +682,7 @@ def retrieve_fine_tuning_job(
extra_headers: dict[str, str] | None = None,
extra_body: dict[str, str] | None = None,
**kwargs,
) -> LiteLLMFineTuningJob | Coroutine[Any, Any, LiteLLMFineTuningJob]:
) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
"""
Get info about a fine-tuning job.
"""

3
litellm/images/utils.py
View file

@ -1,3 +1,4 @@
from collections.abc import Mapping
from io import BufferedReader, BytesIO
from typing import Any, Final, cast, get_type_hints
@ -61,7 +62,7 @@ class ImageEditRequestUtils:
@staticmethod
def get_requested_image_edit_optional_param(
params: dict[str, Any],
params: Mapping[str, object],
) -> ImageEditOptionalRequestParams:
"""
Filter parameters to only include those defined in ImageEditOptionalRequestParams.

23
litellm/integrations/arize/_utils.py
View file

@ -2,7 +2,7 @@ import json
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
from typing_extensions import override
from typing_extensions import ReadOnly, TypedDict, override
from litellm._logging import verbose_logger
from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes import (
@ -492,12 +492,12 @@ def _sanitize_optional_params(optional_params: dict | None) -> dict:
return optional_params
def _set_metadata_attributes(span: "Span", metadata: Any | None, span_attrs) -> None:
def _set_metadata_attributes(span: "Span", metadata: object | None, span_attrs) -> None:
if metadata is not None:
safe_set_attribute(span, span_attrs.METADATA, safe_dumps(metadata))
def _extract_metadata_tools(metadata: Any | None) -> list | None:
def _extract_metadata_tools(metadata: object | None) -> list | None:
if not isinstance(metadata, dict):
return None
llm_obj: Final = metadata.get("llm")
@ -670,7 +670,22 @@ def _get_tool_calls(message) -> list | None:
return tool_calls if isinstance(tool_calls, list) and tool_calls else None
def _normalize_tool_call(raw_tc) -> dict[str, Any] | None:
class _NormalizedToolCallFunction(TypedDict):
"""The ``function`` sub-object of a normalized tool call."""
name: ReadOnly[object]
arguments: ReadOnly[object]
class _NormalizedToolCall(TypedDict):
"""A tool call reduced to the stable shape the OpenInference emitters read."""
id: ReadOnly[object]
type: ReadOnly[object]
function: ReadOnly[_NormalizedToolCallFunction]
def _normalize_tool_call(raw_tc) -> _NormalizedToolCall | None:
"""Normalize a single tool_call (dict or Pydantic) into a stable shape:
{"id": str|None, "type": str, "function": {"name": str|None, "arguments": str|None}}

6
litellm/integrations/cloudzero/cz_stream_api.py
View file

@ -97,7 +97,11 @@ class CloudZeroStreamer:
continue
# Convert lists back to DataFrames
return {date_key: pl.DataFrame(records) for date_key, records in daily_batches.items() if records}
return {
date_key: pl.DataFrame(records, infer_schema_length=None)
for date_key, records in daily_batches.items()
if records
}
def _parse_and_convert_timestamp(self, timestamp_str: str) -> datetime:
"""Parse timestamp string and convert to UTC."""

2
litellm/integrations/cloudzero/transform.py
View file

@ -95,7 +95,7 @@ class CBFTransformer:
if len(cbf_data) > 0:
console.print(f"[green]✓ Successfully transformed {len(cbf_data):,} records[/green]")
return pl.DataFrame(cbf_data)
return pl.DataFrame(cbf_data, infer_schema_length=None)
def _create_cbf_record(self, row: dict[str, object]) -> CBFRecord:
"""Create a single CBF record from LiteLLM daily spend row."""

29
litellm/integrations/custom_guardrail.py
View file

@ -46,6 +46,7 @@ dc: Final = DualCache()
from litellm.constants import (
GUARDRAIL_SCANNED_MESSAGES_CACHE_TTL_SECONDS,
LOGS_GUARDRAIL_INFORMATION_MARKER,
PRE_CALL_EXECUTED_GUARDRAILS_KEY,
)
from litellm.exceptions import (
@ -151,6 +152,13 @@ class CustomGuardrail(CustomLogger):
records_own_guardrail_information: ClassVar[bool] = False
def __init_subclass__(cls, **kwargs: object) -> None: # kwargs-ok: forwarded to cooperative __init_subclass__ hooks
super().__init_subclass__(**kwargs)
own_apply_guardrail: Final = cls.__dict__.get("apply_guardrail")
if own_apply_guardrail is None or LOGS_GUARDRAIL_INFORMATION_MARKER in vars(own_apply_guardrail):
return
cls.apply_guardrail = log_guardrail_information(own_apply_guardrail)
def __init__(
self,
guardrail_name: str | None = None,
@ -940,6 +948,23 @@ class CustomGuardrail(CustomLogger):
"""
return False
def _suppressed_by_auto_router_compression(self) -> bool:
"""True when an auto router's own compression policy suppresses this guardrail.
Reads request-scoped state set by `arm_pre_call`, never request metadata. The
caller controls metadata, and metadata reaches spend logs the caller can read,
so a suppression list carried there would be one a request could replay to
switch off a PII or content-filter guardrail for itself.
"""
name: Final = self.guardrail_name
if not name:
return False
from litellm.proxy.guardrails.auto_router_compression import (
suppressed_compression_guardrails,
)
return name in suppressed_compression_guardrails()
def should_run_guardrail(
self,
data,
@ -948,6 +973,9 @@ class CustomGuardrail(CustomLogger):
"""
Returns True if the guardrail should be run on the event_type
"""
if self._suppressed_by_auto_router_compression():
return False
requested_guardrails: Final = self.get_guardrail_from_metadata(data)
disable_global_guardrail: Final = self.get_disable_global_guardrail(data)
opted_out_global_guardrails: Final = self.get_opted_out_global_guardrails_from_metadata(data)
@ -1559,4 +1587,5 @@ def log_guardrail_information(func):
return async_wrapper(*args, **kwargs)
return sync_wrapper(*args, **kwargs)
vars(wrapper)[LOGS_GUARDRAIL_INFORMATION_MARKER] = True # rebind-ok: stamps the wrapper this call just built
return wrapper

5
litellm/integrations/datadog/datadog_cost_management.py
View file

@ -1,6 +1,7 @@
import asyncio
import os
import time
from collections.abc import Mapping
from datetime import datetime
from typing import Any, Final, cast
@ -181,7 +182,7 @@ class DatadogCostManagementLogger(CustomBatchLogger):
# cast because StandardLoggingMetadata is a TypedDict; we iterate it
# as a generic mapping below.
metadata: Final[dict[str, Any]] = cast(dict[str, Any], log.get("metadata") or {})
metadata: Final[Mapping[str, object]] = cast(dict[str, Any], log.get("metadata") or {})
# Backwards-compat: team/user/model_group preserved regardless of allowlist.
if metadata.get("user_api_key_alias"):
@ -233,7 +234,7 @@ class DatadogCostManagementLogger(CustomBatchLogger):
tags[key] = normalize_datadog_tag_value(value)
@staticmethod
def _add_tag(tags: dict[str, str], key: str, value: Any) -> None:
def _add_tag(tags: dict[str, str], key: str, value: object) -> None:
if value:
tags[key] = str(value)

7
litellm/integrations/dotprompt/dotprompt_manager.py
View file

@ -4,6 +4,7 @@ Builds on top of PromptManagementBase to provide .prompt file support.
"""
import json
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
from litellm.integrations.custom_prompt_management import CustomPromptManagement
@ -347,14 +348,14 @@ class DotpromptManager(CustomPromptManagement):
metadata: Final = json_data.get("metadata", {})
self.prompt_manager.add_prompt(prompt_id, content, metadata)
def load_prompts_from_json(self, prompts_data: dict[str, dict[str, Any]]) -> None:
def load_prompts_from_json(self, prompts_data: dict[str, dict[str, object]]) -> None:
"""Load multiple prompts from JSON data."""
self.prompt_manager.load_prompts_from_json_data(prompts_data)
def get_prompts_as_json(self) -> dict[str, dict[str, Any]]:
def get_prompts_as_json(self) -> dict[str, dict[str, object]]:
"""Get all prompts in JSON format."""
return self.prompt_manager.get_all_prompts_as_json()
def convert_prompt_file_to_json(self, file_path: str) -> dict[str, Any]:
def convert_prompt_file_to_json(self, file_path: str) -> Mapping[str, object]:
"""Convert a .prompt file to JSON format."""
return self.prompt_manager.prompt_file_to_json(file_path)

50
litellm/integrations/focus/destinations/s3_destination.py
View file

@ -3,14 +3,26 @@
from __future__ import annotations
import asyncio
from collections.abc import Mapping
from datetime import timezone
from typing import Any, Final
from typing import Final, TypedDict
import boto3
from typing_extensions import ReadOnly
from .base import FocusDestination, FocusTimeWindow
class _S3ClientKwargs(TypedDict, total=False):
"""Optional boto3 client arguments the destination config may supply."""
region_name: ReadOnly[str]
endpoint_url: ReadOnly[str]
aws_access_key_id: ReadOnly[str]
aws_secret_access_key: ReadOnly[str]
aws_session_token: ReadOnly[str]
class FocusS3Destination(FocusDestination):
"""Handles uploading serialized exports to S3 buckets."""
@ -18,7 +30,7 @@ class FocusS3Destination(FocusDestination):
self,
*,
prefix: str,
config: dict[str, Any] | None = None,
config: Mapping[str, str] | None = None,
) -> None:
config = config or {}
bucket_name: Final = config.get("bucket_name")
@ -47,25 +59,23 @@ class FocusS3Destination(FocusDestination):
key_prefix: Final = "/".join(filter(None, parts))
return f"{key_prefix}/{filename}" if key_prefix else filename
def _client_kwargs(self) -> _S3ClientKwargs:
"""Collect the boto3 client arguments the destination config provides."""
region: Final = self.config.get("region_name")
endpoint: Final = self.config.get("endpoint_url")
key_id: Final = self.config.get("aws_access_key_id")
secret: Final = self.config.get("aws_secret_access_key")
token: Final = self.config.get("aws_session_token")
return {
**(_S3ClientKwargs(region_name=region) if region else _S3ClientKwargs()),
**(_S3ClientKwargs(endpoint_url=endpoint) if endpoint else _S3ClientKwargs()),
**(_S3ClientKwargs(aws_access_key_id=key_id) if key_id else _S3ClientKwargs()),
**(_S3ClientKwargs(aws_secret_access_key=secret) if secret else _S3ClientKwargs()),
**(_S3ClientKwargs(aws_session_token=token) if token else _S3ClientKwargs()),
}
def _upload(self, content: bytes, object_key: str) -> None:
client_kwargs: Final[dict[str, Any]] = {}
region_name: Final = self.config.get("region_name")
if region_name:
client_kwargs["region_name"] = region_name
endpoint_url: Final = self.config.get("endpoint_url")
if endpoint_url:
client_kwargs["endpoint_url"] = endpoint_url
session_kwargs: Final[dict[str, Any]] = {}
for key in (
"aws_access_key_id",
"aws_secret_access_key",
"aws_session_token",
):
if self.config.get(key):
session_kwargs[key] = self.config[key]
s3_client: Final = boto3.client("s3", **client_kwargs, **session_kwargs)
s3_client: Final = boto3.client("s3", **self._client_kwargs())
s3_client.put_object(
Bucket=self.bucket_name,
Key=object_key,

4
litellm/integrations/focus/focus_logger.py
View file

@ -102,7 +102,7 @@ class FocusLogger(CustomLogger):
# No time bounds → export all available data
await self._export_all(limit=limit)
async def dry_run_export_usage_data(self, limit: int | None = DEFAULT_DRY_RUN_LIMIT) -> dict[str, Any]:
async def dry_run_export_usage_data(self, limit: int | None = DEFAULT_DRY_RUN_LIMIT) -> dict[str, object]:
"""Return transformed data without uploading."""
engine: Final = self._ensure_engine()
return await engine.dry_run_export_usage_data(limit=limit)
@ -153,7 +153,7 @@ class FocusLogger(CustomLogger):
**trigger_kwargs,
)
def _build_scheduler_trigger(self) -> dict[str, Any]:
def _build_scheduler_trigger(self) -> dict[str, str | int]:
"""Return scheduler configuration for the selected frequency."""
if self.frequency == "interval":
seconds: Final = self.interval_seconds or 60

3
litellm/integrations/generic_prompt_management/generic_prompt_manager.py
View file

@ -4,6 +4,7 @@ Fetches prompts from any API that implements the /beta/litellm_prompt_management
"""
import json
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
import httpx
@ -349,7 +350,7 @@ class GenericPromptManager(CustomPromptManagement):
def _apply_variables(
self,
prompt_client: PromptManagementClient,
variables: dict[str, Any],
variables: Mapping[str, object],
) -> PromptManagementClient:
"""
Apply variables to the prompt template.

6
litellm/integrations/humanloop.py
View file

@ -4,7 +4,7 @@ Humanloop integration
https://humanloop.com/
"""
from typing import Any, Final, cast
from typing import Final, cast
import httpx
from typing_extensions import TypedDict
@ -24,7 +24,7 @@ class PromptManagementClient(TypedDict):
prompt_id: str
prompt_template: list[AllMessageValues]
model: str | None
optional_params: dict[str, Any] | None
optional_params: dict[str, object] | None
class HumanLoopPromptManager(DualCache):
@ -36,7 +36,7 @@ class HumanLoopPromptManager(DualCache):
return cast(PromptManagementClient | None, self.get_cache(key=humanloop_prompt_id))
def _compile_prompt_helper(
self, prompt_template: list[AllMessageValues], prompt_variables: dict[str, Any]
self, prompt_template: list[AllMessageValues], prompt_variables: dict[str, object]
) -> list[AllMessageValues]:
"""
Helper function to compile the prompt by substituting variables in the template.

24
litellm/integrations/newrelic/newrelic.py
View file

@ -47,6 +47,8 @@ import os
import threading
import time
import uuid
from collections.abc import Mapping, Sequence
from datetime import datetime
from typing import Any, Final
import litellm
@ -408,8 +410,8 @@ class NewRelicLogger(CustomLogger):
def _get_duration(
self,
kwargs: dict,
start_time: Any,
end_time: Any,
start_time: datetime | float | None,
end_time: datetime | float | None,
standard_logging_object: StandardLoggingPayload | None = None,
) -> float | None:
"""
@ -438,7 +440,7 @@ class NewRelicLogger(CustomLogger):
self,
kwargs: dict,
standard_logging_object: StandardLoggingPayload | None = None,
) -> dict[str, Any]:
) -> dict[str, object]:
"""
Extract request parameters like temperature and max_tokens, preferring
StandardLoggingPayload.model_parameters.
@ -450,7 +452,7 @@ class NewRelicLogger(CustomLogger):
else:
source_params = kwargs.get("optional_params") or {}
params: Final = {}
params: Final[dict[str, object]] = {}
temperature: Final = source_params.get("temperature")
if temperature is not None:
@ -502,7 +504,7 @@ class NewRelicLogger(CustomLogger):
response_model: str,
vendor: str,
standard_logging_object: StandardLoggingPayload | None = None,
) -> list[dict[str, Any]]:
) -> Sequence[Mapping[str, object]]:
"""
Extract all messages (request + response) with sequence numbers and timestamps.
@ -512,7 +514,7 @@ class NewRelicLogger(CustomLogger):
Adds timestamps from StandardLoggingPayload (preferred) or kwargs if available
(converted to epoch milliseconds).
"""
messages: Final = []
messages: Final[list[dict[str, object]]] = []
sequence = 0
# Extract timestamps, preferring StandardLoggingPayload
@ -544,7 +546,7 @@ class NewRelicLogger(CustomLogger):
else:
request_messages = kwargs.get("messages") or []
for msg in request_messages:
message_data = {
message_data: dict[str, object] = {
"role": msg.get("role") or "user",
"sequence": sequence,
"response.model": response_model,
@ -599,11 +601,11 @@ class NewRelicLogger(CustomLogger):
num_messages: int,
usage: dict[str, int],
duration: float | None = None,
request_params: dict[str, Any] | None = None,
request_params: Mapping[str, object] | None = None,
):
"""Record LlmChatCompletionSummary event to New Relic."""
try:
event_data: Final = {
event_data: Final[dict[str, object]] = {
"id": request_id,
"request_id": request_id,
"request.model": request_model,
@ -647,7 +649,7 @@ class NewRelicLogger(CustomLogger):
request_id: str,
llm_response_id: str,
trace_id: str | None,
messages: list[dict[str, Any]],
messages: Sequence[Mapping[str, object]],
):
"""Record LlmChatCompletionMessage events to New Relic.
@ -666,7 +668,7 @@ class NewRelicLogger(CustomLogger):
for message in messages:
sequence = message["sequence"]
event_data = {
event_data: dict[str, object] = {
"id": f"{llm_response_id}-{sequence}",
"request_id": request_id,
"completion_id": request_id,

10
litellm/integrations/opentelemetry.py
View file

@ -1,7 +1,7 @@
import os
import threading
from collections import OrderedDict
from collections.abc import Callable, Mapping
from collections.abc import Callable, Iterable, Mapping
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field
from datetime import datetime
@ -166,7 +166,7 @@ class OTELMetricAttributeFilter:
exclude_list: list[str] | None = None
def _build_metric_attribute_filter(value: Any) -> OTELMetricAttributeFilter:
def _build_metric_attribute_filter(value: object) -> OTELMetricAttributeFilter:
if isinstance(value, OTELMetricAttributeFilter):
return value
if not isinstance(value, dict):
@ -205,7 +205,7 @@ def _resolve_metric_attribute_filter(
)
def _normalize_team_metadata_keys(value: Any) -> list[str]:
def _normalize_team_metadata_keys(value: str | Iterable[object] | None) -> list[str]:
"""Coerce a team-metadata allowlist from a list or comma-separated string.
config.yaml passes a YAML list; an env var passes a comma-separated string.
@ -1569,7 +1569,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
self.safe_set_attribute(span=span, key=RESPONSE_SERVICE_TIER_ATTRIBUTE, value=served_tier)
@staticmethod
def _team_metadata_json(value: Any, allowed_keys: list[str]) -> str | None:
def _team_metadata_json(value: object, allowed_keys: list[str]) -> str | None:
"""JSON-serialize only the allowlisted sub-keys of a team's metadata.
Returns ``None`` when nothing is allowlisted or no allowlisted key is
@ -3524,7 +3524,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
kwargs={"standard_logging_object": {"error_information": error_information}},
)
def set_preprocessing_duration_attribute(self, span: Span | None, container: Any) -> None:
def set_preprocessing_duration_attribute(self, span: Span | None, container: object) -> None:
"""
Set ``litellm.preprocessing.duration_ms`` (proxy-receive -> first
provider handoff) on the proxy SERVER span. ``litellm_received_at``

6
litellm/integrations/opentelemetry_utils/gen_ai_semconv.py
View file

@ -117,7 +117,7 @@ class OTELGenAISemconvMixin:
if TYPE_CHECKING:
config: "OpenTelemetryConfig"
def safe_set_attribute(self, span: Span, key: str, value: Any) -> None: ...
def safe_set_attribute(self, span: Span, key: str, value: object) -> None: ...
def _capture_in_event(self) -> bool: ...
@ -195,13 +195,13 @@ class OTELGenAISemconvMixin:
if value:
self.safe_set_attribute(span=span, key=semconv_key, value=value)
def _build_inference_details_attrs(self, kwargs: dict, response_obj: dict, provider: str) -> dict[str, Any]:
def _build_inference_details_attrs(self, kwargs: dict, response_obj: dict, provider: str) -> dict[str, str]:
"""Build the attribute payload for the inference-details event.
Always includes provider/operation; input/output messages are added
only when content capture is enabled and non-empty. Mixin-internal.
"""
attrs: Final[dict[str, Any]] = {
attrs: Final[dict[str, str]] = {
"event_name": _INFERENCE_DETAILS_EVENT_NAME,
"gen_ai.provider.name": provider,
"gen_ai.operation.name": self._gen_ai_operation_name(kwargs),

8
litellm/integrations/opik/opik_payload_builder/payload_builders.py
View file

@ -15,8 +15,8 @@ def build_trace_payload(
response_obj: dict[str, Any],
start_time: datetime,
end_time: datetime,
input_data: Any,
output_data: Any,
input_data: object,
output_data: object,
metadata: dict[str, object],
tags: list[str],
thread_id: str | None,
@ -45,8 +45,8 @@ def build_span_payload(
response_obj: dict[str, Any],
start_time: datetime,
end_time: datetime,
input_data: Any,
output_data: Any,
input_data: object,
output_data: object,
metadata: dict[str, object],
tags: list[str],
usage: dict[str, int],

9
litellm/integrations/otel/README.md
View file

@ -35,6 +35,15 @@ span orphaned into its own trace). The anchor — a contextvar inherited by thos
child tasks — gives a stable parent in both cases. DB/service spans keep ambient
parenting so an auth DB lookup still nests under `auth`.
The anchor is also what `litellm.request.route` is read from: `request_root_http_route`
returns the server span's own `http.route`, so the LLM call span cannot disagree with
its parent about which endpoint served the request. That means the route template on a
normal route and the literal path on a passthrough prefix, because the passthrough hook
rewrote the attribute; an MCP call anchors the same server span, so it reports the
`/mcp` mount point. Attributes stay readable after a span ends, so the async close
callback reads the same value. Where no server span was anchored at all, the route the
proxy recorded at auth (`metadata.user_api_key_request_route`) is the backstop.
**Which service calls become spans (`spans.span_role_for_service`).** LiteLLM's
service-logging layer instruments many internal functions, but only some are
traceable units of work:

2
litellm/integrations/otel/logger.py
View file

@ -49,6 +49,7 @@ from litellm.integrations.otel.model.utils import to_ns
from litellm.integrations.otel.plumbing.context import (
is_recordable_span,
mcp_message_transport_span,
request_root_http_route,
request_root_span,
resolve_mcp_span_context,
resolve_parent_context,
@ -541,6 +542,7 @@ class OpenTelemetryV2(CustomLogger):
payload,
capture_content=self.config.capture_span_content,
time_to_first_chunk_seconds=call.time_to_first_chunk_seconds,
request_route=request_root_http_route(),
)
end_time_ns: Final = to_ns(end_time)
if carrier is not None and carrier.span is not None:

1
litellm/integrations/otel/mappers/genai.py
View file

@ -89,6 +89,7 @@ class GenAIMapper:
f"{LiteLLM.COST_PREFIX}margin_percent": lambda d: d.cost.margin_percent,
f"{LiteLLM.COST_PREFIX}margin_total_amount": lambda d: d.cost.margin_total_amount,
LiteLLM.REQUEST_STREAMING: lambda d: d.is_streaming,
LiteLLM.REQUEST_ROUTE: lambda d: d.request_route,
}
_TOOL_ATTRS: dict[str, Callable[[ToolDefinition], AttrValue | None]] = {

2
litellm/integrations/otel/model/metadata.py
View file

@ -64,6 +64,7 @@ class RequestIdentity:
# completes (routing has picked a deployment), so it's absent from the
# auth-time seed and filled only from the payload.
provider_model: str | None = None
request_route: str | None = None
metadata: Mapping[str, str] = field(default_factory=dict)
@classmethod
@ -87,6 +88,7 @@ class RequestIdentity:
key_hash=as_str(raw_meta.get("user_api_key_hash")),
end_user=as_str(payload.get("end_user")) or as_str(raw_meta.get("user_api_key_end_user_id")),
provider_model=resolve_provider_model(payload),
request_route=as_str(raw_meta.get("user_api_key_request_route")),
metadata=metadata,
)

3
litellm/integrations/otel/model/payloads.py
View file

@ -386,6 +386,7 @@ class LLMCallSpanData:
# keeps routes the convention folds into one operation distinguishable.
output_type: GenAIOutputType | None = None
call_type: str | None = None
request_route: str | None = None
@classmethod
def from_standard_logging_payload(
@ -393,6 +394,7 @@ class LLMCallSpanData:
payload: StandardLoggingPayload,
capture_content: bool = False,
time_to_first_chunk_seconds: float | None = None,
request_route: str | None = None,
) -> LLMCallSpanData:
params: Final = cast(Mapping[str, object], payload.get("model_parameters") or {})
# The single parse of the request's metadata — the request-vs-provider
@ -433,6 +435,7 @@ class LLMCallSpanData:
time_to_first_chunk_seconds=time_to_first_chunk_seconds,
output_type=resolve_output_type(call_type),
call_type=call_type or None,
request_route=request_route or context.identity.request_route,
)

1
litellm/integrations/otel/model/semconv.py
View file

@ -295,6 +295,7 @@ class LiteLLM:
# ``litellm_params.model``), distinct from the user-facing ``gen_ai.request.model``.
PROVIDER_MODEL: Final = "litellm.provider.model"
REQUEST_STREAMING: Final = "litellm.request.streaming"
REQUEST_ROUTE: Final = "litellm.request.route"
TOOLS_DECLARED: Final = "litellm.request.tools.declared"
GUARDRAIL_NAME: Final = "litellm.guardrail.name"
GUARDRAIL_MODE: Final = "litellm.guardrail.mode"

22
litellm/integrations/otel/plumbing/context.py
View file

@ -6,6 +6,7 @@ from typing import Final
from opentelemetry import baggage
from opentelemetry.context import Context, get_current
from opentelemetry.sdk.trace import ReadableSpan
from opentelemetry.trace import (
Link,
NonRecordingSpan,
@ -18,6 +19,8 @@ from opentelemetry.trace.propagation.tracecontext import (
TraceContextTextMapPropagator,
)
from litellm.integrations.otel.model.semconv import HTTP
_PROPAGATOR: Final = TraceContextTextMapPropagator()
# The request's root span — the FastAPI-owned SERVER span — captured ONCE when the
@ -55,6 +58,25 @@ def request_root_span() -> "Span | None":
return span if is_recordable_span(span) else None
def request_root_http_route() -> str | None:
"""``http.route`` exactly as the request's root SERVER span reports it.
Read off the span rather than re-derived, so the LLM call span cannot disagree
with its own parent about which endpoint served the request: the template the
instrumentation matched, or the literal path where
``mount._passthrough_span_name_hook`` rewrote it, are already in the attribute.
An MCP call anchors that same server span, so it reports the ``/mcp`` mount
point the instrumentation matched. Attributes stay readable after a span ends,
so this answers just as well from the async logging callback.
None when no server span is anchored, which is the SDK path and any deployment
where the FastAPI instrumentation did not mount.
"""
span: Final = request_root_span()
route: Final = span.attributes.get(HTTP.ROUTE) if isinstance(span, ReadableSpan) and span.attributes else None
return route if isinstance(route, str) and route else None
# The W3C trace-context carrier (``traceparent``/``tracestate``/``baggage``) the
# MCP client propagated in the current request's ``params._meta``. The MCP gateway
# sets it per message so the MCP span can record the client's span as a span

19
litellm/integrations/prometheus.py
View file

@ -10,7 +10,7 @@ import sys
from collections.abc import Awaitable, Callable, Mapping, Sequence
from dataclasses import replace
from datetime import datetime, timedelta
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast
from pydantic import BaseModel
@ -142,6 +142,9 @@ class _ExcludedLabelMetric:
return self._metric.labels(*kept_values) if kept_values else self._metric
_MetricLike: TypeAlias = "NoOpMetric | _ExcludedLabelMetric | MetricWrapperBase"
def _get_budget_metrics_per_request_timeout() -> float:
raw: Final = os.getenv("PROMETHEUS_BUDGET_METRICS_PER_REQUEST_TIMEOUT")
if raw is None:
@ -1652,7 +1655,7 @@ class PrometheusLogger(CustomLogger):
cache_creation_detail_tokens: Final = PrometheusLogger._resolve_cache_write_tokens(prompt_details)
detail_metrics: Final[list[tuple[Any, DEFINED_PROMETHEUS_METRICS, object]]] = [
detail_metrics: Final[list[tuple[_MetricLike, DEFINED_PROMETHEUS_METRICS, object]]] = [
(
self.litellm_input_cached_tokens_metric,
"litellm_input_cached_tokens_metric",
@ -1705,7 +1708,7 @@ class PrometheusLogger(CustomLogger):
if not isinstance(usage_object, dict):
return
media_metrics: Final[list[tuple[Any, DEFINED_PROMETHEUS_METRICS, object]]] = [
media_metrics: Final[list[tuple[_MetricLike, DEFINED_PROMETHEUS_METRICS, object]]] = [
(
self.litellm_video_duration_seconds_metric,
"litellm_video_duration_seconds_metric",
@ -1727,7 +1730,7 @@ class PrometheusLogger(CustomLogger):
def _inc_sparse_usage_counters(
self,
counters_with_values: Sequence[tuple[Any, DEFINED_PROMETHEUS_METRICS, object]],
counters_with_values: Sequence[tuple[_MetricLike, DEFINED_PROMETHEUS_METRICS, object]],
enum_values: UserAPIKeyLabelValues,
label_context: PrometheusLabelFactoryContext | None = None,
) -> None:
@ -2607,7 +2610,7 @@ class PrometheusLogger(CustomLogger):
for all successful requests (both streaming and non-streaming).
"""
def _safe_get(self, obj: Any, key: str, default: object = None) -> Any:
def _safe_get(self, obj: object, key: str, default: object = None) -> Any:
"""Get value from dict or Pydantic model."""
if obj is None:
return default
@ -2623,7 +2626,7 @@ class PrometheusLogger(CustomLogger):
"""
standard_logging_payload: Final = request_kwargs.get("standard_logging_object", {}) or {}
_litellm_params: Final = request_kwargs.get("litellm_params", {}) or {}
_metadata_raw: Final = self._safe_get(standard_logging_payload, "metadata") or {}
_metadata_raw: Final[object] = self._safe_get(standard_logging_payload, "metadata") or {}
if isinstance(_metadata_raw, dict):
_metadata = _metadata_raw
else:
@ -4215,8 +4218,8 @@ class PrometheusLogger(CustomLogger):
def _safe_duration_seconds(
self,
start_time: Any,
end_time: Any,
start_time: object,
end_time: object,
) -> float | None:
"""
Compute the duration in seconds between two objects.

52
litellm/integrations/shadow_eval_logger.py
View file

@ -37,6 +37,7 @@ from litellm.litellm_core_utils.llm_judge import (
)
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
from litellm.llms.base_llm.base_utils import type_to_response_format_param
from litellm.router_utils.common_utils import resolve_model_group_alias
from litellm.types.management_endpoints.auto_router_endpoints import ShadowEvalDirection
from litellm.types.utils import SHADOW_EVAL_JUDGE_CALL_ORIGIN, SHADOW_EVAL_ROUTER_CALL_ORIGIN
@ -60,9 +61,9 @@ _MAX_CONCURRENT_SHADOW_TASKS: Final = 16
_MAX_JUDGE_RESPONSE_CHARS: Final = 8_000
_MAX_JUDGE_PROMPT_CHARS: Final = 24_000
# The judge answers with a small JSON object; a tighter budget truncates the JSON
# mid-object and the attempt is lost to an error row.
JUDGE_MAX_OUTPUT_TOKENS: Final = 1500
# Covers the judge's reasoning tokens as well as its small JSON answer: a judge deployment
# carrying an elevated reasoning_effort spends a tight cap before it ever answers.
JUDGE_MAX_OUTPUT_TOKENS: Final = 4096
_MAX_ERROR_CHARS: Final = 500
@ -418,6 +419,20 @@ def _failure_detail(e: BaseException) -> str:
return f"{type(e).__name__}{location}: {e}"
def _judge_reply_shape(response: object) -> str:
"""How an unparseable judge reply was shaped. The parser's own message cannot separate a
judge that answered with nothing from one truncated mid-object, and those want opposite
fixes. Shape only, never the reply text: the judge quotes the sampled turns it compares,
and no attempt row carries sampled content today."""
read: Final = _chat_message_reader(response)
if read is None:
return "unreadable judge reply"
content: Final = read("content")
served: Final = str(_field_reader(response)("model") or "unknown")
body: Final = f"{len(str(content))} chars" if content else "no content"
return f"finish_reason={_chat_finish_reason(response)}, content={body}, model={served}"
def _call_cost(response: object) -> float:
"""Price one eval-arm call with the figure the spend pipeline bills: the router client
stamps _hidden_params.response_cost from the deployment's own pricing, which the public
@ -650,6 +665,7 @@ class ActiveShadowEvalJob(BaseModel):
id: str
router_name: str
router_names: tuple[str, ...] = ()
models: frozenset[str] = frozenset()
direction: ShadowEvalDirection = "forward"
baseline_model: str | None = None
shadow_percentage: float
@ -692,6 +708,21 @@ class ActiveShadowEvalJob(BaseModel):
return self.baseline_model or arm_router
def _canonical_group(router: "Router | None", model_group: str) -> str:
"""A model group in the one spelling both a job's scope and a request's model compare
under: an alias resolves to its target so the two never fail to match on spelling."""
return (
resolve_model_group_alias(router.model_group_alias, model_group) if router is not None else None
) or model_group
def _scope_admits(router: "Router | None", job: "ActiveShadowEvalJob", model_group: str) -> bool:
"""Whether the request's group is in the job's model scope. Both sides resolve through
the router's alias map at match time, so a re-pointed alias applies to the next request
rather than after the jobs cache rolls."""
return not job.models or any(_canonical_group(router, name) == model_group for name in job.models)
def _as_active_job(record: object, attempts: int, spend: float) -> ActiveShadowEvalJob | None:
"""The sampling path's view of one job row, or None for a row it cannot sample: an
unknown direction, or a reverse job with no baseline model to duplicate against.
@ -714,7 +745,8 @@ class ShadowEvalLogger(CustomLogger):
A job targets a virtual key, a team, or a user; a request qualifies for a job when
any of its resolved identities (key hash, team id, user id) matches the job's
target, so team and user jobs cover JWT-authenticated traffic, which carries no
key hash at all."""
key hash at all. A job scoped to model groups further requires the request's
requested group to be one of them."""
def __init__(
self,
@ -801,19 +833,24 @@ class ShadowEvalLogger(CustomLogger):
active_jobs: Sequence[ActiveShadowEvalJob],
request_metadata: Mapping[str, object],
request_id: str,
model_group: str,
) -> tuple[ActiveShadowEvalJob, ...]:
"""The jobs that sample this request. A key can hold one job per direction, and a
request routed by one job's router while bypassing the other's qualifies for both;
each is separately budgeted, so both fire. An admitting job that loses the sampling
dice is counted, so results can weigh judged rows against the traffic they stand for."""
dice is counted, so results can weigh judged rows against the traffic they stand for.
A request outside a job's direction or model scope is not that job's traffic and
goes uncounted, so the funnel stays a fraction of the traffic the job admits."""
eligible: list[ActiveShadowEvalJob] = [] # mutable-ok: bucketed per-job admission
now: Final = datetime.now(timezone.utc)
router: Final = self._router_provider()
for job in active_jobs:
if (
now >= job.ends_at
or job.attempts + self._job_starts.get(job.id, 0) >= job.max_turns
or (job.max_budget is not None and job.spend >= job.max_budget)
or not _direction_admits(request_metadata, job)
or not _scope_admits(router, job, model_group)
):
continue
if not _sample_hits(request_id, job.id, job.shadow_percentage):
@ -868,6 +905,7 @@ class ShadowEvalLogger(CustomLogger):
tuple(job for target in targets for job in active_jobs.get(target, ())),
request_metadata,
request_id,
_canonical_group(self._router_provider(), str(payload.get("model_group") or "")),
)
if not eligible:
return
@ -1242,7 +1280,9 @@ class ShadowEvalLogger(CustomLogger):
verdict: Final = PairwiseVerdict.model_validate(parse_json_verdict(raw))
except Exception as e: # noqa: BLE001 # malformed verdicts become error rows
verbose_logger.debug("shadow_eval: unparseable judge verdict: %s", e)
return _CallFailure(f"unparseable judge verdict: {e}", cost=_call_cost(response))
return _CallFailure(
f"unparseable judge verdict: {e}; {_judge_reply_shape(response)}", cost=_call_cost(response)
)
return _JudgeVerdict(
preference=_unmask_preference(verdict.preference, real_is_a),
confidence=max(0.0, min(1.0, verdict.confidence)),

5
litellm/integrations/weave/weave_otel.py
View file

@ -3,6 +3,7 @@ from __future__ import annotations
import base64
import json
import os
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
from opentelemetry.trace import Status, StatusCode
@ -59,7 +60,7 @@ class WeaveLLMObsOTELAttributes(BaseLLMObsOTELAttributes):
safe_set_attribute(span, OpenInferenceSpanAttributes.INPUT_VALUE, json.dumps(prompt))
def _set_weave_specific_attributes(span: Span, kwargs: dict[str, Any], response_obj: Any):
def _set_weave_specific_attributes(span: Span, kwargs: Mapping[str, Any], response_obj: Any):
"""
Sets Weave-specific metadata attributes onto the OTEL span.
@ -169,7 +170,7 @@ def get_weave_otel_config() -> WeaveOtelConfig:
)
def set_weave_otel_attributes(span: Span, kwargs: dict[str, Any], response_obj: Any):
def set_weave_otel_attributes(span: Span, kwargs: Mapping[str, object], response_obj: object):
"""
Sets OpenTelemetry span attributes for Weave observability.
Uses the same attribute setting logic as other OTEL integrations for consistency.

11
litellm/integrations/websearch_interception/tools.py
View file

@ -6,12 +6,13 @@ Native provider tools (like Anthropic's web_search_20250305) are converted
to this format for consistent interception and execution.
"""
from collections.abc import Mapping
from typing import Any, Final
from litellm.constants import LITELLM_WEB_SEARCH_TOOL_NAME
def get_litellm_web_search_tool() -> dict[str, Any]:
def get_litellm_web_search_tool() -> dict[str, object]:
"""
Get the standard LiteLLM web search tool definition.
@ -49,7 +50,7 @@ def get_litellm_web_search_tool() -> dict[str, Any]:
}
def get_litellm_web_search_tool_openai() -> dict[str, Any]:
def get_litellm_web_search_tool_openai() -> dict[str, object]:
"""
Get the standard LiteLLM web search tool definition in OpenAI format.
@ -82,7 +83,7 @@ def get_litellm_web_search_tool_openai() -> dict[str, Any]:
}
def get_litellm_web_search_tool_responses() -> dict[str, Any]:
def get_litellm_web_search_tool_responses() -> dict[str, object]:
"""
Get the standard LiteLLM web search tool definition in Responses API format.
@ -114,7 +115,7 @@ def get_litellm_web_search_tool_responses() -> dict[str, Any]:
}
def is_web_search_tool_responses(tool: dict[str, Any]) -> bool:
def is_web_search_tool_responses(tool: Mapping[str, object]) -> bool:
"""
Check if a tool is a web search tool for the Responses API.
@ -195,7 +196,7 @@ def is_web_search_tool_chat_completion(tool: dict[str, Any]) -> bool:
return False
def is_anthropic_native_web_search_tool(tool: dict[str, Any]) -> bool:
def is_anthropic_native_web_search_tool(tool: Mapping[str, object]) -> bool:
"""
Check if a tool is an Anthropic-native ``web_search_*`` tool.

8
litellm/integrations/websearch_interception/transformation.py
View file

@ -24,7 +24,7 @@ class WebSearchTransformation:
@staticmethod
def transform_request(
response: Any,
response: object,
stream: bool,
response_format: str = "anthropic",
) -> tuple[bool, list[dict]]:
@ -66,7 +66,7 @@ class WebSearchTransformation:
@staticmethod
def _detect_from_responses_response(
response: Any,
response: object,
) -> tuple[bool, list[dict]]:
"""Parse a Responses API response for ``litellm_web_search`` function calls.
@ -399,7 +399,7 @@ class WebSearchTransformation:
def build_web_search_tool_result_block(
tool_use_id: str,
search_response: SearchResponse | None,
) -> dict[str, Any]:
) -> dict[str, object]:
"""
Build an Anthropic-native ``web_search_tool_result`` content block.
@ -433,7 +433,7 @@ class WebSearchTransformation:
emitted with an empty result list (signals "search ran, no
results" rather than "search did not run").
"""
items: Final[list[dict[str, Any]]] = []
items: Final[list[dict[str, object]]] = []
if search_response is not None:
results: Final = getattr(search_response, "results", None) or []
for r in results:

16
litellm/interactions/agents/http_handler.py
View file

@ -6,7 +6,7 @@ Extends InteractionsHTTPHandler so that the shared HTTP infrastructure
duplicated. BaseAgentsAPIConfig stays as pure transform code.
"""
from collections.abc import Coroutine
from collections.abc import Coroutine, Mapping
from typing import Any, Final
import httpx
@ -39,11 +39,11 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
litellm_params: GenericLiteLLMParams,
logging_obj: LiteLLMLoggingObj,
extra_headers: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
extra_body: Mapping[str, object] | None = None,
timeout: float | httpx.Timeout | None = None,
client: HTTPHandler | None = None,
_is_async: bool = False,
) -> AgentCreateResponse | Coroutine[Any, Any, AgentCreateResponse]:
) -> AgentCreateResponse | Coroutine[object, object, AgentCreateResponse]:
if _is_async:
return self.async_create_agent(
agents_api_config=agents_api_config,
@ -94,7 +94,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
litellm_params: GenericLiteLLMParams,
logging_obj: LiteLLMLoggingObj,
extra_headers: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
extra_body: Mapping[str, object] | None = None,
timeout: float | httpx.Timeout | None = None,
client: AsyncHTTPHandler | None = None,
) -> AgentCreateResponse:
@ -145,7 +145,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
timeout: float | httpx.Timeout | None = None,
client: HTTPHandler | None = None,
_is_async: bool = False,
) -> AgentListResponse | Coroutine[Any, Any, AgentListResponse]:
) -> AgentListResponse | Coroutine[object, object, AgentListResponse]:
if _is_async:
return self.async_list_agents(
agents_api_config=agents_api_config,
@ -220,7 +220,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
timeout: float | httpx.Timeout | None = None,
client: HTTPHandler | None = None,
_is_async: bool = False,
) -> AgentCreateResponse | Coroutine[Any, Any, AgentCreateResponse]:
) -> AgentCreateResponse | Coroutine[object, object, AgentCreateResponse]:
if _is_async:
return self.async_get_agent(
agents_api_config=agents_api_config,
@ -299,7 +299,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
timeout: float | httpx.Timeout | None = None,
client: HTTPHandler | None = None,
_is_async: bool = False,
) -> AgentDeleteResult | Coroutine[Any, Any, AgentDeleteResult]:
) -> AgentDeleteResult | Coroutine[object, object, AgentDeleteResult]:
if _is_async:
return self.async_delete_agent(
agents_api_config=agents_api_config,
@ -378,7 +378,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
timeout: float | httpx.Timeout | None = None,
client: HTTPHandler | None = None,
_is_async: bool = False,
) -> AgentVersionsResponse | Coroutine[Any, Any, AgentVersionsResponse]:
) -> AgentVersionsResponse | Coroutine[object, object, AgentVersionsResponse]:
if _is_async:
return self.async_list_agent_versions(
agents_api_config=agents_api_config,

18
litellm/interactions/agents/main.py
View file

@ -30,7 +30,7 @@ Usage:
import asyncio
import contextvars
from collections.abc import Coroutine
from collections.abc import Coroutine, Mapping
from functools import partial
from typing import Any, Final
@ -75,7 +75,7 @@ def _make_logging_obj(
model: str,
custom_llm_provider: str,
call_type: str,
optional_params: dict[str, Any],
optional_params: dict[str, object],
) -> LiteLLMLoggingObj:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
@ -102,7 +102,7 @@ async def acreate(
base_environment: InteractionEnvironment | None = None,
custom_llm_provider: str | None = None,
extra_headers: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
extra_body: Mapping[str, object] | None = None,
timeout: float | httpx.Timeout | None = None,
**kwargs,
) -> AgentCreateResponse:
@ -146,10 +146,10 @@ def create(
base_environment: InteractionEnvironment | None = None,
custom_llm_provider: str | None = None,
extra_headers: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
extra_body: Mapping[str, object] | None = None,
timeout: float | httpx.Timeout | None = None,
**kwargs,
) -> AgentCreateResponse | Coroutine[Any, Any, AgentCreateResponse]:
) -> AgentCreateResponse | Coroutine[object, object, AgentCreateResponse]:
"""
Sync: Create a managed agent on the provider side.
@ -244,7 +244,7 @@ def list(
extra_headers: dict[str, Any] | None = None,
timeout: float | httpx.Timeout | None = None,
**kwargs,
) -> AgentListResponse | Coroutine[Any, Any, AgentListResponse]:
) -> AgentListResponse | Coroutine[object, object, AgentListResponse]:
"""Sync: List all agents on the provider side."""
local_vars: Final = locals()
custom_llm_provider = custom_llm_provider or kwargs.get("custom_llm_provider") or "gemini"
@ -320,7 +320,7 @@ def get(
extra_headers: dict[str, Any] | None = None,
timeout: float | httpx.Timeout | None = None,
**kwargs,
) -> AgentCreateResponse | Coroutine[Any, Any, AgentCreateResponse]:
) -> AgentCreateResponse | Coroutine[object, object, AgentCreateResponse]:
"""Sync: Get a specific agent by name."""
local_vars: Final = locals()
custom_llm_provider = custom_llm_provider or kwargs.get("custom_llm_provider") or "gemini"
@ -397,7 +397,7 @@ def delete(
extra_headers: dict[str, Any] | None = None,
timeout: float | httpx.Timeout | None = None,
**kwargs,
) -> AgentDeleteResult | Coroutine[Any, Any, AgentDeleteResult]:
) -> AgentDeleteResult | Coroutine[object, object, AgentDeleteResult]:
"""Sync: Delete a specific agent by name."""
local_vars: Final = locals()
custom_llm_provider = custom_llm_provider or kwargs.get("custom_llm_provider") or "gemini"
@ -474,7 +474,7 @@ def list_versions(
extra_headers: dict[str, Any] | None = None,
timeout: float | httpx.Timeout | None = None,
**kwargs,
) -> AgentVersionsResponse | Coroutine[Any, Any, AgentVersionsResponse]:
) -> AgentVersionsResponse | Coroutine[object, object, AgentVersionsResponse]:
"""Sync: List versions of a specific agent."""
local_vars: Final = locals()
custom_llm_provider = custom_llm_provider or kwargs.get("custom_llm_provider") or "gemini"

30
litellm/interactions/http_handler.py
View file

@ -4,7 +4,7 @@ HTTP Handler for Interactions API requests.
This module handles the HTTP communication for the Google Interactions API.
"""
from collections.abc import AsyncIterator, Coroutine, Iterator
from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping
from typing import Any, Final
import httpx
@ -96,8 +96,8 @@ class InteractionsHTTPHandler(_BaseHTTPHandler):
model: str | None = None,
agent: str | None = None,
input: InteractionInput | None = None,
extra_headers: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
extra_headers: dict[str, str] | None = None,
extra_body: Mapping[str, object] | None = None,
timeout: float | httpx.Timeout | None = None,
client: HTTPHandler | None = None,
_is_async: bool = False,
@ -105,7 +105,7 @@ class InteractionsHTTPHandler(_BaseHTTPHandler):
) -> (
InteractionsAPIResponse
| Iterator[InteractionsAPIStreamingResponse]
| Coroutine[Any, Any, InteractionsAPIResponse | AsyncIterator[InteractionsAPIStreamingResponse]]
| Coroutine[object, object, InteractionsAPIResponse | AsyncIterator[InteractionsAPIStreamingResponse]]
):
"""
Create a new interaction (synchronous or async based on _is_async flag).
@ -211,8 +211,8 @@ class InteractionsHTTPHandler(_BaseHTTPHandler):
model: str | None = None,
agent: str | None = None,
input: InteractionInput | None = None,
extra_headers: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
extra_headers: dict[str, str] | None = None,
extra_body: Mapping[str, object] | None = None,
timeout: float | httpx.Timeout | None = None,
client: AsyncHTTPHandler | None = None,
stream: bool | None = None,
@ -345,11 +345,11 @@ class InteractionsHTTPHandler(_BaseHTTPHandler):
custom_llm_provider: str,
litellm_params: GenericLiteLLMParams,
logging_obj: LiteLLMLoggingObj,
extra_headers: dict[str, Any] | None = None,
extra_headers: dict[str, str] | None = None,
timeout: float | httpx.Timeout | None = None,
client: HTTPHandler | None = None,
_is_async: bool = False,
) -> InteractionsAPIResponse | Coroutine[Any, Any, InteractionsAPIResponse]:
) -> InteractionsAPIResponse | Coroutine[object, object, InteractionsAPIResponse]:
"""Get an interaction by ID."""
if _is_async:
return self.async_get_interaction(
@ -407,7 +407,7 @@ class InteractionsHTTPHandler(_BaseHTTPHandler):
custom_llm_provider: str,
litellm_params: GenericLiteLLMParams,
logging_obj: LiteLLMLoggingObj,
extra_headers: dict[str, Any] | None = None,
extra_headers: dict[str, str] | None = None,
timeout: float | httpx.Timeout | None = None,
client: AsyncHTTPHandler | None = None,
) -> InteractionsAPIResponse:
@ -464,11 +464,11 @@ class InteractionsHTTPHandler(_BaseHTTPHandler):
custom_llm_provider: str,
litellm_params: GenericLiteLLMParams,
logging_obj: LiteLLMLoggingObj,
extra_headers: dict[str, Any] | None = None,
extra_headers: dict[str, str] | None = None,
timeout: float | httpx.Timeout | None = None,
client: HTTPHandler | None = None,
_is_async: bool = False,
) -> DeleteInteractionResult | Coroutine[Any, Any, DeleteInteractionResult]:
) -> DeleteInteractionResult | Coroutine[object, object, DeleteInteractionResult]:
"""Delete an interaction by ID."""
if _is_async:
return self.async_delete_interaction(
@ -527,7 +527,7 @@ class InteractionsHTTPHandler(_BaseHTTPHandler):
custom_llm_provider: str,
litellm_params: GenericLiteLLMParams,
logging_obj: LiteLLMLoggingObj,
extra_headers: dict[str, Any] | None = None,
extra_headers: dict[str, str] | None = None,
timeout: float | httpx.Timeout | None = None,
client: AsyncHTTPHandler | None = None,
) -> DeleteInteractionResult:
@ -585,11 +585,11 @@ class InteractionsHTTPHandler(_BaseHTTPHandler):
custom_llm_provider: str,
litellm_params: GenericLiteLLMParams,
logging_obj: LiteLLMLoggingObj,
extra_headers: dict[str, Any] | None = None,
extra_headers: dict[str, str] | None = None,
timeout: float | httpx.Timeout | None = None,
client: HTTPHandler | None = None,
_is_async: bool = False,
) -> CancelInteractionResult | Coroutine[Any, Any, CancelInteractionResult]:
) -> CancelInteractionResult | Coroutine[object, object, CancelInteractionResult]:
"""Cancel an interaction by ID."""
if _is_async:
return self.async_cancel_interaction(
@ -648,7 +648,7 @@ class InteractionsHTTPHandler(_BaseHTTPHandler):
custom_llm_provider: str,
litellm_params: GenericLiteLLMParams,
logging_obj: LiteLLMLoggingObj,
extra_headers: dict[str, Any] | None = None,
extra_headers: dict[str, str] | None = None,
timeout: float | httpx.Timeout | None = None,
client: AsyncHTTPHandler | None = None,
) -> CancelInteractionResult:

14
litellm/interactions/litellm_responses_transformation/transformation.py
View file

@ -34,8 +34,8 @@ class LiteLLMResponsesInteractionsConfig:
model: str,
input: InteractionInput | None,
optional_params: InteractionsAPIOptionalRequestParams,
**kwargs,
) -> dict[str, Any]:
**kwargs: object,
) -> dict[str, object]:
"""
Transform an Interactions API request to a Responses API request.
@ -45,7 +45,7 @@ class LiteLLMResponsesInteractionsConfig:
- tools -> tools (similar format)
- generation_config -> temperature, top_p, etc.
"""
responses_request: Final[dict[str, Any]] = {
responses_request: Final[dict[str, object]] = {
"model": model,
}
@ -201,15 +201,15 @@ class LiteLLMResponsesInteractionsConfig:
- Extract usage
"""
# Extract text from outputs and build both `outputs` (legacy) and `steps` (new schema).
outputs: Final[list[dict[str, Any]]] = []
steps: Final[list[dict[str, Any]]] = []
outputs: Final[list[dict[str, object]]] = []
steps: Final[list[dict[str, object]]] = []
if hasattr(responses_response, "output") and responses_response.output:
for output_item in responses_response.output:
# Use getattr with None default to safely access content
content = getattr(output_item, "content", None)
if content is not None:
content_items = content if isinstance(content, list) else [content]
model_output_contents: list[dict[str, Any]] = []
model_output_contents: list[dict[str, object]] = []
for content_item in content_items:
# Check if content_item has text attribute
text = getattr(content_item, "text", None)
@ -264,7 +264,7 @@ class LiteLLMResponsesInteractionsConfig:
# Add usage if available
# Map Responses API usage (input_tokens, output_tokens) to Interactions API spec format
# (total_input_tokens, total_output_tokens)
usage: Final = getattr(responses_response, "usage", None)
usage: Final[object] = getattr(responses_response, "usage", None)
if usage:
interactions_response_dict["usage"] = {
"total_input_tokens": getattr(usage, "input_tokens", 0),

8
litellm/interactions/main.py
View file

@ -229,7 +229,7 @@ def create(
) -> (
InteractionsAPIResponse
| Iterator[InteractionsAPIStreamingResponse]
| Coroutine[Any, Any, InteractionsAPIResponse | AsyncIterator[InteractionsAPIStreamingResponse]]
| Coroutine[object, object, InteractionsAPIResponse | AsyncIterator[InteractionsAPIStreamingResponse]]
):
"""
Sync: Create a new interaction using Google's Interactions API.
@ -406,7 +406,7 @@ def get(
timeout: float | httpx.Timeout | None = None,
custom_llm_provider: str | None = None,
**kwargs,
) -> InteractionsAPIResponse | Coroutine[Any, Any, InteractionsAPIResponse]:
) -> InteractionsAPIResponse | Coroutine[object, object, InteractionsAPIResponse]:
"""Sync: Get an interaction by its ID."""
local_vars: Final = locals()
custom_llm_provider = custom_llm_provider or "gemini"
@ -510,7 +510,7 @@ def delete(
timeout: float | httpx.Timeout | None = None,
custom_llm_provider: str | None = None,
**kwargs,
) -> DeleteInteractionResult | Coroutine[Any, Any, DeleteInteractionResult]:
) -> DeleteInteractionResult | Coroutine[object, object, DeleteInteractionResult]:
"""Sync: Delete an interaction by its ID."""
local_vars: Final = locals()
custom_llm_provider = custom_llm_provider or "gemini"
@ -612,7 +612,7 @@ def cancel(
timeout: float | httpx.Timeout | None = None,
custom_llm_provider: str | None = None,
**kwargs,
) -> CancelInteractionResult | Coroutine[Any, Any, CancelInteractionResult]:
) -> CancelInteractionResult | Coroutine[object, object, CancelInteractionResult]:
"""Sync: Cancel an interaction by its ID."""
local_vars: Final = locals()
custom_llm_provider = custom_llm_provider or "gemini"

10
litellm/litellm_core_utils/core_helpers.py
View file

@ -419,7 +419,7 @@ def safe_deep_copy(data):
if litellm.safe_memory_mode is True:
return data
litellm_parent_otel_span: Any | None = None
litellm_parent_otel_span: object | None = None
# Step 1: Remove the litellm_parent_otel_span
litellm_parent_otel_span = None
if isinstance(data, dict):
@ -510,7 +510,7 @@ def independent_snapshot(
}
def filter_exceptions_from_params(data: Any, max_depth: int = 20) -> Any:
def filter_exceptions_from_params(data: object, max_depth: int = 20) -> Any:
"""
Recursively filter out Exception objects and callable objects from dicts/lists.
@ -542,7 +542,7 @@ def filter_exceptions_from_params(data: Any, max_depth: int = 20) -> Any:
return None
if isinstance(data, dict):
result: Final[dict[str, Any]] = {}
result: Final[dict[str, object]] = {}
for k, v in data.items():
# Skip exception and callable values
if isinstance(v, Exception) or (callable(v) and not isinstance(v, type)):
@ -556,7 +556,7 @@ def filter_exceptions_from_params(data: Any, max_depth: int = 20) -> Any:
continue
return result
elif isinstance(data, list):
result_list: Final[list[Any]] = []
result_list: Final[list[object]] = []
for item in data:
# Skip exception and callable items
if isinstance(item, Exception) or (callable(item) and not isinstance(item, type)):
@ -624,7 +624,7 @@ def redact_nested_match_and_regex_keys(
# Iterative traversal; `seen` guards against cyclic refs preserved by deepcopy.
try:
seen: Final[set] = set()
stack: Final[list[Any]] = [redacted]
stack: Final[list[object]] = [redacted]
while stack:
node = stack.pop()
node_id = id(node)

17
litellm/litellm_core_utils/dot_notation_indexing.py
View file

@ -23,12 +23,13 @@ Used by JWT Auth to get the user role from the token, and by
additional_drop_params to remove nested fields from optional parameters.
"""
from collections.abc import Mapping
from typing import Any, Final, TypeVar
T = TypeVar("T")
def get_nested_value(data: dict[str, Any], key_path: str, default: T | None = None) -> T | None:
def get_nested_value(data: Mapping[str, object], key_path: str, default: T | None = None) -> T | None:
"""
Retrieves a value from a nested dictionary using dot notation.
@ -107,7 +108,7 @@ def _parse_path_segments(path: str) -> list:
def _delete_nested_value_custom(
data: dict[str, Any] | list[Any],
data: dict[str, object] | list[object],
segments: list,
segment_index: int = 0,
) -> None:
@ -168,13 +169,15 @@ def _delete_nested_value_custom(
if segment in data:
next_segment: Final = segments[segment_index + 1] if segment_index + 1 < len(segments) else None
child: Final = data[segment]
# If next segment is array notation, current field should be list
if next_segment and (next_segment.startswith("[")):
if isinstance(data[segment], list):
_delete_nested_value_custom(data[segment], segments, segment_index + 1)
if isinstance(child, list):
_delete_nested_value_custom(child, segments, segment_index + 1)
# Otherwise navigate into dict
elif isinstance(data[segment], dict):
_delete_nested_value_custom(data[segment], segments, segment_index + 1)
elif isinstance(child, dict):
_delete_nested_value_custom(child, segments, segment_index + 1)
def delete_nested_value(
@ -182,7 +185,7 @@ def delete_nested_value(
path: str,
depth: int = 0,
max_depth: int = 20,
) -> dict[str, Any]:
) -> dict[str, object]:
"""
Delete a field from nested data using JSONPath notation.

6
litellm/litellm_core_utils/json_validation_rule.py
View file

@ -5,10 +5,10 @@ from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
def normalize_json_schema_types(
schema: dict[str, Any] | list[Any] | Any,
schema: object,
depth: int = 0,
max_depth: int = DEFAULT_MAX_RECURSE_DEPTH,
) -> dict[str, Any] | list[Any] | Any:
) -> object:
"""
Normalize JSON schema types from uppercase to lowercase format.
@ -47,7 +47,7 @@ def normalize_json_schema_types(
return [normalize_json_schema_types(item, depth + 1, max_depth) for item in schema]
if isinstance(schema, dict):
normalized_schema: Final[dict[str, Any]] = {}
normalized_schema: Final[dict[str, object]] = {}
for key, value in schema.items():
if key == "type" and isinstance(value, str) and value in type_mapping:

42
litellm/litellm_core_utils/litellm_logging.py
View file

@ -78,7 +78,10 @@ from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
InteractionsUsageObjectTransformation,
)
from litellm.litellm_core_utils.logging_utils import truncate_base64_in_messages
from litellm.litellm_core_utils.logging_utils import (
truncate_base64_in_messages,
truncate_base64_in_messages_async,
)
from litellm.litellm_core_utils.model_param_helper import ModelParamHelper
from litellm.litellm_core_utils.redact_messages import (
redact_message_input_output_from_custom_logger,
@ -538,6 +541,7 @@ class Logging(LiteLLMLoggingBaseClass):
self.standard_built_in_tools_params: StandardBuiltInToolsParams = (
self.initialize_standard_built_in_tools_params(kwargs)
)
self.truncated_messages_for_logging: str | list | dict | None = None # mutable-ok: logged messages shape
## TIME TO FIRST TOKEN LOGGING ##
self.completion_start_time: datetime.datetime | None = None
self._llm_caching_handler: LLMCachingHandler | None = None
@ -1820,6 +1824,7 @@ class Logging(LiteLLMLoggingBaseClass):
and litellm_params.get(CallTypes.aanthropic_messages.value, False) is not True
and litellm_params.get(CallTypes.agenerate_content.value, False) is not True
and litellm_params.get(CallTypes.agenerate_content_stream.value, False) is not True
and litellm_params.get(CallTypes.arealtime.value, False) is not True
)
def _is_assembled_stream_success(self, result=None) -> bool:
@ -1913,7 +1918,9 @@ class Logging(LiteLLMLoggingBaseClass):
two paths cannot mutate it at the same time. ``prefer_async_handlers`` only
bypasses the sync-SDK-only shortcut (e.g. ``async for`` on a stream from
``completion()``); legacy string callbacks still run via
``executor.submit(failure_handler)`` when configured.
``executor.submit(failure_handler)`` when configured, and still get submitted
when the awaiting task is cancelled (e.g. the event loop shuts down right after
the request failed).
"""
litellm_params: Final = self.model_call_details.get("litellm_params", {}) or {}
sync_sdk: Final = self._is_sync_litellm_request(litellm_params)
@ -1922,12 +1929,11 @@ class Logging(LiteLLMLoggingBaseClass):
self.failure_handler(exception, traceback_exception)
return
await self.async_failure_handler(exception, traceback_exception)
if not self._should_run_sync_failure_callbacks_for_async_calls():
return
executor.submit(self.failure_handler, exception, traceback_exception)
try:
await self.async_failure_handler(exception, traceback_exception)
finally:
if self._should_run_sync_failure_callbacks_for_async_calls():
executor.submit(self.failure_handler, exception, traceback_exception)
def should_run_logging(
self,
@ -2932,6 +2938,11 @@ class Logging(LiteLLMLoggingBaseClass):
result._hidden_params["batch_failed_requests"] = batch_result.failed_requests # pyright: ignore[reportPrivateUsage] # rebind-ok: same pattern as above
result.usage = batch_result.usage
self.truncated_messages_for_logging = await truncate_base64_in_messages_async(
StandardLoggingPayloadSetup.append_system_prompt_messages(
kwargs=self.model_call_details, messages=self.model_call_details.get("messages")
)
)
start_time, end_time, result = self._success_handler_helper_fn(
start_time=start_time,
end_time=end_time,
@ -3224,8 +3235,7 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details = {}
if (
self.model_call_details.get("log_event_type") == "failed_api_call"
and self.model_call_details.get("exception") is exception
self.model_call_details.get("exception") is exception
and self.model_call_details.get("standard_logging_object") is not None
):
return start_time, self.model_call_details["end_time"]
@ -3821,7 +3831,7 @@ class Logging(LiteLLMLoggingBaseClass):
def record_streamed_anthropic_message_id(self, message_id: str) -> None:
self.streamed_anthropic_message_id = message_id
def _anthropic_messages_logged_response(self, result: Any) -> ModelResponse:
def _anthropic_messages_logged_response(self, result: object) -> ModelResponse:
"""
The ModelResponse a /v1/messages spend_logs row is built from.
@ -6201,9 +6211,13 @@ def get_standard_logging_object_payload(
model_id=_model_id,
requester_ip_address=clean_metadata.get("requester_ip_address", None),
user_agent=clean_metadata.get("user_agent", None),
messages=truncate_base64_in_messages(
StandardLoggingPayloadSetup.append_system_prompt_messages(
kwargs=kwargs, messages=kwargs.get("messages")
messages=(
logging_obj.truncated_messages_for_logging
if logging_obj.truncated_messages_for_logging is not None
else truncate_base64_in_messages(
StandardLoggingPayloadSetup.append_system_prompt_messages(
kwargs=kwargs, messages=kwargs.get("messages")
)
)
),
response=final_response_obj,

2
litellm/litellm_core_utils/llm_response_utils/response_metadata.py
View file

@ -168,7 +168,7 @@ class ResponseMetadata:
def update_response_metadata(
result: Any,
result: object,
logging_obj: LiteLLMLoggingObject,
model: str | None,
kwargs: dict,

42
litellm/litellm_core_utils/logging_utils.py
View file

@ -3,12 +3,15 @@ import functools
import inspect
import re
import time
from collections.abc import Mapping
from collections.abc import Iterator, Mapping, Sequence
from datetime import datetime
from typing import TYPE_CHECKING, Any, Final
from litellm._logging import verbose_logger
from litellm.constants import MAX_BASE64_LENGTH_FOR_LOGGING
from litellm.constants import (
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS,
MAX_BASE64_LENGTH_FOR_LOGGING,
)
from litellm.types.utils import (
ModelResponse,
ModelResponseStream,
@ -141,6 +144,39 @@ def truncate_base64_in_messages(
return messages
_StringTree = str | Sequence["_StringTree"] | Mapping[str, "_StringTree"] | None
def _iter_string_leaves(value: _StringTree) -> Iterator[str]:
stack: Final[list[_StringTree]] = [value] # mutable-ok: explicit stack, recursive functions are banned in litellm/
while stack:
match stack.pop():
case str() as text:
yield text
case Mapping() as mapping:
stack.extend(mapping.values())
case Sequence() as items:
stack.extend(items)
case None:
pass
async def truncate_base64_in_messages_async(
messages: str | list | dict | None, # mutable-ok: same contract as truncate_base64_in_messages
) -> str | list | dict | None: # mutable-ok: same contract as truncate_base64_in_messages
"""
Same result as truncate_base64_in_messages, but payloads whose string content
reaches BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS are scanned in a worker
thread so the regex pass over multi-MB base64 images does not block the event loop.
"""
if messages is None or MAX_BASE64_LENGTH_FOR_LOGGING <= 0:
return messages
total_chars: Final = sum(len(leaf) for leaf in _iter_string_leaves(messages))
if total_chars < BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS:
return truncate_base64_in_messages(messages)
return await asyncio.to_thread(truncate_base64_in_messages, messages)
# Global service logger instance to avoid recreating it
_service_logger = None
@ -184,7 +220,7 @@ def _get_parent_otel_span_from_logging_obj(
def convert_litellm_response_object_to_str(
response_obj: Any | LiteLLMModelResponse,
response_obj: object,
) -> str | None:
"""
Get the string of the response object from LiteLLM

6
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -1708,8 +1708,8 @@ def _find_server_tool_result(
def convert_to_anthropic_tool_invoke(
tool_calls: list[ChatCompletionAssistantToolCall],
web_search_results: list[Any] | None = None,
tool_results: list[Any] | None = None,
web_search_results: Sequence[object] | None = None,
tool_results: Sequence[object] | None = None,
) -> list[AnthropicMessagesToolUseParam | dict[str, Any]]:
"""
OpenAI tool invokes:
@ -5349,7 +5349,7 @@ class NormalizedToolCall(TypedDict):
arguments: dict[str, object]
def _parse_tool_call_arguments(raw: Any, tool_name: str | None, context: str) -> dict[str, object]:
def _parse_tool_call_arguments(raw: object, tool_name: str | None, context: str) -> dict[str, object]:
# Anthropic's tool_use blocks already carry a parsed dict in "input";
# chat completions and the Responses API carry a JSON string that may be
# truncated by the model, so route those through the repair-aware parser.

8
litellm/litellm_core_utils/realtime_errors.py
View file

@ -29,3 +29,11 @@ def websocket_close_reason(message: str, fallback: str) -> str:
if len(encoded) <= WEBSOCKET_CLOSE_REASON_MAX_BYTES:
return message
return encoded[:WEBSOCKET_CLOSE_REASON_MAX_BYTES].decode("utf-8", errors="ignore")
def client_close_code(upstream_code: int) -> int:
from websockets.frames import EXTERNAL_CLOSE_CODES, CloseCode
if upstream_code in EXTERNAL_CLOSE_CODES or 3000 <= upstream_code < 5000:
return upstream_code
return int(CloseCode.INTERNAL_ERROR)

220
litellm/litellm_core_utils/realtime_streaming.py
View file

@ -1,12 +1,15 @@
import asyncio
import json
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, Protocol, TypedDict, cast
import traceback
from collections.abc import Coroutine, Mapping, Sequence
from dataclasses import dataclass
from enum import Enum, auto
from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypedDict, cast
from typing_extensions import ReadOnly
import litellm
from litellm._logging import verbose_logger
from litellm._logging import redact_internal_details_from_client_message, verbose_logger
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
from litellm.types.llms.openai import (
@ -19,9 +22,11 @@ from litellm.types.llms.openai import (
from litellm.types.realtime import ALL_DELTA_TYPES
from .litellm_logging import Logging as LiteLLMLogging
from .realtime_errors import client_close_code, realtime_error_event, websocket_close_reason
if TYPE_CHECKING:
from websockets.asyncio.client import ClientConnection
from websockets.exceptions import ConnectionClosed
from litellm.types.guardrails import GuardrailEventHooks
@ -30,8 +35,30 @@ else:
CLIENT_CONNECTION_CLASS = Any
class _ClientWebSocketExceptions(Protocol):
ConnectionClosed: type[Exception]
REALTIME_SESSION_SUCCESS_LOGGED_KEY: Final = "realtime_session_success_logged"
@dataclass(frozen=True, slots=True)
class BackendClose:
code: int
reason: str
@property
def message(self) -> str:
if not self.reason:
return f"upstream websocket closed with code {self.code}"
return f"upstream websocket closed with code {self.code}: {self.reason}"
class ClientLoopExit(Enum):
CLIENT_DISCONNECTED = auto()
BACKEND_CLOSED = auto()
def backend_close_from(error: "ConnectionClosed") -> BackendClose:
if error.rcvd is None:
return BackendClose(code=1006, reason=str(error))
return BackendClose(code=error.rcvd.code, reason=error.rcvd.reason)
class _ASGIScope(TypedDict, total=False):
@ -69,10 +96,13 @@ class _ScopedWebSocket(Protocol):
class _ClientWebSocket(_ScopedWebSocket, Protocol):
exceptions: _ClientWebSocketExceptions
async def send_text(self, data: str) -> None: ...
async def receive_text(self) -> str: ...
async def close(self, code: int = 1000, reason: str | None = None) -> None: ...
class _LoggingWorker(Protocol):
def ensure_initialized_and_enqueue(self, async_coroutine: Coroutine[object, object, None]) -> None: ...
def _decode_json_object(payload: str) -> Mapping[str, object]:
@ -108,11 +138,14 @@ class RealTimeStreaming:
backend_uses_beta_protocol: bool | None = None,
force_transcription_model: str | None = None,
event_normalizer: RealtimeEventNormalizer | None = None,
logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER,
):
self.websocket: _ClientWebSocket = websocket
self.backend_ws = backend_ws
self.logging_obj = logging_obj
self._logging_worker = logging_worker
self.messages: list[OpenAIRealtimeEvents] = []
self._backend_sent_frames: bool = False
self.input_message: dict = {}
self.input_messages: list[dict[str, str]] = []
self.session_tools: list[dict] = []
@ -388,9 +421,10 @@ class RealTimeStreaming:
# Route through the bounded logging worker (per-coroutine timeout +
# concurrency cap) instead of a bare create_task, so a slow callback
# can't leave suspended tasks pinning each call's response in memory.
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
self._logging_worker.ensure_initialized_and_enqueue(
self.logging_obj.dispatch_success_handlers(self.messages, prefer_async_handlers=True)
)
self.logging_obj.model_call_details[REALTIME_SESSION_SUCCESS_LOGGED_KEY] = True
async def _send_to_backend(self, message: str) -> bool:
"""Send a message to the backend WebSocket.
@ -1035,60 +1069,84 @@ class RealTimeStreaming:
return True
return False
async def backend_to_client_send_messages(self):
async def _relay_backend_messages(self) -> NoReturn:
while True:
try:
raw_response = await self.backend_ws.recv(decode=False)
except TypeError:
raw_response = await self.backend_ws.recv()
self._backend_sent_frames = True
if isinstance(raw_response, bytes):
try:
raw_response = raw_response.decode("utf-8")
except UnicodeDecodeError:
verbose_logger.warning("Received non-UTF-8 binary frame from backend, skipping.")
continue
if self.provider_config:
try:
await self._handle_provider_config_message(raw_response)
except Exception as e:
verbose_logger.exception("Error processing backend message, skipping: %s", e)
continue
else:
event = self._parse_backend_event(raw_response)
if event is None:
await self.websocket.send_text(raw_response)
continue
if self._should_drop_event_from_client(event):
continue
if await self._handle_raw_backend_message(event, raw_response):
continue
event = self._normalize_event_for_ga_client(event)
self.store_message(event)
if not self._client_wants_beta:
await self.websocket.send_text(json.dumps(event))
continue
translated = self._translate_event_to_beta(event)
if translated is None:
continue
await self.websocket.send_text(json.dumps(translated))
async def backend_to_client_send_messages(self) -> BackendClose:
import websockets
try:
while True:
try:
raw_response = await self.backend_ws.recv(decode=False)
except TypeError:
raw_response = await self.backend_ws.recv()
if isinstance(raw_response, bytes):
try:
raw_response = raw_response.decode("utf-8")
except UnicodeDecodeError:
verbose_logger.warning("Received non-UTF-8 binary frame from backend, skipping.")
continue
if self.provider_config:
try:
await self._handle_provider_config_message(raw_response)
except Exception as e:
verbose_logger.exception("Error processing backend message, skipping: %s", e)
continue
else:
event = self._parse_backend_event(raw_response)
if event is None:
await self.websocket.send_text(raw_response)
continue
if self._should_drop_event_from_client(event):
continue
if await self._handle_raw_backend_message(event, raw_response):
continue
event = self._normalize_event_for_ga_client(event)
self.store_message(event)
if not self._client_wants_beta:
await self.websocket.send_text(json.dumps(event))
continue
translated = self._translate_event_to_beta(event)
if translated is None:
continue
await self.websocket.send_text(json.dumps(translated))
await self._relay_backend_messages()
except websockets.exceptions.ConnectionClosed as e:
verbose_logger.exception("Connection closed in backend to client send messages - %s", e)
except Exception as e:
verbose_logger.exception("Error in backend to client send messages: %s", e)
finally:
close: Final = backend_close_from(e)
self._flush_unbilled_transcription_usage()
if self._backend_refused_session(close):
await self.log_backend_refusal(e)
else:
await self.log_messages()
return close
except asyncio.CancelledError:
self._flush_unbilled_transcription_usage()
await self.log_messages()
raise
except Exception as e:
verbose_logger.exception("Error in backend to client send messages: %s", e)
self._flush_unbilled_transcription_usage()
await self.log_messages()
return BackendClose(code=1011, reason="proxy failed while relaying the upstream websocket")
def _backend_refused_session(self, close: BackendClose) -> bool:
return close.code != 1000 and not self._backend_sent_frames
async def log_backend_refusal(self, error: Exception) -> None:
if not self.logging_obj:
return
self._logging_worker.ensure_initialized_and_enqueue(
self.logging_obj.dispatch_failure_handlers(error, traceback.format_exc(), prefer_async_handlers=True)
)
@staticmethod
def _detect_beta_header(websocket: _ScopedWebSocket) -> bool:
@ -1243,11 +1301,22 @@ class RealTimeStreaming:
item["content"] = new_content
return item
async def client_ack_messages(self):
async def _receive_client_message(self) -> str | None:
try:
return await self.websocket.receive_text()
except Exception as e: # noqa: BLE001 # whatever the client socket raises, the client is gone
verbose_logger.debug("Client disconnected: %s", e)
return None
async def client_ack_messages(self) -> ClientLoopExit:
import websockets
client_event: _ClientEventFrame
try:
while True:
message = await self.websocket.receive_text()
message = await self._receive_client_message()
if message is None:
return ClientLoopExit.CLIENT_DISCONNECTED
## GUARDRAIL: intercept conversation.item.create for text-based injection.
guardrail_turn_detection_injected = False
@ -1481,23 +1550,38 @@ class RealTimeStreaming:
if guardrail_turn_detection_injected and sent:
self._guardrail_turn_detection_update_sent = True
except websockets.exceptions.ConnectionClosed as e:
verbose_logger.debug("Backend closed while forwarding a client message: %s", e)
return ClientLoopExit.BACKEND_CLOSED
except Exception as e:
verbose_logger.debug("Error in client ack messages: %s", e)
return ClientLoopExit.CLIENT_DISCONNECTED
async def bidirectional_forward(self):
async def bidirectional_forward(self) -> None:
forward_task: Final = asyncio.create_task(self.backend_to_client_send_messages())
client_task: Final = asyncio.create_task(self.client_ack_messages())
try:
await self.client_ack_messages()
except self.websocket.exceptions.ConnectionClosed:
verbose_logger.debug("Connection closed")
forward_task.cancel()
await asyncio.wait((forward_task, client_task), return_when=asyncio.FIRST_COMPLETED)
if client_task.done() and client_task.result() is ClientLoopExit.CLIENT_DISCONNECTED:
return
await self._close_client(await forward_task)
finally:
if not forward_task.done():
forward_task.cancel()
try:
await forward_task
except asyncio.CancelledError:
pass
forward_task.cancel()
client_task.cancel()
await asyncio.gather(forward_task, client_task, return_exceptions=True)
async def _close_client(self, close: BackendClose) -> None:
redacted_message: Final = redact_internal_details_from_client_message(close.message)
redacted_reason: Final = redact_internal_details_from_client_message(close.reason)
try:
if close.code != 1000:
await self.websocket.send_text(realtime_error_event(redacted_message, error_type="server_error"))
await self.websocket.close(
code=client_close_code(close.code),
reason=websocket_close_reason(redacted_reason, fallback=redacted_message),
)
except Exception as e: # noqa: BLE001 # the client may already be gone; the session is over either way
verbose_logger.debug("Could not relay the upstream close to the client: %s", e)
def client_sent_openai_beta_realtime_header(websocket: _ScopedWebSocket) -> bool:

2
litellm/litellm_core_utils/safe_json_dumps.py
View file

@ -30,7 +30,7 @@ def safe_dumps(
def _transform(key: str | None, value: str) -> str:
return value if value_transform is None else value_transform(key, value)
def _serialize(obj: Any, seen: set, depth: int, key: str | None = None) -> Any:
def _serialize(obj: object, seen: set[int], depth: int, key: str | None = None) -> Any:
# Check for maximum depth.
if depth > max_depth:
return "MaxDepthExceeded"

3
litellm/llms/a2a/common_utils.py
View file

@ -2,6 +2,7 @@
Common utilities for A2A (Agent-to-Agent) Protocol
"""
from collections.abc import Mapping
from typing import Any, Final
from pydantic import BaseModel
@ -91,7 +92,7 @@ def extract_text_from_a2a_message(message: dict[str, Any], depth: int = 0, max_d
return " ".join(text_parts)
def extract_text_from_a2a_response(response_dict: dict[str, Any], max_depth: int = 10) -> str:
def extract_text_from_a2a_response(response_dict: Mapping[str, object], max_depth: int = 10) -> str:
"""
Extract text content from A2A response result.

8
litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py
View file

@ -4,7 +4,7 @@ import copy
import json
import traceback
from collections import deque
from collections.abc import AsyncIterator, Iterator, Sequence
from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
from typing import (
TYPE_CHECKING,
Any,
@ -423,7 +423,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
augmented["usage"] = augmented_usage
return augmented
def _next_compaction_event(self) -> dict[str, Any] | None:
def _next_compaction_event(self) -> dict[str, object] | None:
"""Return the next compaction content-block SSE event, or ``None``.
Anthropic delivers compaction as a single delta (no token-by-token
@ -462,7 +462,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
"delta": {"type": "compaction_delta", "content": summary_content},
}
stop_event: Final = {
stop_event: Final[dict[str, object]] = {
"type": "content_block_stop",
"index": compaction_index,
}
@ -994,7 +994,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
self.current_content_block_index += 1
@staticmethod
def _delta_has_content(processed_chunk: dict[str, Any]) -> bool:
def _delta_has_content(processed_chunk: Mapping[str, object]) -> bool:
"""Return True if a translated chunk carries a non-empty
``content_block_delta`` payload.

32
litellm/llms/anthropic/experimental_pass_through/messages/fake_stream_iterator.py
View file

@ -9,6 +9,7 @@ the LLM doesn't make a tool call, and we need to return a stream to the user.
"""
import json
from collections.abc import Mapping
from typing import Any, Final, cast
from litellm.types.llms.anthropic_messages.anthropic_response import (
@ -38,7 +39,7 @@ class FakeAnthropicMessagesStreamIterator:
self.chunks = self._create_streaming_chunks()
self.current_index = 0
def _create_content_block_chunks(self, block_dict: dict[str, Any], index: int) -> list[bytes]:
def _create_content_block_chunks(self, block_dict: Mapping[str, object], index: int) -> list[bytes]:
"""Build SSE chunks for a single content block."""
chunks: Final = []
block_type: Final = block_dict.get("type")
@ -133,14 +134,14 @@ class FakeAnthropicMessagesStreamIterator:
response_dict: Final = cast(dict[str, Any], self.response)
# 1. message_start event
usage: Final = response_dict.get("usage", {})
usage: Final = self.response.get("usage")
message_start: Final = {
"type": "message_start",
"message": {
"id": response_dict.get("id"),
"id": self.response.get("id"),
"type": "message",
"role": response_dict.get("role", "assistant"),
"model": response_dict.get("model"),
"role": self.response.get("role", "assistant"),
"model": self.response.get("model"),
"content": [],
"stop_reason": None,
"stop_sequence": None,
@ -161,21 +162,24 @@ class FakeAnthropicMessagesStreamIterator:
# 5. message_delta event (with final usage and stop_reason)
# Include cache usage fields so clients that only read message_delta
# (like Claude Code's SDK) see the full input token breakdown.
delta_usage: Final[dict[str, Any]] = {
delta_usage: Final[dict[str, int]] = {
"output_tokens": usage.get("output_tokens", 0) if usage else 0,
}
if usage:
if usage.get("input_tokens") is not None:
delta_usage["input_tokens"] = usage["input_tokens"]
if usage.get("cache_creation_input_tokens") is not None:
delta_usage["cache_creation_input_tokens"] = usage["cache_creation_input_tokens"]
if usage.get("cache_read_input_tokens") is not None:
delta_usage["cache_read_input_tokens"] = usage["cache_read_input_tokens"]
input_tokens: Final = usage.get("input_tokens")
if input_tokens is not None:
delta_usage["input_tokens"] = input_tokens
cache_creation_input_tokens: Final = usage.get("cache_creation_input_tokens")
if cache_creation_input_tokens is not None:
delta_usage["cache_creation_input_tokens"] = cache_creation_input_tokens
cache_read_input_tokens: Final = usage.get("cache_read_input_tokens")
if cache_read_input_tokens is not None:
delta_usage["cache_read_input_tokens"] = cache_read_input_tokens
message_delta: Final = {
"type": "message_delta",
"delta": {
"stop_reason": response_dict.get("stop_reason"),
"stop_sequence": response_dict.get("stop_sequence"),
"stop_reason": self.response.get("stop_reason"),
"stop_sequence": self.response.get("stop_sequence"),
},
"usage": delta_usage,
}

10
litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py
View file

@ -266,7 +266,7 @@ def _make_synthetic_advisor_tool() -> dict:
}
def _find_advisor_tool_use(response: Any) -> dict | None:
def _find_advisor_tool_use(response: object) -> dict | None:
"""Return the first tool_use block with name='advisor', or None."""
content: Final = response.get("content") if isinstance(response, dict) else []
if not isinstance(content, list):
@ -277,7 +277,7 @@ def _find_advisor_tool_use(response: Any) -> dict | None:
return None
def _extract_response_text(response: Any) -> str:
def _extract_response_text(response: object) -> str:
"""Extract concatenated text from all text blocks in a response."""
content: Final = response.get("content") if isinstance(response, dict) else []
if not isinstance(content, list):
@ -291,7 +291,7 @@ _PROVIDER_SPECIFIC_KEYS: Final = frozenset({"provider_specific_fields"})
def _build_advisor_context(
messages: list[dict],
executor_response: Any,
executor_response: object,
advisor_use_block: dict,
) -> list[dict]:
"""
@ -327,7 +327,7 @@ def _build_advisor_context(
def _inject_advisor_turn(
messages: list[dict],
executor_response: Any,
executor_response: object,
advisor_use_block: dict,
advisor_text: str,
) -> list[dict]:
@ -355,7 +355,7 @@ def _inject_advisor_turn(
def _inject_max_uses_error(
messages: list[dict],
executor_response: Any,
executor_response: object,
advisor_use_block: dict,
) -> list[dict]:
"""

6
litellm/llms/anthropic/experimental_pass_through/messages/transformation.py
View file

@ -82,7 +82,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
Processes both `system` and `messages` content blocks.
"""
def _sanitize(cache_control: Any) -> None:
def _sanitize(cache_control: object) -> None:
if isinstance(cache_control, dict):
cache_control.pop("scope", None)
@ -147,7 +147,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
return system_param
@staticmethod
def _as_system_content_blocks(value: Any) -> list:
def _as_system_content_blocks(value: object) -> list:
if value is None:
return []
if isinstance(value, list):
@ -157,7 +157,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
return [value]
@staticmethod
def _is_system_role_message(message: Any) -> bool:
def _is_system_role_message(message: object) -> bool:
return isinstance(message, dict) and message.get("role") == "system"
_CONVERTED_SYSTEM_NOTE: Final = (

10
litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py
View file

@ -48,9 +48,9 @@ class AnthropicResponsesStreamWrapper:
self._pending_tool_ids: dict[str, str] = {} # item_id -> call_id / name accumulator
self._sent_message_start = False
self._sent_message_stop = False
self._chunk_queue: deque = deque()
self._chunk_queue: deque[dict[str, object]] = deque()
def _make_message_start(self) -> dict[str, Any]:
def _make_message_start(self) -> dict[str, object]:
return {
"type": "message_start",
"message": {
@ -74,7 +74,7 @@ class AnthropicResponsesStreamWrapper:
self._current_block_index += 1
return self._current_block_index
def _open_block(self, item_id: str | None, content_block: Mapping[str, Any]) -> int:
def _open_block(self, item_id: str | None, content_block: Mapping[str, object]) -> int:
block_idx = self._next_block_index()
if item_id:
self._item_id_to_block_index[item_id] = block_idx
@ -87,7 +87,7 @@ class AnthropicResponsesStreamWrapper:
)
return block_idx
def _process_event(self, event: Any) -> None:
def _process_event(self, event: object) -> None:
"""Convert one Responses API event into zero or more Anthropic chunks queued for emission."""
event_type = getattr(event, "type", None)
if event_type is None and isinstance(event, dict):
@ -253,7 +253,7 @@ class AnthropicResponsesStreamWrapper:
def __aiter__(self) -> "AnthropicResponsesStreamWrapper":
return self
async def __anext__(self) -> dict[str, Any]:
async def __anext__(self) -> dict[str, object]:
# Return any queued chunks first
if self._chunk_queue:
return self._chunk_queue.popleft()

4
litellm/llms/anthropic/files/transformation.py
View file

@ -14,7 +14,7 @@ Anthropic Files API endpoints:
import calendar
import time
from typing import Any, Final, cast
from typing import Final, cast
import httpx
from openai.types.file_deleted import FileDeleted
@ -226,7 +226,7 @@ class AnthropicFilesConfig(BaseFilesConfig):
) -> tuple[str, dict]:
api_base: Final = AnthropicModelInfo.get_api_base(litellm_params.get("api_base")) or ANTHROPIC_FILES_API_BASE
url: Final = f"{api_base.rstrip('/')}/v1/files"
params: Final[dict[str, Any]] = {}
params: Final[dict[str, str]] = {}
if purpose:
params["purpose"] = purpose
return url, params

13
litellm/llms/aws_polly/text_to_speech/transformation.py
View file

@ -20,6 +20,7 @@ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.types.llms.openai import HttpxBinaryResponseContent
else:
LiteLLMLoggingObj = Any
@ -75,15 +76,15 @@ class AWSPollyTextToSpeechConfig(BaseTextToSpeechConfig, BaseAWSLLM):
litellm_params_dict: dict,
logging_obj: "LiteLLMLoggingObj",
timeout: float | httpx.Timeout,
extra_headers: dict[str, Any] | None,
base_llm_http_handler: Any,
extra_headers: dict[str, object] | None,
base_llm_http_handler: "BaseLLMHTTPHandler",
aspeech: bool,
api_base: str | None,
api_key: str | None,
**kwargs: Any,
**kwargs: object,
) -> Union[
"HttpxBinaryResponseContent",
Coroutine[Any, Any, "HttpxBinaryResponseContent"],
Coroutine[object, object, "HttpxBinaryResponseContent"],
]:
"""
Dispatch method to handle AWS Polly TTS requests
@ -251,7 +252,7 @@ class AWSPollyTextToSpeechConfig(BaseTextToSpeechConfig, BaseAWSLLM):
def _sign_polly_request(
self,
request_body: dict[str, Any],
request_body: dict[str, object],
endpoint_url: str,
litellm_params: dict,
) -> tuple[dict[str, str], str]:
@ -337,7 +338,7 @@ class AWSPollyTextToSpeechConfig(BaseTextToSpeechConfig, BaseAWSLLM):
engine: Final = optional_params.get("engine", self.DEFAULT_ENGINE)
# Build request body
request_body: Final[dict[str, Any]] = {
request_body: Final[dict[str, object]] = {
"Engine": engine,
"OutputFormat": output_format,
"Text": input,

14
litellm/llms/azure/realtime/handler.py
View file

@ -6,7 +6,7 @@ This requires websockets, and is currently only supported on LiteLLM Proxy.
from collections.abc import Mapping
from types import MappingProxyType
from typing import Any, Final, cast
from typing import Any, Final, Protocol, cast
from litellm._logging import _redact_string, verbose_proxy_logger
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
@ -31,6 +31,12 @@ async def forward_messages(client_ws: Any, backend_ws: Any):
pass
class _ProxyClientWebSocket(Protocol):
"""Client-facing websocket handle: this path only closes it after a failed handshake."""
async def close(self, code: int = ..., reason: str | None = ...) -> None: ...
class AzureOpenAIRealtime(AzureChatCompletion):
@staticmethod
def get_auth_headers(api_key: str | None, azure_ad_token: str | None) -> Mapping[str, str]:
@ -104,17 +110,17 @@ class AzureOpenAIRealtime(AzureChatCompletion):
async def async_realtime(
self,
model: str,
websocket: Any,
websocket: _ProxyClientWebSocket,
logging_obj: LiteLLMLogging,
api_base: str | None = None,
api_key: str | None = None,
api_version: str | None = None,
azure_ad_token: str | None = None,
client: Any | None = None,
client: object | None = None,
timeout: float | None = None,
realtime_protocol: str | None = None,
query_params: RealtimeQueryParams | None = None,
user_api_key_dict: Any | None = None,
user_api_key_dict: object | None = None,
litellm_metadata: dict | None = None,
):
import websockets

6
litellm/llms/azure/responses/transformation.py
View file

@ -105,7 +105,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
# Then filter out status from message items
if isinstance(validated_input, list):
filtered_input: Final[list[Any]] = []
filtered_input: Final[list[object]] = []
for item in validated_input:
if isinstance(item, dict) and item.get("type") == "message":
# Filter out status field from message items
@ -132,7 +132,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
if "tools" in response_api_optional_request_params and isinstance(
response_api_optional_request_params["tools"], list
):
new_tools: Final[list[dict[str, Any]]] = []
new_tools: Final[list[dict[str, object]]] = []
for tool in response_api_optional_request_params["tools"]:
if isinstance(tool, dict) and "function" in tool:
new_tool: dict[str, Any] = deepcopy(tool)
@ -300,7 +300,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
url: Final = self._construct_url_for_response_id_in_path(
api_base=api_base, response_id=response_id, path_suffix="/input_items"
)
params: Final[dict[str, Any]] = {}
params: Final[dict[str, str | int]] = {}
if after is not None:
params["after"] = after
if before is not None:

8
litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py
View file

@ -28,12 +28,12 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
async def count_tokens(
self,
model_to_use: str,
messages: list[dict[str, Any]] | None,
contents: list[dict[str, Any]] | None,
messages: list[dict[str, object]] | None,
contents: list[dict[str, object]] | None,
deployment: dict[str, Any] | None = None,
request_model: str = "",
tools: list[dict[str, Any]] | None = None,
system: Any | None = None,
tools: list[dict[str, object]] | None = None,
system: object | None = None,
) -> TokenCountResponse | None:
"""
Count tokens using Azure AI Anthropic's CountTokens API.

24
litellm/llms/base_llm/agents/transformation.py
View file

@ -10,7 +10,7 @@ InteractionsHTTPHandler).
"""
from abc import ABC, abstractmethod
from typing import Any
from collections.abc import Mapping
import httpx
@ -35,7 +35,7 @@ class BaseAgentsAPIConfig(ABC):
def get_complete_url(
self,
api_base: str | None,
litellm_params: dict[str, Any],
litellm_params: Mapping[str, object],
) -> str:
"""Return the full URL for POST /agents (create)."""
@ -43,7 +43,7 @@ class BaseAgentsAPIConfig(ABC):
def validate_environment(
self,
headers: dict[str, str],
litellm_params: dict[str, Any],
litellm_params: dict[str, object],
) -> dict[str, str]:
"""Validate credentials and return auth headers."""
@ -51,8 +51,8 @@ class BaseAgentsAPIConfig(ABC):
def transform_create_request(
self,
name: str,
litellm_params: dict[str, Any],
) -> dict[str, Any]:
litellm_params: Mapping[str, object],
) -> dict[str, object]:
"""Map name + litellm_params to the provider's create-agent body."""
@abstractmethod
@ -71,8 +71,8 @@ class BaseAgentsAPIConfig(ABC):
def transform_list_request(
self,
api_base: str | None,
litellm_params: dict[str, Any],
) -> tuple[str, dict[str, Any]]:
litellm_params: Mapping[str, object],
) -> tuple[str, dict[str, object]]:
"""Return (url, query_params) for GET /agents."""
@abstractmethod
@ -91,8 +91,8 @@ class BaseAgentsAPIConfig(ABC):
self,
name: str,
api_base: str | None,
litellm_params: dict[str, Any],
) -> tuple[str, dict[str, Any]]:
litellm_params: Mapping[str, object],
) -> tuple[str, dict[str, object]]:
"""Return (url, query_params) for GET /agents/{name}."""
@abstractmethod
@ -112,7 +112,7 @@ class BaseAgentsAPIConfig(ABC):
self,
name: str,
api_base: str | None,
litellm_params: dict[str, Any],
litellm_params: Mapping[str, object],
) -> str:
"""Return the URL for DELETE /agents/{name}."""
@ -133,8 +133,8 @@ class BaseAgentsAPIConfig(ABC):
self,
name: str,
api_base: str | None,
litellm_params: dict[str, Any],
) -> tuple[str, dict[str, Any]]:
litellm_params: Mapping[str, object],
) -> tuple[str, dict[str, object]]:
"""Return (url, query_params) for GET /agents/{name}/versions."""
@abstractmethod

10
litellm/llms/base_llm/guardrail_translation/base_translation.py
View file

@ -55,7 +55,7 @@ class BaseTranslation(ABC):
@staticmethod
def transform_user_api_key_dict_to_metadata(
user_api_key_dict: Any | None,
) -> dict[str, Any]:
) -> dict[str, object]:
"""
Transform user_api_key_dict to a metadata dict with prefixed keys.
@ -78,7 +78,7 @@ class BaseTranslation(ABC):
return {}
# Transform keys to be prefixed with 'user_api_key_'
transformed: Final = {}
transformed: Final[dict[str, object]] = {}
for key, value in user_dict.items():
# Skip None values and internal fields
if value is None or key.startswith("_"):
@ -174,7 +174,7 @@ class BaseTranslation(ABC):
self,
exc: "ModifyResponseException",
stream_started: bool = False,
responses_so_far: Sequence[Any] | None = None,
responses_so_far: Sequence[object] | None = None,
) -> Sequence[bytes] | None:
"""
Build the streaming chunks that deliver a guardrail block message and
@ -197,8 +197,8 @@ class BaseTranslation(ABC):
def build_stream_error_items(
self,
exc: "HTTPException",
responses_so_far: Sequence[Any] | None = None,
) -> Sequence[Any] | None:
responses_so_far: Sequence[object] | None = None,
) -> Sequence[object] | None:
"""
Build the stream items that surface a guardrail HTTPException (a block
with the default exception-on-block config, or a failed scan) after the

12
litellm/llms/base_llm/guardrail_translation/utils.py
View file

@ -2,7 +2,7 @@ from __future__ import annotations
import json
from collections.abc import Callable, Iterator, Sequence
from typing import Any, Final, TypeVar
from typing import Final, TypeVar
from pydantic import BaseModel
@ -10,7 +10,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUs
from litellm.types.llms.openai import AllMessageValues, ResponseAPIUsage
def _anthropic_stream_chunk_events(item: Any) -> list[dict]:
def _anthropic_stream_chunk_events(item: object) -> list[dict]:
if isinstance(item, dict):
return [item]
if isinstance(item, bytes):
@ -38,7 +38,7 @@ def _anthropic_stream_chunk_events(item: Any) -> list[dict]:
return events
def _usage_from_anthropic_stream_chunks(original_response: list[Any]) -> AnthropicUsage | None:
def _usage_from_anthropic_stream_chunks(original_response: Sequence[object]) -> AnthropicUsage | None:
input_tokens = 0
output_tokens = 0
found_usage = False
@ -81,7 +81,7 @@ def _usage_tokens(usage_obj: object, key: str, fallback_key: str) -> int:
return int(getattr(usage_obj, key, getattr(usage_obj, fallback_key, 0)) or 0)
def blocked_response_usage(original_response: Any | None) -> AnthropicUsage:
def blocked_response_usage(original_response: object) -> AnthropicUsage:
"""
Token usage for a synthetic guardrail-blocked response.
@ -191,7 +191,7 @@ def blocked_responses_stream_usage(original_response: object) -> ResponseAPIUsag
return blocked_responses_api_usage(completed)
def effective_skip_system_message_for_guardrail(guardrail_to_apply: Any) -> bool:
def effective_skip_system_message_for_guardrail(guardrail_to_apply: object) -> bool:
per: Final = getattr(guardrail_to_apply, "skip_system_message_in_guardrail", None)
if per is not None:
return bool(per)
@ -200,7 +200,7 @@ def effective_skip_system_message_for_guardrail(guardrail_to_apply: Any) -> bool
return bool(getattr(litellm, "skip_system_message_in_guardrail", False))
def effective_skip_tool_message_for_guardrail(guardrail_to_apply: Any) -> bool:
def effective_skip_tool_message_for_guardrail(guardrail_to_apply: object) -> bool:
per: Final = getattr(guardrail_to_apply, "skip_tool_message_in_guardrail", None)
if per is not None:
return bool(per)

23
litellm/llms/base_llm/vector_store_files/transformation.py
View file

@ -1,4 +1,5 @@
from abc import ABC, abstractmethod
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any
import httpx
@ -43,10 +44,10 @@ class BaseVectorStoreFilesConfig(ABC):
self,
*,
operation: str,
non_default_params: dict[str, Any],
optional_params: dict[str, Any],
non_default_params: Mapping[str, object],
optional_params: Mapping[str, object],
drop_params: bool,
) -> dict[str, Any]:
) -> Mapping[str, object]:
"""Map non-default OpenAI params to provider-specific params."""
return optional_params
@ -87,7 +88,7 @@ class BaseVectorStoreFilesConfig(ABC):
vector_store_id: str,
create_request: VectorStoreFileCreateRequest,
api_base: str,
) -> tuple[str, dict[str, Any]]: ...
) -> tuple[str, dict[str, object]]: ...
@abstractmethod
def transform_create_vector_store_file_response(
@ -103,7 +104,7 @@ class BaseVectorStoreFilesConfig(ABC):
vector_store_id: str,
query_params: VectorStoreFileListQueryParams,
api_base: str,
) -> tuple[str, dict[str, Any]]: ...
) -> tuple[str, dict[str, object]]: ...
@abstractmethod
def transform_list_vector_store_files_response(
@ -119,7 +120,7 @@ class BaseVectorStoreFilesConfig(ABC):
vector_store_id: str,
file_id: str,
api_base: str,
) -> tuple[str, dict[str, Any]]: ...
) -> tuple[str, dict[str, object]]: ...
@abstractmethod
def transform_retrieve_vector_store_file_response(
@ -135,7 +136,7 @@ class BaseVectorStoreFilesConfig(ABC):
vector_store_id: str,
file_id: str,
api_base: str,
) -> tuple[str, dict[str, Any]]: ...
) -> tuple[str, dict[str, object]]: ...
@abstractmethod
def transform_retrieve_vector_store_file_content_response(
@ -152,7 +153,7 @@ class BaseVectorStoreFilesConfig(ABC):
file_id: str,
update_request: VectorStoreFileUpdateRequest,
api_base: str,
) -> tuple[str, dict[str, Any]]: ...
) -> tuple[str, dict[str, object]]: ...
@abstractmethod
def transform_update_vector_store_file_response(
@ -168,7 +169,7 @@ class BaseVectorStoreFilesConfig(ABC):
vector_store_id: str,
file_id: str,
api_base: str,
) -> tuple[str, dict[str, Any]]: ...
) -> tuple[str, dict[str, object]]: ...
@abstractmethod
def transform_delete_vector_store_file_response(
@ -196,8 +197,8 @@ class BaseVectorStoreFilesConfig(ABC):
self,
*,
headers: dict[str, str],
optional_params: dict[str, Any],
request_data: dict[str, Any],
optional_params: Mapping[str, object],
request_data: Mapping[str, object],
api_base: str,
api_key: str | None = None,
) -> tuple[dict[str, str], bytes | None]:

22
litellm/llms/bedrock/base_aws_llm.py
View file

@ -4,7 +4,7 @@ import json
import os
import re
import urllib.parse
from collections.abc import Callable
from collections.abc import Callable, Mapping
from datetime import datetime
from threading import Lock
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, cast, get_args, overload
@ -137,7 +137,7 @@ class BaseAWSLLM:
return get_ssl_verify(ssl_verify=ssl_verify)
def get_cache_key(self, credential_args: dict[str, str | None]) -> str:
def get_cache_key(self, credential_args: Mapping[str, str | bool | None]) -> str:
"""
Generate a unique cache key based on the credential arguments.
"""
@ -147,8 +147,8 @@ class BaseAWSLLM:
def _get_or_set_cached_credentials(
self,
credential_args: dict[str, str | None],
credential_fetcher: Callable[[], tuple[Any, int | None]],
credential_args: Mapping[str, str | bool | None],
credential_fetcher: Callable[[], tuple[Credentials, int | None]],
) -> Any:
"""
Read-through IAM cache on the process-wide ``DualCache``.
@ -283,7 +283,19 @@ class BaseAWSLLM:
aws_external_id,
)
args: Final = {k: v for k, v in locals().items() if k.startswith("aws_") or k == "ssl_verify"}
args: Final = {
"aws_access_key_id": aws_access_key_id,
"aws_secret_access_key": aws_secret_access_key,
"aws_session_token": aws_session_token,
"aws_region_name": aws_region_name,
"aws_session_name": aws_session_name,
"aws_profile_name": aws_profile_name,
"aws_role_name": aws_role_name,
"aws_web_identity_token": aws_web_identity_token,
"aws_sts_endpoint": aws_sts_endpoint,
"aws_external_id": aws_external_id,
"ssl_verify": ssl_verify,
}
#########################################################
# Handle diff boto3 auth flows

12
litellm/llms/bedrock/chat/invoke_handler.py
View file

@ -163,7 +163,7 @@ async def make_call(
json_mode: bool | None = False,
bedrock_invoke_provider: litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None = None,
stream_chunk_size: int | None = None,
) -> tuple[Any, httpx.Headers]:
) -> "tuple[MockResponseIterator | AsyncIterator[GChunk | ModelResponseStream | dict], httpx.Headers]":
try:
if client is None:
client = get_async_httpx_client(
@ -199,7 +199,9 @@ async def make_call(
messages=messages,
encoding=litellm.encoding,
)
completion_stream: Any = MockResponseIterator(model_response=model_response, json_mode=json_mode)
completion_stream: MockResponseIterator | AsyncIterator[GChunk | ModelResponseStream | dict] = (
MockResponseIterator(model_response=model_response, json_mode=json_mode)
)
elif bedrock_invoke_provider == "anthropic":
decoder: AWSEventStreamDecoder = AmazonAnthropicClaudeStreamDecoder(
model=model,
@ -248,7 +250,7 @@ def make_sync_call(
json_mode: bool | None = False,
bedrock_invoke_provider: litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None = None,
stream_chunk_size: int | None = None,
) -> tuple[Any, httpx.Headers]:
) -> "tuple[MockResponseIterator | Iterator[GChunk | ModelResponseStream | dict], httpx.Headers]":
try:
if client is None:
client = _get_httpx_client(
@ -283,7 +285,9 @@ def make_sync_call(
messages=messages,
encoding=litellm.encoding,
)
completion_stream: Any = MockResponseIterator(model_response=model_response, json_mode=json_mode)
completion_stream: MockResponseIterator | Iterator[GChunk | ModelResponseStream | dict] = (
MockResponseIterator(model_response=model_response, json_mode=json_mode)
)
elif bedrock_invoke_provider == "anthropic":
decoder: AWSEventStreamDecoder = AmazonAnthropicClaudeStreamDecoder(
model=model,

19
litellm/llms/bedrock/common_utils.py
View file

@ -52,8 +52,8 @@ _BEDROCK_AWS_AUTH_PARAMETER_KEYS: Final[tuple[str, ...]] = (
def merge_bedrock_aws_request_params(
litellm_params: Mapping[str, Any],
optional_params: Mapping[str, Any],
litellm_params: Mapping[str, object],
optional_params: Mapping[str, object],
) -> dict[str, Any]:
"""Merge deployment and request parameters without allowing auth escalation.
@ -303,7 +303,7 @@ def normalize_json_schema_custom_types_to_object(schema: dict) -> None:
Uses an explicit stack (not recursion) to satisfy recursive-function guards in CI.
"""
stack: Final[list[Any]] = [schema]
stack: Final[list[object]] = [schema]
seen: Final[set[int]] = set()
while stack:
node = stack.pop()
@ -913,7 +913,7 @@ def _get_bedrock_converse_strict_tools_flag(base_model: str) -> bool | None:
return None
def normalize_bedrock_opus_output_config_effort(model: str, output_config: Any) -> None:
def normalize_bedrock_opus_output_config_effort(model: str, output_config: object) -> None:
"""
Normalize Anthropic ``output_config.effort`` values for Bedrock Opus ids.
@ -1436,6 +1436,11 @@ class BedrockEventStreamDecoderBase:
return chunk.decode()
def _decoded_json_value(raw: str) -> object:
"""Decode a JSON document into an opaque value for isinstance narrowing."""
return json.loads(raw)
def get_anthropic_beta_from_headers(headers: dict) -> list[str]:
"""
Extract anthropic-beta header values and convert them to a list.
@ -1463,7 +1468,7 @@ def get_anthropic_beta_from_headers(headers: dict) -> list[str]:
anthropic_beta_header = anthropic_beta_header.strip()
if anthropic_beta_header.startswith("[") and anthropic_beta_header.endswith("]"):
try:
parsed: Final = json.loads(anthropic_beta_header)
parsed: Final = _decoded_json_value(anthropic_beta_header)
if isinstance(parsed, list):
return [str(beta).strip() for beta in parsed]
except json.JSONDecodeError:
@ -1476,8 +1481,8 @@ def get_anthropic_beta_from_headers(headers: dict) -> list[str]:
def resolve_s3_encryption_key_id(
litellm_params: Mapping[str, Any],
optional_params: Mapping[str, Any] | None = None,
litellm_params: Mapping[str, object],
optional_params: Mapping[str, object] | None = None,
) -> str | None:
"""
Resolve the SSE-KMS key configured for Bedrock batch/file S3 objects.

16
litellm/llms/bedrock/image_edit/amazon_nova_canvas_image_edit_transformation.py
View file

@ -47,7 +47,7 @@ def _nova_canvas_task_body(
task_type: str | None,
mask_prompt: str | None,
out_painting_mode: str | None,
) -> dict[str, Any]:
) -> dict[str, object]:
"""Build InvokeModel body task section (without imageGenerationConfig)."""
if task_type == "BACKGROUND_REMOVAL":
return {
@ -60,7 +60,7 @@ def _nova_canvas_task_body(
"OUTPAINTING requires either a mask image or a mask prompt. "
"Pass mask=<file> or maskPrompt=<str> in the request."
)
out_params: Final[dict[str, Any]] = {
out_params: Final[dict[str, object]] = {
"image": image_b64,
"text": text,
}
@ -79,7 +79,7 @@ def _nova_canvas_task_body(
# Honour explicit IMAGE_VARIATION even when a mask is present (mask is ignored
# for this task type; callers use INPAINTING when they want mask semantics).
if task_type == "IMAGE_VARIATION":
var_params_explicit: Final[dict[str, Any]] = {
var_params_explicit: Final[dict[str, object]] = {
"images": [image_b64],
"text": text,
}
@ -100,7 +100,7 @@ def _nova_canvas_task_body(
"or omit taskType for automatic routing (mask → INPAINTING, else IMAGE_VARIATION)."
)
if mask_b64 is not None or mask_prompt is not None or task_type == "INPAINTING":
in_params: Final[dict[str, Any]] = {"image": image_b64, "text": text}
in_params: Final[dict[str, object]] = {"image": image_b64, "text": text}
if mask_prompt is not None:
in_params["maskPrompt"] = mask_prompt
elif mask_b64 is not None:
@ -114,7 +114,7 @@ def _nova_canvas_task_body(
"See https://docs.aws.amazon.com/nova/latest/userguide/image-gen-req-resp-structure.html"
)
return {"taskType": "INPAINTING", "inPaintingParams": in_params}
var_params: Final[dict[str, Any]] = {
var_params: Final[dict[str, object]] = {
"images": [image_b64],
"text": text,
}
@ -250,9 +250,9 @@ class BedrockAmazonNovaCanvasImageEditConfig(BaseImageEditConfig):
image_edit_optional_params: ImageEditOptionalRequestParams,
model: str,
drop_params: bool,
) -> dict[str, Any]:
) -> dict[str, object]:
supported: Final = set(self.get_supported_openai_params(model))
mapped: Final[dict[str, Any]] = dict(image_edit_optional_params)
mapped: Final[dict[str, object]] = dict(image_edit_optional_params)
_size: Final = mapped.pop("size", None)
if _size is not None and isinstance(_size, str) and "x" in _size:
w, h = _size.split("x", 1)
@ -327,7 +327,7 @@ class BedrockAmazonNovaCanvasImageEditConfig(BaseImageEditConfig):
cfg_scale: Final = op.pop("cfgScale", None)
seed: Final = op.pop("seed", None)
image_generation_config: Final[dict[str, Any]] = {}
image_generation_config: Final[dict[str, object]] = {}
nested_igc: Final = op.pop("imageGenerationConfig", None)
if isinstance(nested_igc, dict):
image_generation_config.update(nested_igc)

4
litellm/llms/bedrock/vector_stores/transformation.py
View file

@ -203,7 +203,7 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
encoded_vector_store_id: Final = encode_url_path_segment(vector_store_id, field_name="vector_store_id")
url: Final = f"{api_base}/{encoded_vector_store_id}/retrieve"
request_body: Final[dict[str, Any]] = {
request_body: Final[dict[str, object]] = {
"retrievalQuery": BedrockKBRetrievalQuery(text=query),
}
@ -288,7 +288,7 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
data_source_id: Final = metadata.get("x-amz-bedrock-kb-data-source-id", "unknown") if metadata else "unknown"
return f"bedrock-kb-document-{data_source_id}"
def _get_attributes_from_metadata(self, metadata: dict[str, Any]) -> dict[str, Any]:
def _get_attributes_from_metadata(self, metadata: dict[str, object]) -> dict[str, object]:
"""
Extract all attributes from Bedrock KB metadata.
Returns a copy of the metadata dictionary.

Some files were not shown because too many files have changed in this diff Show more