Merge pull request #37400 from BerriAI/litellm_internal_staging

chore(ci): promote internal staging to main

.github/workflows/test-unit-proxy-endpoints.yml

@@ -43,6 +43,7 @@ jobs:
         tests/test_litellm/proxy/video_endpoints
         tests/test_litellm/proxy/response_api_endpoints
         tests/test_litellm/proxy/image_endpoints
+        tests/test_litellm/proxy/ocr_endpoints
         tests/test_litellm/proxy/vector_store_endpoints
         tests/test_litellm/proxy/agent_endpoints
         tests/test_litellm/proxy/a2a

backend/routes/allowlist.py

@@ -35,6 +35,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     # Models & routing config
     "/model/",
     "/v1/model/info",
+    "/v1/model/deprecations",
     "/v2/model/",
     "/model_group",
     "/model_access_group/",

basedpyright-code-budget.json

@@ -1,9 +1,9 @@
 {
   "reportAny": {
-    "limit": 22945
+    "limit": 22343
   },
   "reportArgumentType": {
-    "limit": 2579
+    "limit": 2578
   },
   "reportAssignmentType": {
     "limit": 323
@@ -24,13 +24,13 @@
     "limit": 19
   },
   "reportExplicitAny": {
-    "limit": 7311
+    "limit": 6991
   },
   "reportFunctionMemberAccess": {
     "limit": 7
   },
   "reportGeneralTypeIssues": {
-    "limit": 157
+    "limit": 154
   },
   "reportIncompatibleMethodOverride": {
     "limit": 56
@@ -54,10 +54,10 @@
     "limit": 0
   },
   "reportMissingParameterType": {
-    "limit": 5707
+    "limit": 5681
   },
   "reportMissingTypeArgument": {
-    "limit": 15640
+    "limit": 15605
   },
   "reportMissingTypeStubs": {
     "limit": 40
@@ -72,7 +72,7 @@
     "limit": 0
   },
   "reportOptionalMemberAccess": {
-    "limit": 1069
+    "limit": 1061
   },
   "reportOptionalOperand": {
     "limit": 0
@@ -84,7 +84,7 @@
     "limit": 56
   },
   "reportPrivateUsage": {
-    "limit": 1824
+    "limit": 1823
   },
   "reportRedeclaration": {
     "limit": 8
@@ -99,19 +99,19 @@
     "limit": 0
   },
   "reportUnknownArgumentType": {
-    "limit": 44776
+    "limit": 44709
   },
   "reportUnknownLambdaType": {
-    "limit": 113
+    "limit": 112
   },
   "reportUnknownMemberType": {
-    "limit": 39237
+    "limit": 39154
   },
   "reportUnknownParameterType": {
-    "limit": 19967
+    "limit": 19944
   },
   "reportUnknownVariableType": {
-    "limit": 30881
+    "limit": 30772
   },
   "reportUnnecessaryCast": {
     "limit": 117
@@ -123,7 +123,7 @@
     "limit": 5
   },
   "reportUnnecessaryIsInstance": {
-    "limit": 853
+    "limit": 851
   },
   "reportUntypedBaseClass": {
     "limit": 0
@@ -132,7 +132,7 @@
     "limit": 27
   },
   "reportUnusedClass": {
-    "limit": 23
+    "limit": 21
   },
   "reportUnusedFunction": {
     "limit": 139

ci_cd/generate_model_prices_schema.py

@@ -41,6 +41,11 @@ OBJECT_KEYS: dict[str, JsonSchema] = {
         },
         "additionalProperties": False,
     },
+    "guardrail_cost_per_unit": {
+        "type": "object",
+        "description": "USD cost per billable guardrail unit, keyed by the provider's usage counter name (e.g. Bedrock's contentPolicyUnits).",
+        "additionalProperties": NONNEG_NUMBER,
+    },
     "metadata": {
         "type": "object",
         "description": "Free-form notes about the entry (e.g. pricing derivation).",

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -24,12 +24,16 @@ if TYPE_CHECKING:
 
 CHECK_BATCH_COST_USER_AGENT = "LiteLLM Proxy/CheckBatchCost"
 
-TERMINAL_MANAGED_OBJECT_STATUSES: Final[Tuple[str, ...]] = (
+PROVIDER_TERMINAL_BATCH_STATUSES: Final[Tuple[str, ...]] = (
     "completed",
     "complete",
     "failed",
     "expired",
     "cancelled",
+)
+
+TERMINAL_MANAGED_OBJECT_STATUSES: Final[Tuple[str, ...]] = (
+    *PROVIDER_TERMINAL_BATCH_STATUSES,
     "stale_expired",
 )
 
@@ -286,6 +290,57 @@ class CheckBatchCost:
         404 must not retire the row; the staleness sweep bounds it instead."""
         return self.llm_router.get_deployment(model_id=model_id) is not None
 
+    @staticmethod
+    def _is_output_file_gone_at_provider(error: Exception, output_file_id: Optional[str]) -> bool:
+        """A 404 naming the output file means there is nothing to fetch on this or any
+        later poll: providers like Vertex AI advertise an output path for every batch,
+        including terminal ones that never wrote it. Any other failure may be
+        transient, so it keeps retrying until the staleness sweep bounds it."""
+        import openai
+
+        from litellm.exceptions import NotFoundError
+
+        if not output_file_id:
+            return False
+        return isinstance(error, (NotFoundError, openai.NotFoundError)) and output_file_id in str(error)
+
+    async def _finalize_unbilled_terminal_job(
+        self, job: "LiteLLM_ManagedObjectTable", response: "LiteLLMBatch"
+    ) -> None:
+        """Persist a terminal batch that has nothing billable, converting any raw
+        provider file ids to managed ids, and take it out of the poll page."""
+        try:
+            from litellm.proxy.openai_files_endpoints.common_utils import (
+                _is_base64_encoded_unified_file_id,
+                ensure_batch_response_managed_file_ids,
+            )
+
+            response.id = job.unified_object_id
+            await ensure_batch_response_managed_file_ids(
+                response=response,
+                managed_files_obj=self.proxy_logging_obj.get_proxy_hook("managed_files"),
+                prisma_client=self.prisma_client,
+                verbose_proxy_logger=verbose_proxy_logger,
+                db_batch_object=job,
+                unified_batch_id=_is_base64_encoded_unified_file_id(job.unified_object_id),
+            )
+            update_data: Final[dict] = {
+                "status": response.status,
+                "file_object": response.model_dump_json(),
+                **({"batch_processed": True} if self._has_batch_processed_column else {}),
+            }
+            await self.prisma_client.db.litellm_managedobjecttable.update(
+                where={"id": job.id},
+                data=update_data,
+            )
+            verbose_proxy_logger.info(
+                f"CheckBatchCost: marked job {job.id} as {response.status} in DB"
+            )
+        except Exception as db_err:
+            verbose_proxy_logger.error(
+                f"CheckBatchCost: failed to mark job {job.id} as {response.status} in DB: {db_err}"
+            )
+
     @staticmethod
     def _record_error(
         prom_logger: Optional["PrometheusLogger"], error_type: str
@@ -528,6 +583,7 @@ class CheckBatchCost:
         from litellm.files.main import afile_content
         from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
         from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
+        from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info
         from litellm.proxy.openai_files_endpoints.common_utils import (
             _is_base64_encoded_unified_file_id,
         )
@@ -648,15 +704,20 @@ class CheckBatchCost:
                             f"{_file_attr}={_raw_file_id!r}: {_e}"
                         )
 
-        # Pass deployment model_info so custom batch pricing
-        # (input_cost_per_token_batches etc.) is used for cost calc
-        deployment_model_info = deployment_info.model_info.model_dump() if deployment_info.model_info else {}
+        # Pass the deployment's router-registered pricing (litellm_params custom
+        # rates merged with the model's published rates) so custom batch pricing
+        # (input_cost_per_token_batches etc.) is used for cost calc, exactly as
+        # the inline retrieve path does.
+        deployment_model_info = deployment_pricing_model_info(
+            model_id=model_id,
+            deployment_model=litellm_model_name,
+        )
         batch_cost, batch_usage, batch_models = (
             await calculate_batch_cost_and_usage(
                 file_content_dictionary=file_content_as_dict,
                 custom_llm_provider=llm_provider,  # type: ignore
                 model_name=model_name,
-                model_info=deployment_model_info,  # type: ignore[arg-type]
+                model_info=deployment_model_info,
             )
         )
         logging_obj = LiteLLMLogging(
@@ -796,7 +857,7 @@ class CheckBatchCost:
 
             ## RETRIEVE THE BATCH JOB OUTPUT FILE
             if (
-                response.status in ("completed", "complete", "expired")
+                response.status in PROVIDER_TERMINAL_BATCH_STATUSES
                 and response.output_file_id is not None
             ):
                 try:
@@ -808,6 +869,15 @@ class CheckBatchCost:
                         prom_logger=prom_logger,
                     )
                 except Exception as tracking_err:
+                    if self._is_output_file_gone_at_provider(
+                        tracking_err, response.output_file_id
+                    ) and self._batch_deployment_exists(model_id):
+                        verbose_proxy_logger.warning(
+                            f"CheckBatchCost: output file {response.output_file_id} of batch {batch_id} "
+                            f"does not exist at the provider; retiring job {job.id} unbilled"
+                        )
+                        await self._finalize_unbilled_terminal_job(job, response)
+                        continue
                     verbose_proxy_logger.error(
                         f"CheckBatchCost: failed to track cost for batch {batch_id} "
                         f"(job {job.id}); leaving it unprocessed so the next poll retries: {tracking_err}"
@@ -837,45 +907,8 @@ class CheckBatchCost:
                         f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}"
                     )
 
-            elif response.status in (
-                "completed",
-                "complete",
-                "failed",
-                "expired",
-                "cancelled",
-            ):
-                try:
-                    from litellm.proxy.openai_files_endpoints.common_utils import (
-                        _is_base64_encoded_unified_file_id,
-                        ensure_batch_response_managed_file_ids,
-                    )
-
-                    response.id = job.unified_object_id
-                    await ensure_batch_response_managed_file_ids(
-                        response=response,
-                        managed_files_obj=self.proxy_logging_obj.get_proxy_hook("managed_files"),
-                        prisma_client=self.prisma_client,
-                        verbose_proxy_logger=verbose_proxy_logger,
-                        db_batch_object=job,
-                        unified_batch_id=_is_base64_encoded_unified_file_id(job.unified_object_id),
-                    )
-                    update_data = {
-                        "status": response.status,
-                        "file_object": response.model_dump_json(),
-                    }
-                    if self._has_batch_processed_column:
-                        update_data["batch_processed"] = True
-                    await self.prisma_client.db.litellm_managedobjecttable.update(
-                        where={"id": job.id},
-                        data=update_data,
-                    )
-                    verbose_proxy_logger.info(
-                        f"CheckBatchCost: marked job {job.id} as {response.status} in DB"
-                    )
-                except Exception as db_err:
-                    verbose_proxy_logger.error(
-                        f"CheckBatchCost: failed to mark job {job.id} as {response.status} in DB: {db_err}"
-                    )
+            elif response.status in PROVIDER_TERMINAL_BATCH_STATUSES:
+                await self._finalize_unbilled_terminal_job(job, response)
 
         # Record polling run metrics (always, even if nothing was processed)
         if prom_logger:

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -41,6 +41,7 @@ from litellm.proxy._types import (
     CallTypes,
     LiteLLM_ManagedFileTable,
     LiteLLM_ManagedObjectTable,
+    ProxyException,
     UserAPIKeyAuth,
 )
 from litellm.proxy.openai_files_endpoints.common_utils import (
@@ -423,13 +424,26 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         # This is because the encoded object ids stored in the managed objects table do not contain the provider information
         # To support provider filtering, we would need to store the provider information in the encoded object ids
         if provider:
-            raise Exception("Filtering by 'provider' is not supported when using managed batches.")
+            raise ProxyException(
+                message="Filtering by 'provider' is not supported when using managed batches.",
+                type="invalid_request_error",
+                param="provider",
+                code=400,
+            )
 
         # Model name filtering is not supported for managed batches
         # This is because the encoded object ids stored in the managed objects table do not contain the model name
         # A hash of the model name + litellm_params for the model name is encoded as the model id. This is not sufficient to reliably map the target model names to the model ids.
         if target_model_names:
-            raise Exception("Filtering by 'target_model_names' is not supported when using managed batches.")
+            raise ProxyException(
+                message="Filtering by 'target_model_names' is not supported when using managed batches.",
+                type="invalid_request_error",
+                param="target_model_names",
+                code=400,
+            )
+
+        if limit == 0:
+            return build_list_page([])
 
         owner_filter = build_owner_filter(user_api_key_dict)
         if owner_filter is None:

enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py

@@ -11,7 +11,7 @@ Endpoints for /project operations
 #### PROJECT MANAGEMENT ####
 
 import json
-from collections.abc import Mapping, Sequence
+from collections.abc import Sequence
 from typing import TYPE_CHECKING
 
 from fastapi import APIRouter, Depends, HTTPException, Request
@@ -29,7 +29,11 @@ from litellm.proxy.utils import PrismaClient, handle_exception_on_proxy
 
 if TYPE_CHECKING:
     from prisma import models as prisma_models
-    from prisma.actions import LiteLLM_TeamTableActions
+    from prisma.actions import (
+        LiteLLM_ProjectTableActions,
+        LiteLLM_TeamTableActions,
+        LiteLLM_VerificationTokenActions,
+    )
 
 router = APIRouter()
 
@@ -39,6 +43,27 @@ def _team_table(prisma_client: PrismaClient) -> "LiteLLM_TeamTableActions[prisma
     return team_table
 
 
+def _project_table(prisma_client: PrismaClient) -> "LiteLLM_ProjectTableActions[prisma_models.LiteLLM_ProjectTable]":
+    project_table: LiteLLM_ProjectTableActions[prisma_models.LiteLLM_ProjectTable] = (
+        prisma_client.db.litellm_projecttable
+    )
+    return project_table
+
+
+def _verification_token_table(
+    prisma_client: PrismaClient,
+) -> "LiteLLM_VerificationTokenActions[prisma_models.LiteLLM_VerificationToken]":
+    verification_token_table: LiteLLM_VerificationTokenActions[prisma_models.LiteLLM_VerificationToken] = (
+        prisma_client.db.litellm_verificationtoken
+    )
+    return verification_token_table
+
+
+def _jsonified(prisma_client: PrismaClient, payload: dict[str, object]) -> dict[str, object]:
+    jsonified: dict[str, object] = prisma_client.jsonify_object(payload)
+    return jsonified
+
+
 async def _check_user_permission_for_project(
     user_api_key_dict: UserAPIKeyAuth,
     team_id: str | None,
@@ -137,7 +162,7 @@ def _check_team_project_limits(
 
     # --- Validate project models are a subset of team models ---
     project_models = data.models
-    team_models = team_object.models or []
+    team_models: list[str] = team_object.models or []
     if project_models and len(team_models) > 0:
         # If team has 'all-proxy-models', skip validation as it allows all models
         if SpecialModelNames.all_proxy_models.value not in team_models:
@@ -188,11 +213,11 @@ async def _create_budget_for_project(
 ) -> str:
     """Create a budget for the project and return budget_id."""
     budget_params = LiteLLM_BudgetTable.model_fields.keys()
-    _json_data: Mapping[str, object] = data.json(exclude_none=True)
+    _json_data: dict[str, object] = data.model_dump(exclude_none=True)
     _budget_data = {k: v for k, v in _json_data.items() if k in budget_params}
     budget_row = LiteLLM_BudgetTable.model_validate(_budget_data)
 
-    new_budget = prisma_client.jsonify_object(budget_row.json(exclude_none=True))
+    new_budget = _jsonified(prisma_client, budget_row.model_dump(exclude_none=True))
 
     _budget: prisma_models.LiteLLM_BudgetTable = await prisma_client.db.litellm_budgettable.create(
         data={
@@ -227,7 +252,7 @@ async def _set_project_object_permission(
     return None
 
 
-def _remove_budget_fields_from_project_data(project_data: dict) -> dict:
+def _remove_budget_fields_from_project_data(project_data: dict[str, object]) -> dict[str, object]:
     """
     Remove budget fields from project data.
     Budget fields belong to LiteLLM_BudgetTable, not LiteLLM_ProjectTable.
@@ -396,9 +421,7 @@ async def new_project(
             data.project_id = str(uuid.uuid4())
         else:
             # Check if project_id already exists
-            existing_project = await prisma_client.db.litellm_projecttable.find_unique(
-                where={"project_id": data.project_id}
-            )
+            existing_project = await _project_table(prisma_client).find_unique(where={"project_id": data.project_id})
             if existing_project is not None:
                 raise ProxyException(
                     message=f"Project id = {data.project_id} already exists. Please use a different project id.",
@@ -423,11 +446,14 @@ async def new_project(
         )
 
         # Create project row (following organization_endpoints.py pattern)
-        project_row = LiteLLM_ProjectTable(
-            **data.json(exclude_none=True),
-            object_permission_id=object_permission_id,
-            created_by=user_api_key_dict.user_id or litellm_proxy_admin_name,
-            updated_by=user_api_key_dict.user_id or litellm_proxy_admin_name,
+        project_row_payload: dict[str, object] = data.model_dump(exclude_none=True)
+        project_row = LiteLLM_ProjectTable.model_validate(
+            {
+                **project_row_payload,
+                "object_permission_id": object_permission_id,
+                "created_by": user_api_key_dict.user_id or litellm_proxy_admin_name,
+                "updated_by": user_api_key_dict.user_id or litellm_proxy_admin_name,
+            }
         )
 
         for field in LiteLLM_ManagementEndpoint_MetadataFields:
@@ -438,7 +464,7 @@ async def new_project(
                     value=getattr(data, field),
                 )
 
-        new_project_row = prisma_client.jsonify_object(project_row.json(exclude_none=True))
+        new_project_row = _jsonified(prisma_client, project_row.model_dump(exclude_none=True))
 
         # Remove budget fields (following organization_endpoints.py pattern)
         new_project_row = _remove_budget_fields_from_project_data(new_project_row)
@@ -560,7 +586,7 @@ async def update_project(
         # Fetch existing project
         existing_project: (
             prisma_models.LiteLLM_ProjectTable | None
-        ) = await prisma_client.db.litellm_projecttable.find_unique(where={"project_id": data.project_id})
+        ) = await _project_table(prisma_client).find_unique(where={"project_id": data.project_id})
 
         if existing_project is None:
             raise ProxyException(
@@ -617,8 +643,7 @@ async def update_project(
             )
 
         # Prepare update data
-        update_data = data.json(exclude_none=True, exclude={"project_id"})
-        update_data = prisma_client.jsonify_object(update_data)
+        update_data = _jsonified(prisma_client, data.model_dump(exclude_none=True, exclude={"project_id"}))
         update_data["updated_by"] = user_api_key_dict.user_id or litellm_proxy_admin_name
 
         # Handle budget updates
@@ -660,9 +685,10 @@ async def update_project(
         # Handle metadata fields
         for field in LiteLLM_ManagementEndpoint_MetadataFields:
             if field in update_data:
-                if update_data.get("metadata") is None:
-                    update_data["metadata"] = {}
-                update_data["metadata"][field] = update_data.pop(field)
+                existing_metadata = update_data.get("metadata")
+                metadata_dict: dict[str, object] = existing_metadata if isinstance(existing_metadata, dict) else {}
+                metadata_dict[field] = update_data.pop(field)
+                update_data["metadata"] = metadata_dict
 
         # Remove budget fields (following organization_endpoints.py pattern)
         update_data = _remove_budget_fields_from_project_data(update_data)
@@ -748,11 +774,11 @@ async def delete_project(
                 detail={"error": "Only admins can delete projects"},
             )
 
-        deleted_projects = []
+        deleted_projects: list[prisma_models.LiteLLM_ProjectTable | None] = []
 
         for project_id in data.project_ids:
             # Check if project exists
-            existing_project = await prisma_client.db.litellm_projecttable.find_unique(where={"project_id": project_id})
+            existing_project = await _project_table(prisma_client).find_unique(where={"project_id": project_id})
 
             if existing_project is None:
                 raise ProxyException(
@@ -765,7 +791,7 @@ async def delete_project(
             # Check if there are any keys associated with this project
             associated_keys: Sequence[
                 prisma_models.LiteLLM_VerificationToken
-            ] = await prisma_client.db.litellm_verificationtoken.find_many(where={"project_id": project_id})
+            ] = await _verification_token_table(prisma_client).find_many(where={"project_id": project_id})
 
             if len(associated_keys) > 0:
                 raise ProxyException(
@@ -778,7 +804,7 @@ async def delete_project(
             # Delete the project
             deleted_project: (
                 prisma_models.LiteLLM_ProjectTable | None
-            ) = await prisma_client.db.litellm_projecttable.delete(where={"project_id": project_id})
+            ) = await _project_table(prisma_client).delete(where={"project_id": project_id})
 
             await delete_cached_project_object(
                 project_id=project_id,
@@ -829,7 +855,7 @@ async def project_info(
             )
 
         # Fetch project
-        project: prisma_models.LiteLLM_ProjectTable | None = await prisma_client.db.litellm_projecttable.find_unique(
+        project: prisma_models.LiteLLM_ProjectTable | None = await _project_table(prisma_client).find_unique(
             where={"project_id": project_id},
             include={"litellm_budget_table": True, "object_permission": True},
         )
@@ -901,7 +927,7 @@ async def list_projects(
         if user_api_key_has_admin_view(user_api_key_dict):
             projects: Sequence[
                 prisma_models.LiteLLM_ProjectTable
-            ] = await prisma_client.db.litellm_projecttable.find_many(
+            ] = await _project_table(prisma_client).find_many(
                 include={"litellm_budget_table": True, "object_permission": True}
             )
         else:
@@ -911,9 +937,9 @@ async def list_projects(
             user_record: prisma_models.LiteLLM_UserTable | None = await prisma_client.db.litellm_usertable.find_unique(
                 where={"user_id": user_api_key_dict.user_id},
             )
-            user_team_ids: Sequence[str] = user_record.teams if user_record is not None and user_record.teams else []
+            user_team_ids: list[str] = user_record.teams if user_record is not None and user_record.teams else []
 
-            projects = await prisma_client.db.litellm_projecttable.find_many(
+            projects = await _project_table(prisma_client).find_many(
                 where={"team_id": {"in": user_team_ids}},
                 include={"litellm_budget_table": True, "object_permission": True},
             )

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.56"
+version = "0.1.57"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.56"
+version = "0.1.57"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

gateway/routes/allowlist.py

@@ -83,6 +83,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
     "/azure_ai/",
     "/aws/",
     "/bedrock/",
+    "/comprehendmedical",
     "/cohere/",
     "/gemini/",
     "/google/",

helm/litellm-helm/templates/migrations-job.yaml

@@ -119,4 +119,7 @@ spec:
       {{- end }}
   ttlSecondsAfterFinished: {{ .Values.migrationJob.ttlSecondsAfterFinished }}
   backoffLimit: {{ .Values.migrationJob.backoffLimit }}
+  {{- with .Values.migrationJob.activeDeadlineSeconds }}
+  activeDeadlineSeconds: {{ . }}
+  {{- end }}
 {{- end }}

helm/litellm-helm/tests/migrations-job_tests.yaml

@@ -314,3 +314,31 @@ tests:
               operator: Equal
               value: litellm-e2e
               effect: NoSchedule
+
+  - it: bounds the Job with a deadline by default, so a blocked migration cannot stall the release forever
+    set:
+      migrationJob:
+        enabled: true
+    asserts:
+      - equal:
+          path: spec.activeDeadlineSeconds
+          value: 1800
+
+  - it: honours an operator-supplied deadline
+    set:
+      migrationJob:
+        enabled: true
+        activeDeadlineSeconds: 600
+    asserts:
+      - equal:
+          path: spec.activeDeadlineSeconds
+          value: 600
+
+  - it: omits the deadline entirely when it is nulled out, restoring the unbounded behaviour
+    set:
+      migrationJob:
+        enabled: true
+        activeDeadlineSeconds: null
+    asserts:
+      - notExists:
+          path: spec.activeDeadlineSeconds

helm/litellm-helm/values.yaml

@@ -427,6 +427,13 @@ migrationJob:
   enabled: true # Enable or disable the schema migration Job
   retries: 3 # Number of retries for the Job in case of failure
   backoffLimit: 4 # Backoff limit for Job restarts
+  # Wall-clock budget for the whole Job, shared across every `backoffLimit`
+  # retry rather than granted per attempt. Without it a migration that blocks
+  # on the database never fails, and when the Helm hook is enabled the release
+  # waits on it forever: `helm upgrade` and any GitOps controller driving it
+  # stop reconciling the whole chart until someone deletes the Job by hand.
+  # Set to null to opt out and restore the unbounded behaviour.
+  activeDeadlineSeconds: 1800
   disableSchemaUpdate: false # Skip schema migrations for specific environments. When True, the job will exit with code 0.
   # Optional service account for the migration job.
   # Only used when migrationJob.hooks.helm.enabled=true and serviceAccount.create=true.

helm/litellm/templates/ingress.yaml

@@ -24,7 +24,7 @@
     "/v1/video" "/v1/videos" "/video" "/videos" "/v1/search" "/search"
     "/v1/containers" "/containers" "/v1/evals" "/v1/memory" "/queue/chat"
     "/v1beta" "/interactions"
-    "/anthropic" "/azure" "/azure_ai" "/aws" "/bedrock" "/cohere" "/gemini" "/google"
+    "/anthropic" "/azure" "/azure_ai" "/aws" "/bedrock" "/comprehendmedical" "/cohere" "/gemini" "/google"
     "/vertex_ai" "/vertex-ai" "/assemblyai" "/eu.assemblyai" "/langfuse" "/vllm"
     "/mistral" "/groq" "/voyage" "/cursor" "/milvus" "/openai_passthrough"
     "/toolset"

helm/litellm/templates/migrations-job.yaml

@@ -21,6 +21,9 @@ metadata:
 spec:
   backoffLimit: {{ .Values.migrationJob.backoffLimit }}
   ttlSecondsAfterFinished: {{ .Values.migrationJob.ttlSecondsAfterFinished }}
+  {{- with .Values.migrationJob.activeDeadlineSeconds }}
+  activeDeadlineSeconds: {{ . }}
+  {{- end }}
   template:
     metadata:
       {{- /* The Job's selector is generated by the controller rather than

helm/litellm/tests/migration_job_tests.yaml

@@ -167,3 +167,24 @@ tests:
       - equal:
           path: spec.template.metadata.labels['app.kubernetes.io/component']
           value: batch-migrations
+
+  - it: bounds the Job with a deadline by default, so a blocked migration cannot stall the release forever
+    asserts:
+      - equal:
+          path: spec.activeDeadlineSeconds
+          value: 1800
+
+  - it: honours an operator-supplied deadline
+    set:
+      migrationJob.activeDeadlineSeconds: 600
+    asserts:
+      - equal:
+          path: spec.activeDeadlineSeconds
+          value: 600
+
+  - it: omits the deadline entirely when it is nulled out, restoring the unbounded behaviour
+    set:
+      migrationJob.activeDeadlineSeconds: null
+    asserts:
+      - notExists:
+          path: spec.activeDeadlineSeconds

helm/litellm/values.yaml

@@ -56,6 +56,15 @@ migrationJob:
   enabled: true
   backoffLimit: 4
   ttlSecondsAfterFinished: 120
+  # Wall-clock budget for the whole Job, shared across every `backoffLimit`
+  # retry rather than granted per attempt. Without it a migration that blocks
+  # on the database never fails, and because this is a pre-upgrade hook the
+  # release waits on it forever: `helm upgrade` and any GitOps controller
+  # driving it stop reconciling the whole chart until someone deletes the Job
+  # by hand. A migration that has exhausted its retries is not going to
+  # succeed on the next one, so failing is strictly better than hanging.
+  # Set to null to opt out and restore the unbounded behaviour.
+  activeDeadlineSeconds: 1800
   resources: {}
   # ServiceAccount for the Job pod only.
   #

litellm-proxy-extras/litellm_proxy_extras/migrations/20260817143646_add_daily_guardrail_usage_units/migration.sql

@@ -0,0 +1,16 @@
+-- CreateTable
+CREATE TABLE "LiteLLM_DailyGuardrailUsageUnits" (
+    "guardrail_id" TEXT NOT NULL,
+    "date" TEXT NOT NULL,
+    "team_id" TEXT NOT NULL,
+    "api_key" TEXT NOT NULL,
+    "usage_unit" TEXT NOT NULL,
+    "units" BIGINT NOT NULL DEFAULT 0,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "LiteLLM_DailyGuardrailUsageUnits_pkey" PRIMARY KEY ("guardrail_id","date","team_id","api_key","usage_unit")
+);
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_DailyGuardrailUsageUnits_date_idx" ON "LiteLLM_DailyGuardrailUsageUnits"("date");

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -1069,6 +1069,21 @@ model LiteLLM_DailyGuardrailMetrics {
   @@index([guardrail_id])
 }
 
+// Daily guardrail billable usage units (one row per guardrail/day/team/key/unit type)
+model LiteLLM_DailyGuardrailUsageUnits {
+  guardrail_id String
+  date         String   // YYYY-MM-DD
+  team_id      String   // empty string when the request had no team
+  api_key      String   // hashed virtual key; empty string when unknown
+  usage_unit   String   // provider counter name, e.g. Bedrock's contentPolicyUnits
+  units        BigInt   @default(0)
+  created_at   DateTime @default(now())
+  updated_at   DateTime @updatedAt
+
+  @@id([guardrail_id, date, team_id, api_key, usage_unit])
+  @@index([date])
+}
+
 // Daily policy metrics for usage dashboard (one row per policy per day)
 model LiteLLM_DailyPolicyMetrics {
   policy_id           String

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.86"
+version = "0.4.87"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.86"
+version = "0.4.87"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm/__init__.py

@@ -26,6 +26,7 @@ def _dev_env_hot_reload_enabled() -> bool:
 if os.getenv("LITELLM_MODE", "DEV") == "DEV":
     _dotenv.load_dotenv(override=_dev_env_hot_reload_enabled())
 
+from collections.abc import Sequence
 from typing import (
     Any,
     Callable,
@@ -217,6 +218,9 @@ add_user_information_to_llm_headers: Optional[bool] = (
 overwrite_user_with_key_hash: bool = (
     False  # force the outgoing `user` param to the hashed api key, so providers see a stable, tamper-proof id
 )
+bedrock_request_metadata_fields: Optional[Sequence[str]] = (
+    None  # allow-list of `user_api_key_*` fields (+ `spend_logs_metadata`) sent as Bedrock `requestMetadata`
+)
 store_audit_logs = False  # Enterprise feature, allow users to see audit logs
 skip_system_message_in_guardrail: bool = False
 skip_tool_message_in_guardrail: bool = False
@@ -788,6 +792,8 @@ def _populate_provider_model_sets(model_cost_map: Dict) -> None:
             nlp_cloud_models.add(key)
         elif value.get("litellm_provider") == "aleph_alpha":
             aleph_alpha_models.add(key)
+        elif value.get("litellm_provider") == "bedrock" and value.get("mode") == "guardrail":
+            pass
         elif value.get("litellm_provider") == "bedrock" and not is_bedrock_pricing_only_model(key):
             bedrock_models.add(key)
         elif value.get("litellm_provider") == "bedrock_converse":

litellm/_logging.py

@@ -10,7 +10,7 @@ from typing import Any, Final
 import litellm
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
-from litellm.litellm_core_utils.secret_redaction import redact_string
+from litellm.litellm_core_utils.secret_redaction import redact_string, redact_structured_value
 
 set_verbose = False
 
@@ -59,6 +59,12 @@ def _redact_string(value: str) -> str:
     return redact_string(value)
 
 
+def _redact_structured_value(key: str | None, value: str) -> str:
+    if not _ENABLE_SECRET_REDACTION:
+        return value
+    return redact_structured_value(key, value)
+
+
 def redact_secrets(value: str) -> str:
     """Public API: redact known secret/credential patterns from an arbitrary string.
 
@@ -265,7 +271,7 @@ class JsonFormatter(Formatter):
         if record.exc_info:
             json_record["stacktrace"] = record.exc_text or self.formatException(record.exc_info)
 
-        return safe_dumps(json_record)
+        return safe_dumps(json_record, value_transform=_redact_structured_value)
 
 
 class CorrelationPlainFormatter(logging.Formatter):
@@ -276,7 +282,7 @@ class CorrelationPlainFormatter(logging.Formatter):
     """
 
     def format(self, record: logging.LogRecord) -> str:
-        formatted: Final = super().format(record)
+        formatted: Final = _redact_string(super().format(record))
         trace_id: Final = getattr(record, "trace_id", None)
         session_id: Final = getattr(record, "session_id", None)
         if not trace_id and not session_id:

litellm/batches/batch_utils.py

@@ -48,6 +48,7 @@ async def _handle_completed_batch(
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"],
     model_name: str | None = None,
     litellm_params: dict | None = None,
+    model_info: ModelInfo | None = None,
 ) -> tuple[float, Usage, list[str]]:
     """Fetch a completed batch's output file and aggregate its cost, usage, and
     models in a single pass over the JSONL lines, so the parsed file content is
@@ -58,7 +59,21 @@ async def _handle_completed_batch(
         custom_llm_provider: The LLM provider
         model_name: Optional model name
         litellm_params: Optional litellm parameters containing credentials (api_key, api_base, etc.)
+        model_info: Optional deployment-level model info with custom pricing,
+            threaded through so a deployment's configured rates win over the
+            global cost map.
     """
+    # A completed batch whose request lines all failed has no output file - the
+    # results are written to a separate error_file_id and output_file_id is None.
+    # There is nothing to price or measure, so report an empty result set instead
+    # of calling _fetch_batch_output_file_content, which raises on a missing
+    # output file. Without this guard the logging worker crashes on every
+    # aretrieve_batch poll and the completed batch's zero-cost accounting is lost.
+    # The generic retrieval helper keeps raising for callers that explicitly ask
+    # for a missing output file.
+    if batch.output_file_id is None:
+        return 0.0, Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0), []
+
     file_content = await _fetch_batch_output_file_content(batch, custom_llm_provider, litellm_params=litellm_params)
 
     if (
@@ -75,6 +90,7 @@ async def _handle_completed_batch(
         entries=_iter_batch_input_entries(file_content),
         custom_llm_provider=custom_llm_provider,
         model_name=model_name,
+        model_info=model_info,
     )
 
 
@@ -430,11 +446,23 @@ def _get_batch_job_usage_from_response_body(response_body: dict, custom_llm_prov
     """
     if custom_llm_provider in ("anthropic", "bedrock"):
         from litellm.llms.anthropic.chat.transformation import AnthropicConfig
+        from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
 
-        return AnthropicConfig().calculate_usage(
-            usage_object=response_body.get("usage", None) or {},
+        usage_object: Final = response_body.get("usage", None) or {}
+        if custom_llm_provider == "bedrock" and AmazonConverseConfig.is_converse_usage_shape(usage_object):
+            return AmazonConverseConfig().usage_from_batch_output(usage_object)
+        anthropic_usage: Final = AnthropicConfig().calculate_usage(
+            usage_object=usage_object,
             reasoning_content=None,
         )
+        if usage_object and anthropic_usage.total_tokens == 0:
+            verbose_logger.warning(
+                "batch output line reported usage this parser does not understand, so it will be billed at $0. "
+                "provider=%s usage_keys=%s",
+                custom_llm_provider,
+                sorted(usage_object.keys()),
+            )
+        return anthropic_usage
     from litellm.responses.utils import ResponseAPILoggingUtils
 
     _usage_dict: Final = response_body.get("usage", None) or {}

litellm/batches/main.py

@@ -107,7 +107,7 @@ async def acreate_batch(
     completion_window: Literal["24h"],
     endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions", "/v1/responses"],
     input_file_id: str,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy"] = "openai",
     metadata: dict[str, str] | None = None,
     extra_headers: dict[str, str] | None = None,
     extra_body: dict[str, str] | None = None,
@@ -157,7 +157,7 @@ def create_batch(
     completion_window: Literal["24h"],
     endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions", "/v1/responses"],
     input_file_id: str,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy"] = "openai",
     metadata: dict[str, str] | None = None,
     extra_headers: dict[str, str] | None = None,
     extra_body: dict[str, str] | None = None,
@@ -339,7 +339,9 @@ def create_batch(
 @client
 async def aretrieve_batch(
     batch_id: str,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "anthropic"] = "openai",
+    custom_llm_provider: Literal[
+        "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic"
+    ] = "openai",
     metadata: dict[str, str] | None = None,
     extra_headers: dict[str, str] | None = None,
     extra_body: dict[str, str] | None = None,
@@ -385,7 +387,9 @@ def _handle_retrieve_batch_providers_without_provider_config(
     litellm_params: dict,
     _retrieve_batch_request: RetrieveBatchRequest,
     _is_async: bool,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "anthropic"] = "openai",
+    custom_llm_provider: Literal[
+        "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic"
+    ] = "openai",
     logging_obj: Any | None = None,
 ):
     api_base: str | None = None
@@ -508,7 +512,9 @@ def _handle_retrieve_batch_providers_without_provider_config(
 @client
 def retrieve_batch(
     batch_id: str,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "anthropic"] = "openai",
+    custom_llm_provider: Literal[
+        "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic"
+    ] = "openai",
     metadata: dict[str, str] | None = None,
     extra_headers: dict[str, str] | None = None,
     extra_body: dict[str, str] | None = None,
@@ -826,7 +832,7 @@ def list_batches(
 async def acancel_batch(
     batch_id: str,
     model: str | None = None,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "litellm_proxy"] = "openai",
     metadata: dict[str, str] | None = None,
     extra_headers: dict[str, str] | None = None,
     extra_body: dict[str, str] | None = None,
@@ -872,7 +878,7 @@ async def acancel_batch(
 def cancel_batch(
     batch_id: str,
     model: str | None = None,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"] | str = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "litellm_proxy"] | str = "openai",
     metadata: dict[str, str] | None = None,
     extra_headers: dict[str, str] | None = None,
     extra_body: dict[str, str] | None = None,
@@ -993,9 +999,14 @@ def cancel_batch(
                 timeout=timeout,
                 max_retries=optional_params.max_retries,
             )
+        elif custom_llm_provider == "bedrock":
+            response = BedrockBatchesHandler.cancel_batch(
+                batch_id=batch_id,
+                **kwargs,
+            )
         else:
             raise litellm.exceptions.BadRequestError(
-                message=f"LiteLLM doesn't support {custom_llm_provider} for 'cancel_batch'. Only 'openai', 'azure', and 'vertex_ai' are supported.",
+                message=f"LiteLLM doesn't support {custom_llm_provider} for 'cancel_batch'. Only 'openai', 'azure', 'vertex_ai', and 'bedrock' are supported.",
                 model="n/a",
                 llm_provider=custom_llm_provider,
                 response=httpx.Response(

litellm/caching/_embedding_router.py

@@ -12,8 +12,11 @@ This module is dependency-injected: callers pass the proxy ``llm_router`` and
 
 from __future__ import annotations
 
+from collections.abc import Sequence
 from typing import TYPE_CHECKING, Any, Final
 
+import litellm
+
 if TYPE_CHECKING:
     from litellm.router import Router
 
@@ -41,3 +44,28 @@ def build_router_embedding_metadata(
     metadata: Final[dict[str, Any]] = dict(request_metadata or {})
     metadata["semantic-cache-embedding"] = True
     return metadata
+
+
+def resolve_embedding_max_input_tokens(
+    configured_max_input_tokens: int | None,
+    embedding_model: str,
+    router: Router | None,
+) -> int | None:
+    """Explicit cache setting first, else the Router deployment's configured ``max_input_tokens``."""
+    if configured_max_input_tokens is not None:
+        return configured_max_input_tokens
+    if router is None:
+        return None
+    deployment_max_input_tokens, _ = router.get_configured_token_limits(embedding_model)
+    return deployment_max_input_tokens
+
+
+def truncate_embedding_input(prompt: str, embedding_model: str, max_input_tokens: int | None) -> str:
+    """Keep only the first ``max_input_tokens`` tokens of ``prompt`` for the embedding call."""
+    if max_input_tokens is None:
+        return prompt
+    tokens: Final[Sequence[int]] = litellm.encode(model=embedding_model, text=prompt)
+    if len(tokens) <= max_input_tokens:
+        return prompt
+    truncated: Final[str] = litellm.decode(model=embedding_model, tokens=tokens[:max_input_tokens])
+    return truncated

litellm/caching/caching.py

@@ -97,6 +97,7 @@ class Cache:
         qdrant_quantization_config: str | None = None,
         qdrant_semantic_cache_embedding_model: str = "text-embedding-ada-002",
         qdrant_semantic_cache_vector_size: int | None = None,
+        semantic_cache_embedding_max_input_tokens: int | None = None,
         # GCP IAM authentication parameters
         gcp_service_account: str | None = None,
         gcp_ssl_ca_certs: str | None = None,
@@ -122,6 +123,7 @@ class Cache:
             qdrant_api_key (str, optional): The api_key for the local or cloud qdrant cluster.
             qdrant_collection_name (str, optional): The name for your qdrant collection. Required if type is "qdrant-semantic".
             similarity_threshold (float, optional): The similarity threshold for semantic-caching, Required if type is "redis-semantic" or "qdrant-semantic".
+            semantic_cache_embedding_max_input_tokens (int, optional): Truncate prompts to this many tokens before embedding them for semantic caching. Defaults to the embedding deployment's configured max_input_tokens.
 
             # Disk Cache Args
             disk_cache_dir (str, optional): The directory for the disk cache. Defaults to None.
@@ -192,6 +194,7 @@ class Cache:
                 similarity_threshold=similarity_threshold,
                 embedding_model=redis_semantic_cache_embedding_model,
                 index_name=redis_semantic_cache_index_name,
+                embedding_max_input_tokens=semantic_cache_embedding_max_input_tokens,
                 **kwargs,
             )
         elif type == LiteLLMCacheType.VALKEY_SEMANTIC:
@@ -207,6 +210,7 @@ class Cache:
                 embedding_model=valkey_semantic_cache_embedding_model,
                 index_name=valkey_semantic_cache_index_name,
                 startup_nodes=redis_startup_nodes,
+                embedding_max_input_tokens=semantic_cache_embedding_max_input_tokens,
                 **kwargs,
             )
         elif type == LiteLLMCacheType.QDRANT_SEMANTIC:
@@ -218,6 +222,7 @@ class Cache:
                 quantization_config=qdrant_quantization_config,
                 embedding_model=qdrant_semantic_cache_embedding_model,
                 vector_size=qdrant_semantic_cache_vector_size,
+                embedding_max_input_tokens=semantic_cache_embedding_max_input_tokens,
             )
         elif type == LiteLLMCacheType.LOCAL:
             self.cache = InMemoryCache()

litellm/caching/qdrant_semantic_cache.py

@@ -12,7 +12,7 @@ import ast
 import asyncio
 import json
 import os
-from typing import Any, Final, cast
+from typing import TYPE_CHECKING, Any, Final, cast
 
 import litellm
 from litellm._logging import print_verbose
@@ -22,12 +22,21 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
 )
 from litellm.types.utils import EmbeddingResponse
 
-from ._embedding_router import build_router_embedding_metadata, resolve_embedding_router
+from ._embedding_router import (
+    build_router_embedding_metadata,
+    resolve_embedding_max_input_tokens,
+    resolve_embedding_router,
+    truncate_embedding_input,
+)
 from .base_cache import BaseCache
 
+if TYPE_CHECKING:
+    from litellm.router import Router
+
 
 class QdrantSemanticCache(BaseCache):
     CACHE_KEY_FIELD_NAME = "litellm_cache_key"
+    embedding_max_input_tokens: int | None = None
 
     def __init__(
         self,
@@ -39,6 +48,7 @@ class QdrantSemanticCache(BaseCache):
         embedding_model="text-embedding-ada-002",
         host_type=None,
         vector_size=None,
+        embedding_max_input_tokens: int | None = None,
     ):
         from litellm.llms.custom_httpx.http_handler import (
             _get_httpx_client,
@@ -57,6 +67,7 @@ class QdrantSemanticCache(BaseCache):
             raise Exception("similarity_threshold must be provided, passed None")
         self.similarity_threshold = similarity_threshold
         self.embedding_model = embedding_model
+        self.embedding_max_input_tokens = embedding_max_input_tokens
         self.vector_size = vector_size if vector_size is not None else QDRANT_VECTOR_SIZE
         headers = {}
 
@@ -188,6 +199,13 @@ class QdrantSemanticCache(BaseCache):
         cached_key: Final = payload.get(self.CACHE_KEY_FIELD_NAME)
         return cached_key is not None and str(cached_key) == str(key)
 
+    def _embedding_input(self, prompt: str, router: "Router | None") -> str:
+        return truncate_embedding_input(
+            prompt,
+            self.embedding_model,
+            resolve_embedding_max_input_tokens(self.embedding_max_input_tokens, self.embedding_model, router),
+        )
+
     def _get_embedding(self, prompt: str, metadata: dict[str, Any] | None = None) -> EmbeddingResponse:
         """Embed via the proxy Router when it serves the model, else direct."""
         try:
@@ -197,16 +215,17 @@ class QdrantSemanticCache(BaseCache):
             llm_router = None
 
         router: Final = resolve_embedding_router(self.embedding_model, llm_router, llm_model_list)
+        embedding_input: Final = self._embedding_input(prompt, router)
         if router is not None:
             return router.embedding(
                 model=self.embedding_model,
-                input=prompt,
+                input=embedding_input,
                 cache={"no-store": True, "no-cache": True},
                 metadata=build_router_embedding_metadata(metadata),
             )
         return litellm.embedding(
             model=self.embedding_model,
-            input=prompt,
+            input=embedding_input,
             cache={"no-store": True, "no-cache": True},
         )
 
@@ -218,17 +237,18 @@ class QdrantSemanticCache(BaseCache):
             llm_router = None
 
         router: Final = resolve_embedding_router(self.embedding_model, llm_router, llm_model_list)
+        embedding_input: Final = self._embedding_input(prompt, router)
         if router is not None:
             return await router.aembedding(
                 model=self.embedding_model,
-                input=prompt,
+                input=embedding_input,
                 cache={"no-store": True, "no-cache": True},
                 metadata=build_router_embedding_metadata(metadata),
             )
 
         return await litellm.aembedding(
             model=self.embedding_model,
-            input=prompt,
+            input=embedding_input,
             cache={"no-store": True, "no-cache": True},
         )
 

litellm/caching/redis_cache.py

@@ -49,7 +49,7 @@ if TYPE_CHECKING:
     cluster_pipeline = ClusterPipeline
     async_redis_client = Redis
     async_redis_cluster_client = RedisCluster
-    Span = _Span | Any
+    Span = _Span
 else:
     pipeline = Any
     cluster_pipeline = Any
@@ -625,7 +625,11 @@ class RedisCache(BaseCache):
             f"{self.namespace}-{hashlib.sha256(script.encode()).hexdigest()[:16]}"
         )
 
-        async def run_script(keys: Sequence[str], args: Sequence[Any], client: Any = None) -> Any:
+        async def run_script(
+            keys: Sequence[str],
+            args: Sequence[str | bytes | int | float],
+            client: object = None,
+        ) -> object:
             async def execute() -> object:
                 executor: Callable[..., Awaitable[Any]] | None = litellm.in_memory_llm_clients_cache.get_cache(
                     key=script_cache_key
@@ -650,7 +654,11 @@ class RedisCache(BaseCache):
         if hasattr(_redis_client, "register_script"):
             registered_script: Final = _redis_client.register_script(script)
 
-            async def standalone_executor(keys: Sequence[str], args: Sequence[Any], client: Any = None) -> Any:
+            async def standalone_executor(
+                keys: Sequence[str],
+                args: Sequence[str | bytes | int | float],
+                client: object = None,
+            ) -> object:
                 namespaced_keys: Final = tuple(self.check_and_fix_namespace(key=key) for key in keys)
                 return await registered_script(keys=namespaced_keys, args=args, client=client)
 
@@ -659,7 +667,11 @@ class RedisCache(BaseCache):
         if hasattr(_redis_client, "script_load"):
             script_sha: Final = _redis_client.script_load(script)
 
-            async def cluster_executor(keys: Sequence[str], args: Sequence[Any], client: Any = None) -> Any:
+            async def cluster_executor(
+                keys: Sequence[str],
+                args: Sequence[str | bytes | int | float],
+                client: object = None,
+            ) -> object:
                 namespaced_keys: Final = tuple(self.check_and_fix_namespace(key=key) for key in keys)
                 return await _redis_client.evalsha(script_sha, len(namespaced_keys), *namespaced_keys, *args)
 
@@ -757,7 +769,7 @@ class RedisCache(BaseCache):
     async def _pipeline_helper(
         self,
         pipe: pipeline | cluster_pipeline,
-        cache_list: list[tuple[Any, Any]],
+        cache_list: Sequence[tuple[str, object]],
         ttl: float | None,
     ) -> list:
         """
@@ -783,7 +795,9 @@ class RedisCache(BaseCache):
         return results
 
     @_redis_circuit_breaker_guard
-    async def async_set_cache_pipeline(self, cache_list: list[tuple[Any, Any]], ttl: float | None = None, **kwargs):
+    async def async_set_cache_pipeline(
+        self, cache_list: Sequence[tuple[str, object]], ttl: float | None = None, **kwargs
+    ):
         """
         Use Redis Pipelines for bulk write operations
         """
@@ -795,7 +809,7 @@ class RedisCache(BaseCache):
         start_time: Final = time.time()
 
         print_verbose(f"Set Async Redis Cache: key list: {cache_list}\nttl={ttl}, redis_version={self.redis_version}")
-        cache_value: Final[Any] = None
+        cache_value: Final = None
         try:
             async with _redis_client.pipeline(transaction=False) as pipe:
                 results: Final = await self._pipeline_helper(pipe, cache_list, ttl)
@@ -1074,7 +1088,7 @@ class RedisCache(BaseCache):
             # NON blocking - notify users Redis is throwing an exception
             verbose_logger.error("litellm.caching.caching: get() - Got exception from REDIS: ", e)
 
-    def _run_redis_mget_operation(self, keys: list[str]) -> list[Any]:
+    def _run_redis_mget_operation(self, keys: list[str]) -> Sequence[bytes | str | None]:
         """
         Wrapper to call `mget` on the redis client
 
@@ -1082,7 +1096,7 @@ class RedisCache(BaseCache):
         """
         return self.redis_client.mget(keys=keys)
 
-    async def _async_run_redis_mget_operation(self, keys: list[str]) -> list[Any]:
+    async def _async_run_redis_mget_operation(self, keys: list[str]) -> Sequence[bytes | str | None]:
         """
         Wrapper to call `mget` on the redis client
 
@@ -1115,7 +1129,7 @@ class RedisCache(BaseCache):
                 cache_key = self.check_and_fix_namespace(key=cache_key or "")
                 _keys.append(cache_key)
             start_time: Final = time.time()
-            results: Final[list] = self._run_redis_mget_operation(keys=_keys)
+            results: Final = self._run_redis_mget_operation(keys=_keys)
             end_time: Final = time.time()
             _duration: Final = end_time - start_time
             self.service_logger_obj.service_success_hook(
@@ -1522,7 +1536,7 @@ class RedisCache(BaseCache):
     async def async_rpush(
         self,
         key: str,
-        values: list[Any],
+        values: Sequence[str | bytes | int | float],
         parent_otel_span: Span | None = None,
         **kwargs,
     ) -> int:

litellm/caching/redis_semantic_cache.py

@@ -14,7 +14,7 @@ import asyncio
 import json
 import os
 from collections.abc import Callable, Mapping
-from typing import Any, Final, cast
+from typing import TYPE_CHECKING, Any, Final, cast
 
 import litellm
 from litellm._logging import print_verbose, verbose_logger
@@ -23,9 +23,17 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
 )
 from litellm.types.utils import EmbeddingResponse
 
-from ._embedding_router import build_router_embedding_metadata, resolve_embedding_router
+from ._embedding_router import (
+    build_router_embedding_metadata,
+    resolve_embedding_max_input_tokens,
+    resolve_embedding_router,
+    truncate_embedding_input,
+)
 from .base_cache import BaseCache
 
+if TYPE_CHECKING:
+    from litellm.router import Router
+
 
 class RedisSemanticCache(BaseCache):
     """
@@ -38,6 +46,7 @@ class RedisSemanticCache(BaseCache):
 
     DEFAULT_REDIS_INDEX_NAME: str = "litellm_semantic_cache_index"
     CACHE_KEY_FIELD_NAME: str = "litellm_cache_key"
+    embedding_max_input_tokens: int | None = None
 
     def __init__(
         self,
@@ -48,6 +57,7 @@ class RedisSemanticCache(BaseCache):
         similarity_threshold: float | None = None,
         embedding_model: str = "text-embedding-ada-002",
         index_name: str | None = None,
+        embedding_max_input_tokens: int | None = None,
         **kwargs: object,
     ):
         """
@@ -62,6 +72,8 @@ class RedisSemanticCache(BaseCache):
                 where 1.0 requires exact matches and 0.0 accepts any match
             embedding_model: Model to use for generating embeddings
             index_name: Name for the Redis index
+            embedding_max_input_tokens: Truncate prompts to this many tokens before
+                embedding; defaults to the Router deployment's configured max_input_tokens
             ttl: Default time-to-live for cache entries in seconds
             **kwargs: Additional arguments passed to the Redis client
 
@@ -86,6 +98,7 @@ class RedisSemanticCache(BaseCache):
         # While similarity: 1 = most similar, 0 = least similar
         self.distance_threshold = 1 - similarity_threshold
         self.embedding_model = embedding_model
+        self.embedding_max_input_tokens = embedding_max_input_tokens
 
         # Set up Redis connection
         if redis_url is None:
@@ -307,6 +320,13 @@ class RedisSemanticCache(BaseCache):
             return dict_method()
         return value
 
+    def _embedding_input(self, prompt: str, router: "Router | None") -> str:
+        return truncate_embedding_input(
+            prompt,
+            self.embedding_model,
+            resolve_embedding_max_input_tokens(self.embedding_max_input_tokens, self.embedding_model, router),
+        )
+
     def _get_embedding(self, prompt: str, metadata: dict[str, Any] | None = None) -> list[float]:
         """
         Routes through the proxy Router when the embedding model is a Router
@@ -320,12 +340,13 @@ class RedisSemanticCache(BaseCache):
             llm_router = None
 
         router: Final = resolve_embedding_router(self.embedding_model, llm_router, llm_model_list)
+        embedding_input: Final = self._embedding_input(prompt, router)
         if router is not None:
             embedding_response = cast(
                 EmbeddingResponse,
                 router.embedding(
                     model=self.embedding_model,
-                    input=prompt,
+                    input=embedding_input,
                     cache={"no-store": True, "no-cache": True},
                     metadata=build_router_embedding_metadata(metadata),
                 ),
@@ -335,7 +356,7 @@ class RedisSemanticCache(BaseCache):
                 EmbeddingResponse,
                 litellm.embedding(
                     model=self.embedding_model,
-                    input=prompt,
+                    input=embedding_input,
                     cache={"no-store": True, "no-cache": True},
                 ),
             )
@@ -490,18 +511,19 @@ class RedisSemanticCache(BaseCache):
             llm_router = None
 
         router: Final = resolve_embedding_router(self.embedding_model, llm_router, llm_model_list)
+        embedding_input: Final = self._embedding_input(prompt, router)
         try:
             if router is not None:
                 embedding_response = await router.aembedding(
                     model=self.embedding_model,
-                    input=prompt,
+                    input=embedding_input,
                     cache={"no-store": True, "no-cache": True},
                     metadata=build_router_embedding_metadata(metadata),
                 )
             else:
                 embedding_response = await litellm.aembedding(
                     model=self.embedding_model,
-                    input=prompt,
+                    input=embedding_input,
                     cache={"no-store": True, "no-cache": True},
                 )
             return embedding_response["data"][0]["embedding"]

litellm/caching/valkey_semantic_cache.py

@@ -17,7 +17,6 @@ RedisSemanticCache since those are backend agnostic.
 import asyncio
 import hashlib
 import os
-import struct
 from dataclasses import dataclass
 from typing import Any, Final
 
@@ -29,6 +28,7 @@ from redis.commands.search.query import Query
 
 from litellm._logging import print_verbose
 from litellm._uuid import uuid
+from litellm.llms.valkey.common_utils import build_valkey_url, pack_vector
 
 from .redis_semantic_cache import RedisSemanticCache
 
@@ -61,6 +61,7 @@ class ValkeySemanticCache(RedisSemanticCache):
         startup_nodes: list | None = None,
         sync_client: Redis | None = None,
         async_client: AsyncRedis | None = None,
+        embedding_max_input_tokens: int | None = None,
         **kwargs: Any,
     ):
         if similarity_threshold is None:
@@ -78,6 +79,7 @@ class ValkeySemanticCache(RedisSemanticCache):
 
         self.similarity_threshold = similarity_threshold
         self.embedding_model = embedding_model
+        self.embedding_max_input_tokens = embedding_max_input_tokens
         self.index_name = index_name or self.DEFAULT_VALKEY_INDEX_NAME
         self.key_prefix = f"{self.index_name}:"
         self._index_dim: int | None = None
@@ -92,19 +94,17 @@ class ValkeySemanticCache(RedisSemanticCache):
 
     @staticmethod
     def _build_valkey_url(host: str | None, port: str | None, password: str | None, ssl: bool = False) -> str:
-        host = host or os.environ.get("VALKEY_HOST") or os.environ.get("REDIS_HOST")
-        port = port or os.environ.get("VALKEY_PORT") or os.environ.get("REDIS_PORT")
-        password = password or os.environ.get("VALKEY_PASSWORD") or os.environ.get("REDIS_PASSWORD")
+        resolved_host: Final = host or os.environ.get("VALKEY_HOST") or os.environ.get("REDIS_HOST")
+        resolved_port: Final = port or os.environ.get("VALKEY_PORT") or os.environ.get("REDIS_PORT")
+        resolved_password: Final = password or os.environ.get("VALKEY_PASSWORD") or os.environ.get("REDIS_PASSWORD")
 
-        if not host or not port:
+        if not resolved_host or not resolved_port:
             raise ValueError(
                 "Missing required Valkey configuration. Provide host and port "
                 "(or VALKEY_HOST/VALKEY_PORT), or pass redis_url."
             )
 
-        credentials: Final = f":{password}@" if password else ""
-        scheme: Final = "rediss" if ssl else "redis"
-        return f"{scheme}://{credentials}{host}:{port}"
+        return build_valkey_url(host=resolved_host, port=resolved_port, password=resolved_password, ssl=ssl)
 
     @classmethod
     def _scope_tag(cls, key: str) -> str:
@@ -116,7 +116,7 @@ class ValkeySemanticCache(RedisSemanticCache):
 
     @staticmethod
     def _embedding_to_bytes(embedding: list[float]) -> bytes:
-        return struct.pack(f"<{len(embedding)}f", *embedding)
+        return pack_vector(embedding)
 
     def _index_schema(self, dim: int) -> tuple[TagField, VectorField]:
         return (

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -185,6 +185,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         if not isinstance(tool_choice, dict):
             return tool_choice
         choice_type: Final = tool_choice.get("type")
+        if isinstance(choice_type, str) and choice_type in ("auto", "none", "required"):
+            return choice_type
         if choice_type not in ("function", "custom"):
             return tool_choice
         if isinstance(tool_choice.get("name"), str) and tool_choice.get("name"):

litellm/constants.py

@@ -1,5 +1,6 @@
 import os
 import sys
+from types import MappingProxyType
 from typing import Final, Literal
 
 from litellm.litellm_core_utils.env_utils import get_env_int, get_env_int_or_none
@@ -1487,6 +1488,7 @@ WEEKLY_SPEND_REPORT_JOB_ID: Final = "weekly_spend_report_job"
 MONTHLY_SPEND_REPORT_JOB_ID: Final = "monthly_spend_report_job"
 PROMETHEUS_FALLBACK_STATS_JOB_ID: Final = "prometheus_fallback_stats_job"
 SLACK_DAILY_REPORT_LOCK_ID: Final = "slack_daily_report"
+SLACK_MODEL_DEPRECATION_LOCK_ID: Final = "slack_model_deprecation_warning"
 SPEND_LOG_RUN_LOOPS: Final = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500))
 SPEND_LOG_CLEANUP_BATCH_SIZE: Final = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000))
 SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES = int(os.getenv("SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3))
@@ -1593,6 +1595,13 @@ DEFAULT_MCP_ACCESS_GROUP_NEGATIVE_CACHE_TTL: Final = 10
 # in a single ``/{name1,name2,...}/mcp`` URL. Bounds the per-request DB / cache
 # fan-out an authenticated caller can trigger by stuffing the path with tokens.
 DEFAULT_MCP_NAMESPACE_CSV_MAX_TOKENS: Final = 16
+# Ceilings on the cached auth registries; larger tables fall back to per-row lookups
+# instead of holding an unbounded id set in every worker.
+TAG_REGISTRY_MAX_SIZE: Final = 5000
+END_USER_RESTRICTED_REGISTRY_MAX_SIZE: Final = 5000
+# How long a failed registry load is remembered as "unusable", so a degraded Postgres
+# is not re-scanned on every request on top of the per-id lookups it falls back to.
+REGISTRY_ERROR_NEGATIVE_CACHE_TTL: Final = 30
 
 # Sentry Scrubbing Configuration
 SENTRY_DENYLIST: Final = [
@@ -1756,3 +1765,7 @@ PTU_LAPSED_ALERT_LIMIT: Final[int] = 10
 # one run delete a charge another just wrote. A stale row is hours old and a concurrent
 # one is seconds old, so a few minutes separates them.
 PTU_PRUNE_SKEW_GRACE_SECONDS: Final[int] = 300
+
+# Shared read-only empty mapping, for defaulting optional Mapping parameters without
+# constructing a fresh mutable dict at each call site.
+EMPTY_MAPPING: Final = MappingProxyType({})

litellm/containers/main.py

@@ -1,9 +1,11 @@
 import asyncio
 import contextvars
 import json
-from collections.abc import Coroutine
+from collections.abc import Coroutine, Mapping
 from functools import partial
-from typing import Any, Final, Literal, overload
+from typing import Final, Literal, overload
+
+import httpx
 
 import litellm
 from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT
@@ -48,16 +50,16 @@ __all__ = [
 @client
 async def acreate_container(
     name: str,
-    expires_after: dict[str, Any] | None = None,
+    expires_after: Mapping[str, object] | None = None,
     file_ids: list[str] | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     # LiteLLM specific params,
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
     # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
     # The extra values given here take precedence over values defined on the client or passed to this method.
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
 ) -> ContainerObject:
     """Asynchronously calls the `create_container` function with the given arguments and keyword arguments.
@@ -120,9 +122,9 @@ async def acreate_container(
 @overload
 def create_container(
     name: str,
-    expires_after: dict[str, Any] | None = None,
+    expires_after: Mapping[str, object] | None = None,
     file_ids: list[str] | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -130,16 +132,16 @@ def create_container(
     *,
     acreate_container: Literal[True],
     **kwargs,
-) -> Coroutine[Any, Any, ContainerObject]:
+) -> Coroutine[object, object, ContainerObject]:
     ...
 
 
 @overload
 def create_container(
     name: str,
-    expires_after: dict[str, Any] | None = None,
+    expires_after: Mapping[str, object] | None = None,
     file_ids: list[str] | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -156,20 +158,20 @@ def create_container(
 @client
 def create_container(
     name: str,
-    expires_after: dict[str, Any] | None = None,
+    expires_after: Mapping[str, object] | None = None,
     file_ids: list[str] | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
     # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
     # The extra values given here take precedence over values defined on the client or passed to this method.
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
-) -> ContainerObject | Coroutine[Any, Any, ContainerObject]:
+) -> ContainerObject | Coroutine[object, object, ContainerObject]:
     """Create a container using the OpenAI Container API.
 
     Currently supports OpenAI
@@ -281,13 +283,13 @@ async def alist_containers(
     after: str | None = None,
     limit: int | None = None,
     order: str | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
     # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
     # The extra values given here take precedence over values defined on the client or passed to this method.
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
 ) -> ContainerListResponse:
     """Asynchronously list containers.
@@ -351,7 +353,7 @@ def list_containers(
     after: str | None = None,
     limit: int | None = None,
     order: str | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -359,7 +361,7 @@ def list_containers(
     *,
     alist_containers: Literal[True],
     **kwargs,
-) -> Coroutine[Any, Any, ContainerListResponse]:
+) -> Coroutine[object, object, ContainerListResponse]:
     ...
 
 
@@ -368,7 +370,7 @@ def list_containers(
     after: str | None = None,
     limit: int | None = None,
     order: str | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -387,18 +389,18 @@ def list_containers(
     after: str | None = None,
     limit: int | None = None,
     order: str | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
     # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
     # The extra values given here take precedence over values defined on the client or passed to this method.
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
-) -> ContainerListResponse | Coroutine[Any, Any, ContainerListResponse]:
+) -> ContainerListResponse | Coroutine[object, object, ContainerListResponse]:
     """List containers using the OpenAI Container API.
 
     Currently supports OpenAI
@@ -481,13 +483,13 @@ def list_containers(
 @client
 async def aretrieve_container(
     container_id: str,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
     # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
     # The extra values given here take precedence over values defined on the client or passed to this method.
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
 ) -> ContainerObject:
     """Asynchronously retrieve a container.
@@ -545,7 +547,7 @@ async def aretrieve_container(
 @overload
 def retrieve_container(
     container_id: str,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -553,14 +555,14 @@ def retrieve_container(
     *,
     aretrieve_container: Literal[True],
     **kwargs,
-) -> Coroutine[Any, Any, ContainerObject]:
+) -> Coroutine[object, object, ContainerObject]:
     ...
 
 
 @overload
 def retrieve_container(
     container_id: str,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -577,18 +579,18 @@ def retrieve_container(
 @client
 def retrieve_container(
     container_id: str,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
     # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
     # The extra values given here take precedence over values defined on the client or passed to this method.
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
-) -> ContainerObject | Coroutine[Any, Any, ContainerObject]:
+) -> ContainerObject | Coroutine[object, object, ContainerObject]:
     """Retrieve a container using the OpenAI Container API.
 
     Currently supports OpenAI
@@ -696,13 +698,13 @@ def retrieve_container(
 @client
 async def adelete_container(
     container_id: str,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
     # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
     # The extra values given here take precedence over values defined on the client or passed to this method.
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
 ) -> DeleteContainerResult:
     """Asynchronously delete a container.
@@ -760,7 +762,7 @@ async def adelete_container(
 @overload
 def delete_container(
     container_id: str,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -768,14 +770,14 @@ def delete_container(
     *,
     adelete_container: Literal[True],
     **kwargs,
-) -> Coroutine[Any, Any, DeleteContainerResult]:
+) -> Coroutine[object, object, DeleteContainerResult]:
     ...
 
 
 @overload
 def delete_container(
     container_id: str,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -792,18 +794,18 @@ def delete_container(
 @client
 def delete_container(
     container_id: str,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
     # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
     # The extra values given here take precedence over values defined on the client or passed to this method.
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
-) -> DeleteContainerResult | Coroutine[Any, Any, DeleteContainerResult]:
+) -> DeleteContainerResult | Coroutine[object, object, DeleteContainerResult]:
     """Delete a container using the OpenAI Container API.
 
     Currently supports OpenAI
@@ -914,11 +916,11 @@ async def alist_container_files(
     after: str | None = None,
     limit: int | None = None,
     order: str | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
 ) -> ContainerFileListResponse:
     """Asynchronously list files in a container.
@@ -985,7 +987,7 @@ def list_container_files(
     after: str | None = None,
     limit: int | None = None,
     order: str | None = None,
-    timeout=600,
+    timeout: float | httpx.Timeout = 600,
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -993,7 +995,7 @@ def list_container_files(
     *,
     alist_container_files: Literal[True],
     **kwargs,
-) -> Coroutine[Any, Any, ContainerFileListResponse]:
+) -> Coroutine[object, object, ContainerFileListResponse]:
     ...
 
 
@@ -1003,7 +1005,7 @@ def list_container_files(
     after: str | None = None,
     limit: int | None = None,
     order: str | None = None,
-    timeout=600,
+    timeout: float | httpx.Timeout = 600,
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -1023,16 +1025,16 @@ def list_container_files(
     after: str | None = None,
     limit: int | None = None,
     order: str | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
-) -> ContainerFileListResponse | Coroutine[Any, Any, ContainerFileListResponse]:
+) -> ContainerFileListResponse | Coroutine[object, object, ContainerFileListResponse]:
     """List files in a container using the OpenAI Container API.
 
     Currently supports OpenAI
@@ -1125,11 +1127,11 @@ def list_container_files(
 async def aupload_container_file(
     container_id: str,
     file: FileTypes,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
 ) -> ContainerFileObject:
     """Asynchronously upload a file to a container.
@@ -1211,7 +1213,7 @@ async def aupload_container_file(
 def upload_container_file(
     container_id: str,
     file: FileTypes,
-    timeout=600,
+    timeout: float | httpx.Timeout = 600,
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -1219,7 +1221,7 @@ def upload_container_file(
     *,
     aupload_container_file: Literal[True],
     **kwargs,
-) -> Coroutine[Any, Any, ContainerFileObject]:
+) -> Coroutine[object, object, ContainerFileObject]:
     ...
 
 
@@ -1227,7 +1229,7 @@ def upload_container_file(
 def upload_container_file(
     container_id: str,
     file: FileTypes,
-    timeout=600,
+    timeout: float | httpx.Timeout = 600,
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -1245,16 +1247,16 @@ def upload_container_file(
 def upload_container_file(
     container_id: str,
     file: FileTypes,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
-) -> ContainerFileObject | Coroutine[Any, Any, ContainerFileObject]:
+) -> ContainerFileObject | Coroutine[object, object, ContainerFileObject]:
     """Upload a file to a container using the OpenAI Container API.
 
     This endpoint allows uploading files directly to a container session,

litellm/cost_calculator.py

@@ -2160,7 +2160,7 @@ def batch_cost_calculator(
     output_cost_per_token: Final = model_info.get("output_cost_per_token")
     total_prompt_cost = 0.0
     total_completion_cost = 0.0
-    if input_cost_per_token_batches:
+    if input_cost_per_token_batches is not None:
         total_prompt_cost = usage.prompt_tokens * input_cost_per_token_batches
     elif input_cost_per_token:
         details: Final = parse_prompt_tokens_details(usage)
@@ -2180,7 +2180,7 @@ def batch_cost_calculator(
 
         cache_creation_cost: Final = model_info.get("cache_creation_input_token_cost") or input_cost_per_token
         total_prompt_cost += cache_creation_tokens * cache_creation_cost / 2
-    if output_cost_per_token_batches:
+    if output_cost_per_token_batches is not None:
         total_completion_cost = usage.completion_tokens * output_cost_per_token_batches
     elif output_cost_per_token:
         total_completion_cost = (

litellm/files/main.py

@@ -23,12 +23,15 @@ FileCreateProvider = Literal[
     "vertex_ai",
     "bedrock",
     "hosted_vllm",
+    "litellm_proxy",
     "manus",
     "anthropic",
 ]
-FileRetrieveProvider = Literal["openai", "azure", "gemini", "vertex_ai", "hosted_vllm", "manus", "anthropic"]
-FileDeleteProvider = Literal["openai", "azure", "gemini", "manus", "anthropic"]
-FileListProvider = Literal["openai", "azure", "manus", "anthropic"]
+FileRetrieveProvider = Literal[
+    "openai", "azure", "gemini", "vertex_ai", "hosted_vllm", "litellm_proxy", "manus", "anthropic"
+]
+FileDeleteProvider = Literal["openai", "azure", "gemini", "litellm_proxy", "manus", "anthropic"]
+FileListProvider = Literal["openai", "azure", "litellm_proxy", "manus", "anthropic"]
 import litellm
 from litellm import get_secret_str
 from litellm.files.streaming import FileContentStreamingResponse

litellm/files/types.py

@@ -1,7 +1,9 @@
 from collections.abc import AsyncIterator, Iterator
 from typing import Literal, NamedTuple
 
-FileContentProvider = Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "anthropic", "manus"]
+FileContentProvider = Literal[
+    "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "manus"
+]
 
 
 class FileContentStreamingResult(NamedTuple):

litellm/integrations/SlackAlerting/slack_alerting.py

@@ -5,6 +5,7 @@ import datetime
 import os
 import random
 import time
+from collections.abc import Callable
 from datetime import timedelta
 from typing import TYPE_CHECKING, Any, Final, Literal
 
@@ -17,7 +18,11 @@ import litellm.litellm_core_utils.litellm_logging
 import litellm.types
 from litellm._logging import verbose_logger, verbose_proxy_logger
 from litellm.caching.caching import DualCache
-from litellm.constants import HOURS_IN_A_DAY, SLACK_DAILY_REPORT_LOCK_ID
+from litellm.constants import (
+    HOURS_IN_A_DAY,
+    SLACK_DAILY_REPORT_LOCK_ID,
+    SLACK_MODEL_DEPRECATION_LOCK_ID,
+)
 from litellm.integrations.custom_batch_logger import CustomBatchLogger
 from litellm.integrations.SlackAlerting.budget_alert_types import get_budget_alert_type
 from litellm.integrations.SlackAlerting.hanging_request_check import (
@@ -45,6 +50,10 @@ from litellm.repositories.table_repositories import InvitationLinkRepository
 from litellm.repositories.team_repository import TeamRepository
 from litellm.repositories.user_repository import UserRepository
 from litellm.types.integrations.slack_alerting import *
+from litellm.types.proxy.model_deprecation import (
+    DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS,
+    DEPRECATION_IDLE_POLL_SECONDS,
+)
 
 from ..email_templates.templates import *
 from .batching_handler import send_to_webhook, squash_payloads
@@ -59,6 +68,12 @@ else:
     Router = Any
 
 
+def _proxy_llm_router() -> Router | None:
+    from litellm.proxy.proxy_server import llm_router
+
+    return llm_router
+
+
 class SlackAlerting(CustomBatchLogger):
     """
     Class for sending Slack Alerts
@@ -1044,6 +1059,99 @@ Model Info:
     async def model_removed_alert(self, model_name: str):
         pass
 
+    def _deprecation_alerts_enabled(self) -> bool:
+        return self.alerting is not None and AlertType.model_deprecation_warnings in self.alert_types
+
+    async def send_model_deprecation_alert(
+        self,
+        llm_router: Router | None = None,
+        pod_lock_manager: "PodLockManager | None" = None,
+    ) -> bool:
+        """Alert on the router's deprecated and imminent models, True when one was sent
+
+        The daily lock is claimed only once there is something to say, so an empty pass never blocks a
+        later real one, and a sent alert is stamped in the shared cache for a day so sibling pods stop asking
+        """
+        if not self._deprecation_alerts_enabled():
+            return False
+
+        from litellm.proxy.common_utils.model_deprecation import (
+            collect_model_deprecations,
+            format_deprecation_alert_message,
+        )
+
+        snapshot: Final = collect_model_deprecations(llm_router=llm_router)
+        message: Final = format_deprecation_alert_message(snapshot)
+        if message is None:
+            return False
+        if not await self._claimed_deprecation_alert_window(pod_lock_manager):
+            return False
+
+        level: Final[Literal["Low", "Medium", "High"]] = "High" if snapshot.deprecated else "Medium"
+
+        await self.send_alert(
+            message=message,
+            level=level,
+            alert_type=AlertType.model_deprecation_warnings,
+            alerting_metadata={  # mutable-ok: send_alert takes a dict payload
+                "deprecated_count": len(snapshot.deprecated),
+                "imminent_count": len(snapshot.imminent),
+                "upcoming_count": len(snapshot.upcoming),
+            },
+        )
+        await self.internal_usage_cache.async_set_cache(
+            key=SlackAlertingCacheKeys.deprecation_alert_sent_key.value,
+            value=time.time(),
+            ttl=DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS,
+        )
+        return True
+
+    async def _claimed_deprecation_alert_window(self, pod_lock_manager: "PodLockManager | None") -> bool:
+        """Without a redis backed lock there is no fleet to coordinate, so a lone pod always alerts"""
+        if pod_lock_manager is None:
+            return True
+        return (
+            await pod_lock_manager.acquire_lock(
+                cronjob_id=SLACK_MODEL_DEPRECATION_LOCK_ID,
+                ttl=DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS,
+                allow_reentrant=False,
+            )
+        ) is not False
+
+    async def _deprecation_alert_sent_within_a_day(self) -> bool:
+        return (
+            await self.internal_usage_cache.async_get_cache(key=SlackAlertingCacheKeys.deprecation_alert_sent_key.value)
+        ) is not None
+
+    async def _run_deprecation_alert_pass(
+        self, llm_router: Router | None, pod_lock_manager: "PodLockManager | None"
+    ) -> bool:
+        if llm_router is None or not self._deprecation_alerts_enabled():
+            return False
+        if await self._deprecation_alert_sent_within_a_day():
+            return False
+        return await self.send_model_deprecation_alert(llm_router=llm_router, pod_lock_manager=pod_lock_manager)
+
+    async def run_scheduled_deprecation_check(
+        self,
+        get_llm_router: Callable[[], Router | None] = _proxy_llm_router,
+        pod_lock_manager: "PodLockManager | None" = None,
+    ) -> None:
+        """Poll every pass for a loaded router, the alert being on, and no alert in the last day, then alert
+
+        A pass that could not alert (no router yet, alert type off, a sibling pod holds the daily lock, or a
+        redis blip at claim time) is retried on the next poll instead of costing a day, while a pass that
+        raised (a missing webhook, say) backs off a full day so a misconfiguration logs once, not every poll
+        """
+        while True:
+            try:
+                await self._run_deprecation_alert_pass(get_llm_router(), pod_lock_manager)
+            except Exception as e:  # noqa: BLE001  # a failed alert must not kill the loop
+                verbose_proxy_logger.exception("Error in model deprecation alert loop: %s", e)
+                await asyncio.sleep(DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS)
+                continue
+            await asyncio.sleep(DEPRECATION_IDLE_POLL_SECONDS)
+
     async def send_webhook_alert(self, webhook_event: WebhookEvent) -> bool:
         """
         Sends structured alert to webhook, if set.

litellm/integrations/custom_guardrail.py

@@ -102,6 +102,9 @@ class CustomGuardrail(CustomLogger):
     # If True, during_call runs async_moderation_hook instead of the unified apply_guardrail path.
     use_native_during_call_hook: ClassVar[bool] = False
 
+    # If True, every proxy lifecycle event runs this guardrail's own hooks, not apply_guardrail.
+    use_native_lifecycle_hooks: ClassVar[bool] = False
+
     records_own_guardrail_information: ClassVar[bool] = False
 
     def __init__(
@@ -632,7 +635,7 @@ class CustomGuardrail(CustomLogger):
         return type(self).apply_guardrail is not CustomGuardrail.apply_guardrail
 
     def _deployment_pre_call_target(self) -> "CustomLogger":
-        if not self.uses_apply_guardrail_interface():
+        if not self.uses_apply_guardrail_interface() or self.use_native_lifecycle_hooks:
             return self
         try:
             from litellm.proxy.utils import unified_guardrail

litellm/integrations/opentelemetry.py

@@ -1,5 +1,8 @@
 import os
-from collections.abc import Mapping
+import threading
+from collections import OrderedDict
+from collections.abc import Callable, Mapping
+from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass, field
 from datetime import datetime
 from typing import TYPE_CHECKING, Any, Final, TypedDict, cast
@@ -17,6 +20,7 @@ from litellm.integrations.opentelemetry_utils.gen_ai_semconv import (
     OTELSemconvCategory,
     parse_semconv_opt_in,
 )
+from litellm.integrations.otel.model.db_endpoint import db_span_attributes
 from litellm.integrations.otel.model.semconv import Metric
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.litellm_core_utils.secret_redaction import redact_string
@@ -83,6 +87,12 @@ class _ResponseWithUsageView(TypedDict, total=False):
     usage: "_UsageCompletionTokensView | None"
 
 
+# Cap on credential-scoped providers held at once; each one owns an exporter thread.
+_MAX_DYNAMIC_TRACER_PROVIDERS: Final = 256
+
+# Dedicated so a slow exporter shutdown cannot starve the shared logging executor.
+_PROVIDER_SHUTDOWN_EXECUTOR: Final = ThreadPoolExecutor(max_workers=4, thread_name_prefix="OtelProviderShutdown")
+
 LITELLM_TRACER_NAME: Final = os.getenv("OTEL_TRACER_NAME", "litellm")
 LITELLM_METER_NAME: Final = os.getenv("LITELLM_METER_NAME", "litellm")
 LITELLM_LOGGER_NAME: Final = os.getenv("LITELLM_LOGGER_NAME", "litellm")
@@ -227,6 +237,34 @@ def _freeze_for_dedupe(value: object, _depth: int = 0) -> HashableScope:
     return repr(value)
 
 
+def _shutdown_tracer_provider(provider: "_SDKTracerProvider") -> None:
+    """Flush and stop a dropped provider so its exporter thread is reclaimed."""
+    try:
+        provider.shutdown()
+    except Exception as e:  # noqa: BLE001  # exporter shutdown must not fail the request that dropped it
+        verbose_logger.debug("OpenTelemetry: error shutting down dropped tracer provider: %s", e)
+
+
+@dataclass(frozen=True, slots=True)
+class _CachedTracerProvider:
+    """A cached credential-scoped provider plus whether it may be shut down when dropped."""
+
+    provider: "_SDKTracerProvider"
+    owns_exporter: bool
+
+
+def _provider_owns_exporter(exporter: "str | _SpanExporter") -> bool:
+    """Whether a provider built for ``exporter`` may be shut down when it is dropped.
+
+    ``_get_span_processor`` builds a fresh exporter for a named kind, but wraps a
+    caller-supplied ``SpanExporter`` instance as-is, and that instance is shared with the
+    logger's own provider. Shutting a dropped provider down would then stop exporting for
+    the whole process. The shared case also uses ``SimpleSpanProcessor``, so it owns no
+    thread and there is nothing to reclaim.
+    """
+    return not hasattr(exporter, "export")
+
+
 @dataclass
 class OpenTelemetryConfig:
     exporter: str | SpanExporter = "console"
@@ -322,6 +360,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
         tracer_provider: object | None = None,
         logger_provider: object | None = None,
         meter_provider: object | None = None,
+        max_dynamic_tracer_providers: int = _MAX_DYNAMIC_TRACER_PROVIDERS,
         **kwargs,
     ):
         team_metadata_keys_override: Final = kwargs.pop("baggage_team_metadata_keys", None)
@@ -347,7 +386,9 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
         self.OTEL_EXPORTER = self.config.exporter
         self.OTEL_ENDPOINT = self.config.endpoint
         self.OTEL_HEADERS = self.config.headers
-        self._tracer_provider_cache: dict[str, _SDKTracerProvider] = {}
+        self._tracer_provider_cache: OrderedDict[str, _CachedTracerProvider] = OrderedDict()
+        self._tracer_provider_cache_lock: Final = threading.Lock()
+        self._max_dynamic_tracer_providers: Final = max(1, max_dynamic_tracer_providers)
         self._init_tracing(tracer_provider)
 
         _debug_otel: Final = str(os.getenv("DEBUG_OTEL", "False")).lower()
@@ -678,6 +719,28 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
     async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
         self._handle_failure(kwargs, response_obj, start_time, end_time)
 
+    def _start_service_span(self, payload: ServiceLoggerPayload, parent_otel_span: Span, start_time_ns: int) -> Span:
+        """Open a service span, named and classified by what the service is.
+
+        A datastore call is an outbound CLIENT span carrying ``db.*`` semconv.
+        Without those a Postgres span says only ``service=postgres``, so the
+        backend falls back to the transport peer, which for Prisma is the local
+        query engine on loopback. Everything else stays an INTERNAL span.
+        """
+        from opentelemetry import trace
+        from opentelemetry.trace import SpanKind
+
+        attributes: Final = db_span_attributes(payload.service.value, payload.call_type)
+        span: Final = self.tracer.start_span(
+            name=payload.service,
+            context=trace.set_span_in_context(parent_otel_span),
+            start_time=start_time_ns,
+            kind=SpanKind.CLIENT if attributes else SpanKind.INTERNAL,
+        )
+        for key, value in attributes.items():
+            self.safe_set_attribute(span=span, key=key, value=value)
+        return span
+
     async def async_service_success_hook(
         self,
         payload: ServiceLoggerPayload,
@@ -686,7 +749,6 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
         end_time: datetime | float | None = None,
         event_metadata: dict | None = None,
     ):
-        from opentelemetry import trace
         from opentelemetry.trace import Status, StatusCode
 
         _start_time_ns = 0
@@ -703,12 +765,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
             _end_time_ns = self._to_ns(end_time)
 
         if parent_otel_span is not None:
-            _span_name: Final = payload.service
-            service_logging_span: Final = self.tracer.start_span(
-                name=_span_name,
-                context=trace.set_span_in_context(parent_otel_span),
-                start_time=_start_time_ns,
-            )
+            service_logging_span: Final = self._start_service_span(payload, parent_otel_span, _start_time_ns)
             self.safe_set_attribute(
                 span=service_logging_span,
                 key="call_type",
@@ -746,7 +803,6 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
         end_time: float | datetime | None = None,
         event_metadata: dict | None = None,
     ):
-        from opentelemetry import trace
         from opentelemetry.trace import Status, StatusCode
 
         _start_time_ns = 0
@@ -763,12 +819,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
             _end_time_ns = self._to_ns(end_time)
 
         if parent_otel_span is not None:
-            _span_name: Final = payload.service
-            service_logging_span: Final = self.tracer.start_span(
-                name=_span_name,
-                context=trace.set_span_in_context(parent_otel_span),
-                start_time=_start_time_ns,
-            )
+            service_logging_span: Final = self._start_service_span(payload, parent_otel_span, _start_time_ns)
             self.safe_set_attribute(
                 span=service_logging_span,
                 key="call_type",
@@ -1027,38 +1078,98 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
 
         return self.construct_dynamic_otel_config(standard_callback_dynamic_params=standard_callback_dynamic_params)
 
-    def _get_tracer_with_dynamic_config(self, dynamic_config: OpenTelemetryConfig):
+    def _insert_or_drop(
+        self, cache_key: str, built: "_CachedTracerProvider"
+    ) -> "tuple[_CachedTracerProvider, _CachedTracerProvider | None]":
+        """Cache ``built`` under ``cache_key``, returning the entry to use and what to drop.
+
+        Caller holds ``_tracer_provider_cache_lock``. The drop is either the loser of a
+        concurrent build for this key or the LRU victim its insertion pushed out.
+        """
+        raced: Final = self._tracer_provider_cache.get(cache_key)
+        if raced is not None:
+            self._tracer_provider_cache.move_to_end(cache_key)
+            return raced, built
+
+        self._tracer_provider_cache[cache_key] = built
+        if len(self._tracer_provider_cache) > self._max_dynamic_tracer_providers:
+            return built, self._tracer_provider_cache.popitem(last=False)[1]
+        return built, None
+
+    def _cached_dynamic_tracer(
+        self,
+        cache_key: str,
+        build: Callable[[], "_SDKTracerProvider"],
+        owns_exporter: bool,
+    ) -> "_Tracer":
+        """Return the tracer for ``cache_key``, building and caching a provider on miss.
+
+        A provider that owns its exporter also owns a ``BatchSpanProcessor`` worker thread
+        that only stops on ``shutdown()``, so the cache is a bounded LRU and whatever it
+        drops is shut down. Without both, a proxy serving key-scoped credentials accumulates
+        one live thread per credential set for the life of the process.
+
+        ``owns_exporter`` also decides ``shutdown_on_exit`` at build time: a provider we may
+        never shut down must not hold an interpreter-exit hook, which would both pin it in
+        memory for the life of the process and stop the shared exporter at exit. Those
+        providers use ``SimpleSpanProcessor``, which buffers nothing, so the hook costs them
+        no flush.
+
+        ``owns_exporter`` describes the provider being built, and is cached with it, because
+        the two dynamic entry points share this cache and can disagree: whether the LRU
+        victim may be shut down is a property of the victim, never of the request that
+        happened to evict it.
+        """
+        with self._tracer_provider_cache_lock:
+            cached: Final = self._tracer_provider_cache.get(cache_key)
+            if cached is not None:
+                self._tracer_provider_cache.move_to_end(cache_key)
+                return cached.provider.get_tracer(LITELLM_TRACER_NAME)
+
+        # Built outside the lock: exporter construction can block on DNS/TLS.
+        built: Final = _CachedTracerProvider(provider=build(), owns_exporter=owns_exporter)
+
+        with self._tracer_provider_cache_lock:
+            winner, dropped = self._insert_or_drop(cache_key, built)
+
+        if dropped is not None and dropped.owns_exporter:
+            # Off the caller's thread: shutdown joins the exporter worker.
+            _PROVIDER_SHUTDOWN_EXECUTOR.submit(_shutdown_tracer_provider, dropped.provider)
+        return winner.provider.get_tracer(LITELLM_TRACER_NAME)
+
+    def _get_tracer_with_dynamic_config(self, dynamic_config: OpenTelemetryConfig) -> "_Tracer":
         """Create (or reuse) a tracer whose exporter target comes from a per-request config."""
         from opentelemetry.sdk.trace import TracerProvider
 
-        cache_key = f"dynamic_config:{dynamic_config.exporter}:{dynamic_config.endpoint}:{dynamic_config.headers}"
-        if cache_key in self._tracer_provider_cache:
-            return self._tracer_provider_cache[cache_key].get_tracer(LITELLM_TRACER_NAME)
+        owns_exporter: Final = _provider_owns_exporter(dynamic_config.exporter)
 
-        temp_provider: Final = TracerProvider(resource=self._get_litellm_resource(self.config))
-        temp_provider.add_span_processor(self._get_span_processor(config_override=dynamic_config))
+        def _build() -> "_SDKTracerProvider":
+            provider: Final = TracerProvider(
+                resource=self._get_litellm_resource(self.config), shutdown_on_exit=owns_exporter
+            )
+            provider.add_span_processor(self._get_span_processor(config_override=dynamic_config))
+            return provider
 
-        self._tracer_provider_cache[cache_key] = temp_provider
+        cache_key: Final = (
+            f"dynamic_config:{dynamic_config.exporter}:{dynamic_config.endpoint}:{dynamic_config.headers}"
+        )
+        return self._cached_dynamic_tracer(cache_key, _build, owns_exporter)
 
-        return temp_provider.get_tracer(LITELLM_TRACER_NAME)
-
-    def _get_tracer_with_dynamic_headers(self, dynamic_headers: dict):
-        """Create a temporary tracer with dynamic headers for this request only."""
+    def _get_tracer_with_dynamic_headers(self, dynamic_headers: Mapping[str, str]) -> "_Tracer":
+        """Create (or reuse) a tracer whose OTLP headers come from a per-request credential set."""
         from opentelemetry.sdk.trace import TracerProvider
 
-        # Prevents thread exhaustion by reusing providers for the same credential sets (e.g. per-team keys)
+        owns_exporter: Final = _provider_owns_exporter(self.OTEL_EXPORTER)
+
+        def _build() -> "_SDKTracerProvider":
+            provider: Final = TracerProvider(
+                resource=self._get_litellm_resource(self.config), shutdown_on_exit=owns_exporter
+            )
+            provider.add_span_processor(self._get_span_processor(dynamic_headers=dynamic_headers))
+            return provider
+
         cache_key: Final = str(sorted(dynamic_headers.items()))
-        if cache_key in self._tracer_provider_cache:
-            return self._tracer_provider_cache[cache_key].get_tracer(LITELLM_TRACER_NAME)
-
-        # Create a temporary tracer provider with dynamic headers
-        temp_provider: Final = TracerProvider(resource=self._get_litellm_resource(self.config))
-        temp_provider.add_span_processor(self._get_span_processor(dynamic_headers=dynamic_headers))
-
-        # Store in cache for reuse
-        self._tracer_provider_cache[cache_key] = temp_provider
-
-        return temp_provider.get_tracer(LITELLM_TRACER_NAME)
+        return self._cached_dynamic_tracer(cache_key, _build, owns_exporter)
 
     def construct_dynamic_otel_headers(
         self, standard_callback_dynamic_params: StandardCallbackDynamicParams
@@ -2832,7 +2943,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
 
     def _get_span_processor(
         self,
-        dynamic_headers: dict | None = None,
+        dynamic_headers: Mapping[str, str] | None = None,
         config_override: OpenTelemetryConfig | None = None,
     ):
         from opentelemetry.sdk.trace.export import (
@@ -3144,7 +3255,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
 
     @staticmethod
     def _get_headers_dictionary(
-        headers: str | dict | None,
+        headers: "str | Mapping[str, str] | None",
     ) -> dict[str, str]:
         """
         Convert a string or dictionary of headers into a dictionary of headers.
@@ -3158,8 +3269,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
                 for part in parts:
                     key, value = part.split("=", 1)
                     _split_otel_headers[key] = value
-            elif isinstance(headers, dict):
-                _split_otel_headers = headers
+            elif isinstance(headers, Mapping):
+                _split_otel_headers.update(headers)
         return _split_otel_headers
 
     async def async_management_endpoint_success_hook(

litellm/integrations/otel/mappers/genai.py

@@ -18,6 +18,7 @@ from litellm.integrations.otel.mappers.utils import (
     serialize_messages,
     tool_definition_attrs,
 )
+from litellm.integrations.otel.model.db_endpoint import db_span_attributes
 from litellm.integrations.otel.model.payloads import (
     GuardrailSpanData,
     LLMCallSpanData,
@@ -27,7 +28,6 @@ from litellm.integrations.otel.model.payloads import (
     ToolDefinition,
 )
 from litellm.integrations.otel.model.semconv import (
-    DB,
     MCP,
     Error,
     GenAI,
@@ -36,7 +36,6 @@ from litellm.integrations.otel.model.semconv import (
     RpcSystem,
     Server,
 )
-from litellm.integrations.otel.model.spans import db_system
 
 
 class GenAIMapper:
@@ -182,12 +181,8 @@ class GenAIMapper:
     def _service(cls, data: ServiceSpanData) -> AttributeMap:
         attrs: Final = collect(cls._SERVICE_ATTRS, data)
         # An outbound datastore call (DB_CALL / CLIENT span) also carries db.*
-        # semconv. Internal services (router, budget jobs, …) have no db.system,
-        # so they get only the litellm.service.* keys above.
-        system: Final = db_system(data.service_name)
-        if system is not None:
-            attrs[DB.SYSTEM_NAME] = system
-            if data.call_type:
-                attrs[DB.OPERATION_NAME] = data.call_type
+        # semconv naming the server it reached. Internal services (router, budget
+        # jobs, …) have no db.system, so they get only the litellm.service.* keys.
+        attrs.update(db_span_attributes(data.service_name, data.call_type))
         attrs.update({f"{LiteLLM.METADATA_PREFIX}{key}": value for key, value in data.event_metadata.items()})
         return attrs

litellm/integrations/otel/model/db_endpoint.py

@@ -0,0 +1,164 @@
+"""OTel ``db.*`` / ``server.*`` attributes naming the database litellm talks to.
+
+Prisma reaches PostgreSQL through a query engine listening on loopback, so
+transport-level instrumentation attributes the work to ``localhost`` and an
+operator cannot tell it is a PostgreSQL call or correlate it with the database's
+own metrics. These attributes name the real server on litellm's DB spans.
+
+Only the host, port, database and schema of the DSN are read, so no credential
+can reach an exporter.
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import Final
+from urllib.parse import ParseResult, parse_qs, unquote, urlparse
+
+from litellm.integrations.otel.model.semconv import DB, Server
+from litellm.integrations.otel.model.spans import POSTGRESQL, db_system
+
+_DATABASE_URL_ENV: Final = "DATABASE_URL"
+_READ_REPLICA_ENV: Final = "DATABASE_URL_READ_REPLICA"
+_DEFAULT_POSTGRES_PORT: Final = 5432
+_DEFAULT_POSTGRES_SCHEMA: Final = "public"
+_POSTGRES_SCHEMES: Final = frozenset({"postgres", "postgresql"})
+_EMPTY_ATTRIBUTES: Final[Mapping[str, str | int]] = MappingProxyType({})
+
+
+@dataclass(frozen=True, slots=True)
+class DatabaseEndpoint:
+    """The non-sensitive identity of a PostgreSQL server, parsed from a DSN."""
+
+    address: str | None
+    port: int | None
+    namespace: str | None
+
+
+def parse_database_endpoint(url: str | None) -> DatabaseEndpoint | None:
+    """Parse a PostgreSQL DSN into its exportable endpoint identity.
+
+    Returns ``None`` for an absent, malformed or non-PostgreSQL URL rather than
+    raising: an unparseable DSN must degrade to a span without endpoint
+    attributes, never break the request that emitted it.
+    """
+    if not url:
+        return None
+    try:
+        parsed: Final = urlparse(url)
+        if parsed.scheme not in _POSTGRES_SCHEMES:
+            return None
+        query: Final = parse_qs(parsed.query)
+        raw_database: Final = (parsed.path or "").lstrip("/")
+        if _is_misparsed_authority(parsed, url, raw_database):
+            return None
+        # ``host=`` beats the netloc: it is how libpq names a Unix socket
+        # directory and how the Cloud SQL connector sits behind a localhost
+        # netloc, where the netloc is the very answer this module replaces.
+        address: Final = _first(query.get("host")) or parsed.hostname
+        # ``port=`` accompanies ``host=`` in a libpq URI, so honour it the same way.
+        port: Final = _port(_first(query.get("port")), parsed.port) if address else None
+        namespace: Final = _namespace(unquote(raw_database), _first(query.get("schema")))
+    except ValueError:
+        return None
+    if address is None and namespace is None:
+        return None
+    return DatabaseEndpoint(address=address, port=port, namespace=namespace)
+
+
+def _is_misparsed_authority(parsed: ParseResult, url: str, raw_database: str) -> bool:
+    """Whether the URL authority may have been truncated by an unencoded character.
+
+    ``/``, ``#`` or ``?`` in a password ends the netloc early, so urlparse hands
+    back the username as the host, the leading digits of the password as the
+    port, and the rest of the credential as the path, query or fragment. The
+    stranded userinfo ``@`` is the only surviving evidence.
+
+    A database name cannot hold an unencoded slash either, so a second path
+    segment is the same evidence.
+
+    A DSN that carries the at-sign in a query parameter instead, such as
+    ``?application_name=svc@prod``, is indistinguishable from a mis-split by any
+    property of the parse: both leave no userinfo, a host, a port and a path.
+    Since guessing wrong publishes a credential fragment to a tracing backend,
+    that ambiguity resolves to refusing the endpoint. Such a DSN loses
+    ``server.address`` and ``db.namespace`` and keeps the rest of the span,
+    which is the cheaper error of the two. Percent-encode the at-sign to keep
+    them.
+    """
+    if "/" in raw_database:
+        return True
+    return "@" in url and "@" not in parsed.netloc
+
+
+def _first(values: Sequence[str] | None) -> str:
+    return values[0] if values else ""
+
+
+def _port(from_query: str, from_netloc: int | None) -> int:
+    return int(from_query) if from_query.isdigit() else (from_netloc or _DEFAULT_POSTGRES_PORT)
+
+
+def _namespace(database: str, schema: str) -> str | None:
+    """``{database}|{schema}`` per the PostgreSQL semconv, dropping absent halves.
+
+    Only Prisma's literal default schema stays implicit. The match is
+    case-sensitive because Prisma quotes the name, so ``?schema=PUBLIC`` builds
+    a second schema alongside ``public`` and the two must not collapse to one
+    namespace.
+    """
+    qualifier: Final = "" if schema == _DEFAULT_POSTGRES_SCHEMA else schema
+    return "|".join(part for part in (database, qualifier) if part) or None
+
+
+def postgres_endpoint() -> DatabaseEndpoint | None:
+    """The PostgreSQL endpoint the process is currently connected to.
+
+    Read from ``os.environ`` on every span, deliberately, on both counts.
+
+    The environment is what Prisma itself connects with, so the span cannot
+    disagree with the connection; ``get_secret_str`` would consult a configured
+    secret manager first and could name a different server than the one serving
+    the query. And the value is not static: the RDS IAM refresh rebuilds the URL
+    from ``DATABASE_HOST``/``PORT``/``NAME``/``SCHEMA`` every rotation, the
+    reconnect path re-reads ``DATABASE_URL``, and the DB-backed
+    ``environment_variables`` config overlay can rewrite any of them after
+    startup, so a value cached for the process lifetime goes stale against a
+    connection that has genuinely moved. Nothing is memoized either: a cache
+    keyed on the URL would hold a rotated credential past its rotation, and the
+    parse is a single ``urlparse`` on a short string.
+
+    A configured read replica yields ``None``: ``RoutingPrismaWrapper`` picks
+    reader or writer per Prisma call, underneath the span, so naming the writer
+    would attribute replica reads to the primary.
+    """
+    if os.environ.get(_READ_REPLICA_ENV):
+        return None
+    return parse_database_endpoint(os.environ.get(_DATABASE_URL_ENV, ""))
+
+
+def db_span_attributes(service_name: str, call_type: str | None = None) -> Mapping[str, str | int]:
+    """The ``db.*``/``server.*`` attributes for a datastore service call.
+
+    Empty for services that are not outbound datastore calls. Endpoint
+    attributes are PostgreSQL-only: ``DATABASE_URL`` says nothing about where
+    the redis-backed services point. ``db.system`` rides alongside the current
+    ``db.system.name`` because Datadog's OTLP intake still types a database span
+    from the older key.
+    """
+    system: Final = db_system(service_name)
+    if system is None:
+        return _EMPTY_ATTRIBUTES
+    endpoint: Final = postgres_endpoint() if system == POSTGRESQL else None
+    pairs: Final[tuple[tuple[str, str | int | None], ...]] = (
+        (DB.SYSTEM_NAME, system),
+        (DB.SYSTEM_LEGACY, system),
+        (DB.OPERATION_NAME, call_type),
+        (Server.ADDRESS, endpoint.address if endpoint is not None else None),
+        (Server.PORT, endpoint.port if endpoint is not None else None),
+        (DB.NAMESPACE, endpoint.namespace if endpoint is not None else None),
+    )
+    return MappingProxyType({key: value for key, value in pairs if value})

litellm/integrations/otel/model/semconv.py

@@ -238,7 +238,11 @@ class DB:
     """
 
     SYSTEM_NAME: Final = "db.system.name"
+    # Superseded by SYSTEM_NAME, dual-emitted because Datadog's OTLP intake
+    # still infers a span's database type from this key.
+    SYSTEM_LEGACY: Final = "db.system"
     OPERATION_NAME: Final = "db.operation.name"
+    NAMESPACE: Final = "db.namespace"
 
 
 class HTTP:

litellm/integrations/otel/model/spans.py

@@ -115,10 +115,12 @@ SPAN_REGISTRY: Final[dict[SpanRole, SpanSpec]] = {
 # redis-backed spend queues. Any service not mapped here is litellm-internal work
 # and stays an INTERNAL ``SERVICE`` span. This table is the single source of
 # datastore knowledge — both the role classifier and the mapper read it.
+POSTGRESQL: Final = "postgresql"
+
 _DB_SYSTEM_BY_SERVICE: Final[dict[str, str]] = {
     "redis": "redis",
-    "postgres": "postgresql",
-    "batch_write_to_db": "postgresql",
+    "postgres": POSTGRESQL,
+    "batch_write_to_db": POSTGRESQL,
 }
 
 

litellm/integrations/shadow_eval_logger.py

@@ -9,13 +9,14 @@ across pods or stop races; the hook reads active jobs through a short-TTL cache.
 import asyncio
 import hashlib
 import random
+import traceback
 from collections.abc import Callable, Mapping, Sequence
 from dataclasses import dataclass
 from datetime import datetime, timezone
 from itertools import groupby
 from operator import itemgetter
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Final
+from typing import TYPE_CHECKING, Final, Literal
 
 from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError, field_validator, model_validator
 
@@ -32,6 +33,7 @@ from litellm.litellm_core_utils.llm_judge import (
     parse_json_verdict,
 )
 from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
+from litellm.llms.base_llm.base_utils import type_to_response_format_param
 from litellm.types.management_endpoints.auto_router_endpoints import ShadowEvalDirection
 from litellm.types.utils import SHADOW_EVAL_JUDGE_CALL_ORIGIN, SHADOW_EVAL_ROUTER_CALL_ORIGIN
 
@@ -55,7 +57,7 @@ _MAX_JUDGE_PROMPT_CHARS: Final = 24_000
 
 # The judge answers with a small JSON object; a tighter budget truncates the JSON
 # mid-object and the attempt is lost to an error row.
-JUDGE_MAX_OUTPUT_TOKENS: Final = 500
+JUDGE_MAX_OUTPUT_TOKENS: Final = 1500
 
 _MAX_ERROR_CHARS: Final = 500
 
@@ -305,16 +307,21 @@ Criteria: correctness, completeness, clarity, conciseness.
 Return ONLY valid JSON in this exact format, no other text:
 {
   "preference": "A" | "B" | "tie",
-  "confidence": <0.0 to 1.0>,
-  "reasoning": "<one sentence>"
+  "confidence": <0.0 to 1.0>
 }"""
 
 
 class PairwiseVerdict(BaseModel):
-    """The judge's blind A/B verdict, validated at the parse boundary."""
+    """The judge's blind A/B verdict: the response_format schema sent with the judge call
+    and the validation contract on its reply. Both fields are required and preference is
+    closed over the prompt's labels, so a malformed or truncated reply is an
+    unparseable-verdict error row, never a defaulted or fabricated verdict."""
 
-    preference: str = "tie"
-    confidence: float = 0.0
+    preference: Literal["A", "B", "tie"]
+    confidence: float
+
+
+PAIRWISE_JUDGE_RESPONSE_FORMAT: Final = type_to_response_format_param(PairwiseVerdict)
 
 
 def _sample_hits(request_id: str, job_id: str, percentage: float) -> bool:
@@ -325,6 +332,14 @@ def _sample_hits(request_id: str, job_id: str, percentage: float) -> bool:
     return bucket * 100.0 < percentage
 
 
+def _failure_detail(e: BaseException) -> str:
+    """Exception class, message, and the raising frame, so an attempt's error row names
+    the faulty code path without needing debug logs on the pod."""
+    frames: Final = traceback.extract_tb(e.__traceback__)
+    location: Final = f" at {frames[-1].filename.rsplit('/', 1)[-1]}:{frames[-1].lineno}" if frames else ""
+    return f"{type(e).__name__}{location}: {e}"
+
+
 def _judge_call_cost(response: object) -> float:
     """Price a judge call, treating an unmapped judge model as free rather than fatal."""
     import litellm
@@ -764,7 +779,9 @@ class ShadowEvalLogger(CustomLogger):
         try:
             response: Final = await router.acompletion(
                 model=target_model,
-                messages=messages,  # pyright: ignore[reportArgumentType]  # snapshot of the SDK's own message dicts
+                messages=[  # mutable-ok: provider transforms rewrite messages in place, so the router gets its own copy
+                    dict(m) for m in messages
+                ],  # pyright: ignore[reportArgumentType]  # snapshot of the SDK's own message dicts
                 metadata=shadow_metadata,
                 num_retries=0,
                 fallbacks=[],  # mutable-ok: SDK kwarg; a failed shadow is a recorded error, never a spend multiplier
@@ -772,7 +789,7 @@ class ShadowEvalLogger(CustomLogger):
             )
         except Exception as e:  # noqa: BLE001  # provider errors become error rows, not crashes
             verbose_logger.debug("shadow_eval: router call failed: %s", e)
-            return _CallFailure(f"shadow router call failed: {e}")
+            return _CallFailure(f"shadow router call failed: {_failure_detail(e)}")
         text: Final = _chat_final_text(response)
         if not text:
             return _CallFailure("shadow router returned an empty response")
@@ -815,6 +832,7 @@ class ShadowEvalLogger(CustomLogger):
                 judge_messages,  # pyright: ignore[reportArgumentType]  # plain SDK message dicts
                 temperature=0,
                 max_tokens=JUDGE_MAX_OUTPUT_TOKENS,
+                response_format=PAIRWISE_JUDGE_RESPONSE_FORMAT,
                 metadata=judge_metadata,
             )
         except Exception as e:  # noqa: BLE001  # judge outages become error rows, not crashes

litellm/litellm_core_utils/litellm_logging.py

@@ -13,6 +13,7 @@ import traceback
 from collections.abc import Callable, Mapping, Sequence
 from datetime import datetime as dt_object
 from functools import lru_cache
+from types import TracebackType
 from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Union, cast
 
 from httpx import Response
@@ -63,6 +64,10 @@ from litellm.integrations.mlflow import MlflowLogger
 from litellm.integrations.sqs import SQSLogger
 from litellm.litellm_core_utils.core_helpers import reconstruct_model_name
 from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
+from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
+    cost_breakdown_with_guardrail,
+    guardrail_information_cost,
+)
 from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
     StandardBuiltInToolCostTracking,
 )
@@ -107,6 +112,7 @@ from litellm.types.utils import (
     LiteLLMBatch,
     LiteLLMLoggingBaseClass,
     LiteLLMRealtimeStreamLoggingObject,
+    ModelInfo,
     ModelResponse,
     ModelResponseStream,
     RawRequestTypedDict,
@@ -306,6 +312,66 @@ def _get_cached_prometheus_logger():
     return _PrometheusLogger
 
 
+_DEPLOYMENT_PRICING_KEYS: Final = (
+    "input_cost_per_token",
+    "output_cost_per_token",
+    "input_cost_per_token_batches",
+    "output_cost_per_token_batches",
+)
+
+
+def deployment_pricing_model_info(model_id: str | None, deployment_model: str | None) -> ModelInfo | None:
+    """Pricing the router registered under this deployment's model_info.id.
+
+    Returns None when the deployment declares no pricing of its own, so the
+    caller falls back to the global cost map. The raw registration is what
+    decides that: the router registers an entry for every deployment, and
+    get_model_info fills absent costs with 0, so asking it directly cannot
+    tell "configured as free" apart from "no pricing configured". A deployment
+    may declare only one side of its pricing, so the side it leaves out keeps
+    the model's published rates instead of billing as zero. Ownership is per
+    token direction: declaring either rate for a direction takes that whole
+    direction, so a published batch rate can never displace a standard rate
+    the deployment configured itself.
+    """
+    if model_id is None:
+        return None
+    registered: Final = litellm.model_cost.get(model_id)
+    if not isinstance(registered, dict) or not any(registered.get(key) is not None for key in _DEPLOYMENT_PRICING_KEYS):
+        return None
+    try:
+        merged: Final = litellm.get_model_info(model=model_id).copy()
+    except Exception:  # noqa: BLE001  # get_model_info raises for ids it cannot resolve a provider for
+        return None
+    published: Final = _published_pricing(deployment_model)
+    if published is None:
+        return merged
+    declares_input: Final = (
+        registered.get("input_cost_per_token") is not None or registered.get("input_cost_per_token_batches") is not None
+    )
+    declares_output: Final = (
+        registered.get("output_cost_per_token") is not None
+        or registered.get("output_cost_per_token_batches") is not None
+    )
+    if not declares_input:
+        merged["input_cost_per_token"] = published.get("input_cost_per_token")
+        merged["input_cost_per_token_batches"] = published.get("input_cost_per_token_batches")
+    if not declares_output:
+        merged["output_cost_per_token"] = published.get("output_cost_per_token")
+        merged["output_cost_per_token_batches"] = published.get("output_cost_per_token_batches")
+    return merged
+
+
+def _published_pricing(deployment_model: str | None) -> ModelInfo | None:
+    """The cost map's own entry for the deployment's model, when it resolves."""
+    if deployment_model is None:
+        return None
+    try:
+        return litellm.get_model_info(model=deployment_model)
+    except Exception:  # noqa: BLE001  # no published entry to layer the declared rates over
+        return None
+
+
 class Logging(LiteLLMLoggingBaseClass):
     global \
         supabaseClient, \
@@ -578,6 +644,28 @@ class Logging(LiteLLMLoggingBaseClass):
                 return model_id
         return None
 
+    def get_deployment_model_for_cost(self) -> str | None:
+        """The provider-qualified model to price against.
+
+        On a batch retrieve both self.model and litellm_params["model"] can be
+        unset, and self.model can otherwise carry the router's model_group alias,
+        which no cost map resolves. model_call_details holds the deployment's own
+        provider-qualified model, so it is preferred.
+        """
+        candidates: Final = (
+            (self.model_call_details or {}).get("model") if hasattr(self, "model_call_details") else None,
+            self.litellm_params.get("model") if hasattr(self, "litellm_params") else None,
+            self.model,
+        )
+        return next((candidate for candidate in candidates if isinstance(candidate, str) and candidate), None)
+
+    def get_router_deployment_model_info(self) -> ModelInfo | None:
+        """See deployment_pricing_model_info; None means fall back to the global cost map."""
+        return deployment_pricing_model_info(
+            model_id=self.get_router_model_id(),
+            deployment_model=self.get_deployment_model_for_cost(),
+        )
+
     def update_environment_variables(
         self,
         litellm_params: dict,
@@ -1006,10 +1094,10 @@ class Logging(LiteLLMLoggingBaseClass):
                             data=additional_args.get("complete_input_dict", {}),
                         )
 
-                        _metadata["raw_request"] = str(curl_command)
+                        _metadata["raw_request"] = _redact_string(str(curl_command))
                         # split up, so it's easier to parse in the UI
                         self.model_call_details["raw_request_typed_dict"] = RawRequestTypedDict(
-                            raw_request_api_base=str(additional_args.get("api_base") or ""),
+                            raw_request_api_base=self._get_masked_api_base(str(additional_args.get("api_base") or "")),
                             raw_request_body=self._get_raw_request_body(additional_args.get("complete_input_dict", {})),
                             # NOTE: setting ignore_sensitive_headers to True will cause
                             # the Authorization header to be leaked when calls to the health
@@ -1023,8 +1111,10 @@ class Logging(LiteLLMLoggingBaseClass):
                     self.model_call_details["raw_request_typed_dict"] = RawRequestTypedDict(
                         error=str(e),
                     )
-                    _metadata["raw_request"] = f"Unable to Log \
+                    _metadata["raw_request"] = _redact_string(
+                        f"Unable to Log \
                         raw request: {e}"
+                    )
             if getattr(self, "logger_fn", None) and callable(self.logger_fn):
                 try:
                     self.logger_fn(
@@ -1118,15 +1208,16 @@ class Logging(LiteLLMLoggingBaseClass):
         if _is_debugging_on() or self.litellm_request_debug:
             if json_logs:
                 masked_headers: Final = self._get_masked_headers(headers)
+                masked_api_base: Final = self._get_masked_api_base(str(api_base or ""))
                 if self.litellm_request_debug:
                     verbose_logger.warning(  # .warning ensures this shows up in all environments
                         "POST Request Sent from LiteLLM",
-                        extra={"api_base": {api_base}, **masked_headers},
+                        extra={"api_base": {masked_api_base}, **masked_headers},
                     )
                 else:
                     verbose_logger.debug(
                         "POST Request Sent from LiteLLM",
-                        extra={"api_base": {api_base}, **masked_headers},
+                        extra={"api_base": {masked_api_base}, **masked_headers},
                     )
             else:
                 headers = additional_args.get("headers", {})
@@ -1166,8 +1257,6 @@ class Logging(LiteLLMLoggingBaseClass):
             curl_command = "\nRequest Sent from LiteLLM:\n"
             request_str: Final = additional_args.get("request_str", "")
             curl_command += request_str
-        elif api_base == "":
-            curl_command = str(self.model_call_details)
         return curl_command
 
     def _get_masked_headers(self, headers: dict, ignore_sensitive_headers: bool = False) -> dict:
@@ -1189,6 +1278,7 @@ class Logging(LiteLLMLoggingBaseClass):
             self.model_call_details["additional_args"] = additional_args
             self.model_call_details["log_event_type"] = "post_api_call"
 
+            attr: Literal["warning", "debug"]
             if self.litellm_request_debug:
                 attr = "warning"
             else:
@@ -1802,7 +1892,7 @@ class Logging(LiteLLMLoggingBaseClass):
         if self.model_call_details.get("litellm_params") is None:
             return
         metadata_hidden_params: Final = hidden_params.copy()
-        response_cost: Final = self.model_call_details.get("response_cost")
+        response_cost: Final[object] = self.model_call_details.get("response_cost")
         if metadata_hidden_params.get("response_cost") is None and response_cost is not None:
             metadata_hidden_params["response_cost"] = response_cost
 
@@ -1844,7 +1934,10 @@ class Logging(LiteLLMLoggingBaseClass):
             logging_result, start_time, end_time
         )
 
-        if (standard_logging_payload := self.model_call_details.get("standard_logging_object")) is not None:
+        standard_logging_payload: Final[StandardLoggingPayload | None] = self.model_call_details.get(
+            "standard_logging_object"
+        )
+        if standard_logging_payload is not None:
             emit_standard_logging_payload(standard_logging_payload)
 
     def _build_standard_logging_payload(
@@ -2109,7 +2202,7 @@ class Logging(LiteLLMLoggingBaseClass):
 
     def _success_handler_body(
         self,
-        result: Any = None,  # heterogeneous response object; varies by call type (ANN401 ignored, see ruff-strict.toml)
+        result: object = None,
         start_time: datetime.datetime | None = None,
         end_time: datetime.datetime | None = None,
         cache_hit: bool | None = None,
@@ -2150,7 +2243,10 @@ class Logging(LiteLLMLoggingBaseClass):
                 self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
                     complete_streaming_response, start_time, end_time
                 )
-                if (standard_logging_payload := self.model_call_details.get("standard_logging_object")) is not None:
+                standard_logging_payload: Final[StandardLoggingPayload | None] = self.model_call_details.get(
+                    "standard_logging_object"
+                )
+                if standard_logging_payload is not None:
                     # Only emit for sync requests (async_success_handler handles async)
                     if is_sync_request:
                         emit_standard_logging_payload(standard_logging_payload)
@@ -2592,7 +2688,9 @@ class Logging(LiteLLMLoggingBaseClass):
                 ) = await _handle_completed_batch(
                     batch=result,
                     custom_llm_provider=self.custom_llm_provider,
+                    model_name=self.get_deployment_model_for_cost(),
                     litellm_params=self.litellm_params,
+                    model_info=self.get_router_deployment_model_info(),
                 )
 
                 result._hidden_params["response_cost"] = response_cost
@@ -2981,7 +3079,7 @@ class Logging(LiteLLMLoggingBaseClass):
                 global_callbacks=litellm.failure_callback,
             )
 
-            result = None  # result sent to all loggers, init this to None incase it's not created
+            result: object = None  # result sent to all loggers, init this to None incase it's not created
 
             result = redact_message_input_output_from_logging(
                 model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
@@ -3395,11 +3493,11 @@ class Logging(LiteLLMLoggingBaseClass):
 
     def _get_assembled_streaming_response(
         self,
-        result: ModelResponse | TextCompletionResponse | ModelResponseStream | ResponseCompletedEvent | Any,
+        result: ModelResponse | TextCompletionResponse | ModelResponseStream | ResponseCompletedEvent | object,
         start_time: datetime.datetime,
         end_time: datetime.datetime,
         is_async: bool,
-        streaming_chunks: list[Any],
+        streaming_chunks: list[object],
     ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | None:
         if self.stream is not True:
             return None
@@ -3677,9 +3775,7 @@ def set_callbacks(callback_list, function_id=None):
                 from sentry_sdk.scrubber import EventScrubber
 
                 sentry_sdk_instance = sentry_sdk
-                sentry_trace_rate = (
-                    os.environ.get("SENTRY_API_TRACE_RATE") if "SENTRY_API_TRACE_RATE" in os.environ else "1.0"
-                )
+                sentry_trace_rate = os.environ.get("SENTRY_API_TRACE_RATE", "1.0")
                 sentry_sample_rate = (
                     os.environ.get("SENTRY_API_SAMPLE_RATE") if "SENTRY_API_SAMPLE_RATE" in os.environ else "1.0"
                 )
@@ -5150,13 +5246,13 @@ class StandardLoggingPayloadSetup:
         # ProxyException uses .code, LiteLLM exceptions use .status_code,
         # httpx.HTTPStatusError exposes status only as .response.status_code.
         # Stringified for Prisma JSON compatibility.
-        error_code_attr: Final = getattr(original_exception, "code", None)
+        error_code_attr: Final[object] = getattr(original_exception, "code", None)
         if error_code_attr is not None and str(error_code_attr) not in ("", "None"):
             error_status: str = str(error_code_attr)
         else:
-            status_code_attr = getattr(original_exception, "status_code", None)
+            status_code_attr: object = getattr(original_exception, "status_code", None)
             if status_code_attr is None:
-                response_attr: Final = getattr(original_exception, "response", None)
+                response_attr: Final[object] = getattr(original_exception, "response", None)
                 status_code_attr = getattr(response_attr, "status_code", None)
             error_status = str(status_code_attr) if status_code_attr is not None else ""
         error_class: Final[str] = str(original_exception.__class__.__name__) if original_exception else ""
@@ -5165,7 +5261,7 @@ class StandardLoggingPayloadSetup:
         # Get traceback information (first 100 lines)
         traceback_info = traceback_str or ""
         if original_exception:
-            tb: Final = getattr(original_exception, "__traceback__", None)
+            tb: Final[TracebackType | None] = getattr(original_exception, "__traceback__", None)
             if tb:
                 tb_lines: Final = traceback.format_tb(tb)
                 traceback_info += "".join(tb_lines[:MAXIMUM_TRACEBACK_LINES_TO_LOG])  # Limit to first 100 lines
@@ -5276,11 +5372,11 @@ class StandardLoggingPayloadSetup:
         """
         dynamic_litellm_session_id: Final = litellm_params.get("litellm_session_id")
         dynamic_litellm_trace_id: Final = litellm_params.get("litellm_trace_id")
-        metadata: Final = litellm_params.get("metadata")
+        metadata: Final[Mapping[str, object] | None] = litellm_params.get("metadata")
         metadata_session_id: Final = metadata.get("session_id") if metadata else None
         metadata_trace_id: Final = metadata.get("trace_id") if metadata else None
 
-        ordered_candidates: Final[tuple[Any, Any, Any, Any]] = (
+        ordered_candidates: Final[tuple[object, object, object, object]] = (
             (dynamic_litellm_trace_id, dynamic_litellm_session_id, metadata_trace_id, metadata_session_id)
             if litellm.request_correlation_in_logs
             else (dynamic_litellm_session_id, dynamic_litellm_trace_id, metadata_session_id, metadata_trace_id)
@@ -5305,10 +5401,10 @@ class StandardLoggingPayloadSetup:
         """
         if not litellm.request_correlation_in_logs:
             return ""
-        dynamic_litellm_session_id: Final = litellm_params.get("litellm_session_id")
+        dynamic_litellm_session_id: Final[object] = litellm_params.get("litellm_session_id")
         if dynamic_litellm_session_id:
             return str(dynamic_litellm_session_id)
-        metadata: Final = litellm_params.get("metadata")
+        metadata: Final[Mapping[str, object] | None] = litellm_params.get("metadata")
         metadata_session_id: Final = metadata.get("session_id") if metadata else None
         if metadata_session_id:
             return str(metadata_session_id)
@@ -5559,12 +5655,14 @@ def get_standard_logging_object_payload(
             base_model = metadata.get("deployment")
         custom_pricing: Final = use_custom_pricing_for_model(litellm_params=litellm_params)
         raw_response_cost: Final = kwargs.get("response_cost")
-        response_cost: Final[float] = raw_response_cost or 0.0
+        llm_response_cost: Final[float] = raw_response_cost or 0.0
+        guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information"))
+        response_cost: Final[float] = llm_response_cost + guardrail_cost
 
         # clean up litellm hidden params
         clean_hidden_params: Final = StandardLoggingPayloadSetup.get_hidden_params(hidden_params)
         if clean_hidden_params["response_cost"] is None and raw_response_cost is not None:
-            clean_hidden_params["response_cost"] = response_cost
+            clean_hidden_params["response_cost"] = llm_response_cost
 
         model_cost_information: Final = StandardLoggingPayloadSetup.get_model_cost_information(
             base_model=base_model,
@@ -5644,7 +5742,7 @@ def get_standard_logging_object_payload(
             metadata=clean_metadata,
             cache_key=clean_hidden_params["cache_key"],
             response_cost=response_cost,
-            cost_breakdown=logging_obj.cost_breakdown,
+            cost_breakdown=cost_breakdown_with_guardrail(logging_obj.cost_breakdown, guardrail_cost),
             total_tokens=usage_dict.get("total_tokens", 0),
             prompt_tokens=usage_dict.get("prompt_tokens", 0),
             completion_tokens=usage_dict.get("completion_tokens", 0),

litellm/litellm_core_utils/llm_cost_calc/guardrail_cost.py

@@ -0,0 +1,78 @@
+import math
+from collections.abc import Mapping
+from typing import Final
+
+from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
+
+import litellm
+from litellm._logging import verbose_logger
+from litellm.types.utils import CostBreakdown
+
+BEDROCK_GUARDRAIL_PRICING_KEY: Final = "bedrock/guardrails"
+
+
+class GuardrailPricing(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    guardrail_cost_per_unit: Mapping[str, float]
+
+
+class GuardrailCostEntry(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    guardrail_cost: float | None = None
+
+
+GuardrailInformationShape = tuple[GuardrailCostEntry, ...] | GuardrailCostEntry | None
+
+_GUARDRAIL_INFORMATION_ADAPTER: Final[TypeAdapter[GuardrailInformationShape]] = TypeAdapter(GuardrailInformationShape)
+
+
+def _bedrock_guardrail_pricing(aws_region_name: str | None) -> GuardrailPricing | None:
+    regional_key: Final = f"bedrock/{aws_region_name}/guardrails" if aws_region_name else None
+    for key in (regional_key, BEDROCK_GUARDRAIL_PRICING_KEY):
+        if key is None or key not in litellm.model_cost:
+            continue
+        try:
+            return GuardrailPricing.model_validate(litellm.model_cost[key])
+        except ValidationError as e:
+            verbose_logger.warning("Ignoring malformed guardrail pricing entry %s: %s", key, e)
+    return None
+
+
+def bedrock_guardrail_cost(usage_units: Mapping[str, int], aws_region_name: str | None) -> float:
+    pricing: Final = _bedrock_guardrail_pricing(aws_region_name)
+    if pricing is None:
+        return 0.0
+    return sum(units * pricing.guardrail_cost_per_unit.get(counter, 0.0) for counter, units in usage_units.items())
+
+
+def _billable_entry_cost(entry: GuardrailCostEntry) -> float:
+    cost: Final = entry.guardrail_cost
+    if cost is None or not math.isfinite(cost) or cost <= 0.0:
+        return 0.0
+    return cost
+
+
+def guardrail_information_cost(guardrail_information: object) -> float:
+    try:
+        parsed: Final = _GUARDRAIL_INFORMATION_ADAPTER.validate_python(guardrail_information)
+    except ValidationError:
+        return 0.0
+    if parsed is None:
+        return 0.0
+    if isinstance(parsed, GuardrailCostEntry):
+        return _billable_entry_cost(parsed)
+    return sum(_billable_entry_cost(entry) for entry in parsed)
+
+
+def cost_breakdown_with_guardrail(cost_breakdown: CostBreakdown | None, guardrail_cost: float) -> CostBreakdown | None:
+    if guardrail_cost <= 0.0:
+        return cost_breakdown
+    existing: Final[CostBreakdown] = cost_breakdown if cost_breakdown is not None else CostBreakdown()
+    merged: Final[CostBreakdown] = {
+        **existing,
+        "guardrail_cost": guardrail_cost,
+        "total_cost": existing.get("total_cost", 0.0) + guardrail_cost,
+    }
+    return merged

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -42,14 +42,18 @@ _VALID_DATA_RESIDENCIES: Final = frozenset(r.value for r in DataResidency)
 
 # Pre-resolved service-tier cost-key suffixes (e.g. "_priority"). Used per
 # request in the cost-calc path, so the f-strings are built once here instead
-# of being rebuilt for every model_info key on every call.
-_SERVICE_TIER_SUFFIXES: Final[tuple[str, ...]] = tuple(f"_{st.value}" for st in ServiceTier)
+# of being rebuilt for every model_info key on every call. Longest-first so a
+# substring match resolves "_ultrafast" before "_fast".
+_SERVICE_TIER_SUFFIXES: Final[tuple[str, ...]] = tuple(
+    sorted((f"_{st.value}" for st in ServiceTier), key=len, reverse=True)
+)
 
 _SERVICE_TIER_TO_COST_KEY_SUFFIX: Final[Mapping[str, str]] = MappingProxyType(
     {
         ServiceTier.FLEX.value: ServiceTier.FLEX.value,
         ServiceTier.PRIORITY.value: ServiceTier.PRIORITY.value,
         ServiceTier.FAST.value: ServiceTier.PRIORITY.value,
+        ServiceTier.ULTRAFAST.value: ServiceTier.ULTRAFAST.value,
     }
 )
 
@@ -191,7 +195,7 @@ def _get_service_tier_cost_key(base_key: str, service_tier: str | None) -> str:
 
     Args:
         base_key: The base cost key (e.g., "input_cost_per_token")
-        service_tier: The service tier ("flex", "priority", "fast", or None for standard)
+        service_tier: The service tier ("flex", "priority", "fast", "ultrafast", or None for standard)
 
     Returns:
         str: The cost key to use (e.g., "input_cost_per_token_flex" or "input_cost_per_token")

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -1816,16 +1816,19 @@ def split_concatenated_json_objects(raw: str) -> list[dict[str, Any]]:
     This helper uses ``json.JSONDecoder.raw_decode()`` to walk the string
     and extract each JSON object individually.
 
+    The walk degrades gracefully: if the string is malformed or truncated
+    (e.g. a stream that ended mid-tool-call), whatever complete objects were
+    parsed before the bad tail are returned and the remainder is discarded
+    with a warning, rather than raising.  The sole caller
+    (``_convert_to_bedrock_tool_call_invoke``) treats an empty result as
+    ``input={}`` so the conversation can continue instead of hard-failing.
+
     Returns
     -------
     list[dict]
         A list of parsed dicts – one per JSON object found.  If *raw* is
-        empty or whitespace-only, an empty list is returned.
-
-    Raises
-    ------
-    json.JSONDecodeError
-        If the string contains text that cannot be parsed as JSON at all.
+        empty, whitespace-only, or wholly unparseable, an empty list is
+        returned.
     """
     import json
 
@@ -1845,7 +1848,17 @@ def split_concatenated_json_objects(raw: str) -> list[dict[str, Any]]:
         if idx >= length:
             break
 
-        obj, end_idx = decoder.raw_decode(raw, idx)
+        try:
+            obj, end_idx = decoder.raw_decode(raw, idx)
+        except json.JSONDecodeError as e:
+            verbose_logger.warning(
+                "split_concatenated_json_objects: discarding unparseable tool-call "
+                "arguments tail after %d complete object(s); decode_start=%d error=%s",
+                len(results),
+                idx,
+                e,
+            )
+            break
         if isinstance(obj, dict):
             results.append(obj)
         else:

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -3712,7 +3712,13 @@ def _convert_to_bedrock_tool_call_invoke(
                         _parts_list.append(cache_point_block)
         return _parts_list
     except Exception as e:
-        raise Exception(f"Unable to convert openai tool calls={tool_calls} to bedrock tool calls. Received error={e}")
+        tool_call_ids: Final = tuple(tool.get("id") for tool in tool_calls if isinstance(tool, dict))
+        raise litellm.BadRequestError(
+            message=f"Unable to convert openai tool calls with ids={tool_call_ids} to bedrock tool calls. "
+            f"Received error={e}",
+            model=model or "",
+            llm_provider="bedrock",
+        ) from e
 
 
 def _append_bedrock_tool_result_media_block(

litellm/litellm_core_utils/realtime_streaming.py

@@ -745,6 +745,8 @@ class RealTimeStreaming:
         for callback in litellm.callbacks:
             if not isinstance(callback, CustomGuardrail):
                 continue
+            if callback.use_native_lifecycle_hooks:
+                continue
             if id(callback) in _already_run:
                 continue
             if not any(callback.should_run_guardrail(data=_check_data, event_type=et) for et in _realtime_event_types):

litellm/litellm_core_utils/redact_messages.py

@@ -258,6 +258,12 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
             # For async objects, return a simple redacted response without deepcopy
             return {"text": "redacted-by-litellm"}
 
+        if not (
+            isinstance(result, (litellm.ModelResponse, litellm.ResponsesAPIResponse, litellm.EmbeddingResponse))
+            or (isinstance(result, dict) and ("choices" in result or "output" in result))
+        ):
+            return {"text": "redacted-by-litellm"}
+
         _result: Final = copy.deepcopy(result)
         if isinstance(_result, litellm.ModelResponse):
             if hasattr(_result, "choices") and _result.choices is not None:

litellm/litellm_core_utils/safe_json_dumps.py

@@ -1,4 +1,5 @@
 import json
+from collections.abc import Callable
 from typing import Any, Final
 
 from pydantic import BaseModel
@@ -11,20 +12,32 @@ def strip_null_bytes(value: str) -> str:
     return value.replace("\x00", "")
 
 
-def safe_dumps(data: Any, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH) -> str:
+def safe_dumps(
+    data: Any,
+    max_depth: int = DEFAULT_MAX_RECURSE_DEPTH,
+    value_transform: Callable[[str | None, str], str] | None = None,
+) -> str:
     """
     Recursively serialize data while detecting circular references.
     If a circular reference is detected then a marker string is returned.
     NUL bytes are stripped from strings to prevent PostgreSQL 22P05 errors.
+
+    value_transform, when given, is applied to every string leaf (and to the
+    str() fallback for non-serializable objects) with the mapping key the leaf
+    was reached under, so callers can rewrite values without touching structure.
     """
 
-    def _serialize(obj: Any, seen: set, depth: int) -> Any:
+    def _transform(key: str | None, value: str) -> str:
+        return value if value_transform is None else value_transform(key, value)
+
+    def _serialize(obj: Any, seen: set, depth: int, key: str | None = None) -> Any:
         # Check for maximum depth.
         if depth > max_depth:
             return "MaxDepthExceeded"
         # Base-case: if it is a primitive, simply return it.
         if isinstance(obj, str):
-            return obj.replace("\x00", "") if "\x00" in obj else obj
+            cleaned = obj.replace("\x00", "") if "\x00" in obj else obj
+            return _transform(key, cleaned)
         if isinstance(obj, (int, float, bool, type(None))):
             return obj
         # Check for circular reference.
@@ -37,30 +50,30 @@ def safe_dumps(data: Any, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH) -> str:
             for k, v in obj.items():
                 if isinstance(k, (str)):
                     clean_k = k.replace("\x00", "") if "\x00" in k else k
-                    result[clean_k] = _serialize(v, seen, depth + 1)
+                    result[clean_k] = _serialize(v, seen, depth + 1, clean_k)
             seen.remove(id(obj))
             return result
         elif isinstance(obj, list):
-            result = [_serialize(item, seen, depth + 1) for item in obj]
+            result = [_serialize(item, seen, depth + 1, key) for item in obj]
             seen.remove(id(obj))
             return result
         elif isinstance(obj, tuple):
-            result = tuple(_serialize(item, seen, depth + 1) for item in obj)
+            result = tuple(_serialize(item, seen, depth + 1, key) for item in obj)
             seen.remove(id(obj))
             return result
         elif isinstance(obj, set):
-            result = sorted([_serialize(item, seen, depth + 1) for item in obj])
+            result = sorted([_serialize(item, seen, depth + 1, key) for item in obj])
             seen.remove(id(obj))
             return result
         elif isinstance(obj, BaseModel):
             dumped: Final = obj.model_dump()
-            result = _serialize(dumped, seen, depth + 1)
+            result = _serialize(dumped, seen, depth + 1, key)
             seen.remove(id(obj))
             return result
         else:
             # Fall back to string conversion for non-serializable objects.
             try:
-                return strip_null_bytes(str(obj))
+                return _transform(key, strip_null_bytes(str(obj)))
             except Exception:
                 return "Unserializable Object"
 

litellm/litellm_core_utils/secret_redaction.py

@@ -24,9 +24,6 @@ def _build_secret_patterns() -> "re.Pattern[str]":
         r"(?:client_secret|azure_password|azure_username)\s+[^\s,'\"})\]{}>]+",
         # AWS access key IDs
         r"(?:AKIA|ASIA)[0-9A-Z]{16}",
-        # AWS secrets / session tokens / access key IDs (key=value)
-        r"(?:aws_secret_access_key|aws_session_token|aws_access_key_id)"
-        r"\s*[:=]\s*[A-Za-z0-9/+=]{20,}",
         # Bearer tokens (OAuth, JWT, etc.)
         r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*",
         # Basic auth headers
@@ -61,6 +58,7 @@ def _build_secret_patterns() -> "re.Pattern[str]":
         # private_key with PEM-aware value capture
         r"""private_key['\"]?\s*[:=]\s*['\"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'\"})\]{}>]+)""",
         r"(?:master_key|xai_key|database_url|db_url|connection_string|"
+        r"aws_secret_access_key|aws_session_token|aws_access_key_id|"
         r"signing_key|encryption_key|"
         r"auth_token|access_token|refresh_token|"
         r"slack_webhook_url|webhook_url|"
@@ -83,3 +81,19 @@ _SECRET_RE: Final = _build_secret_patterns()
 def redact_string(value: str) -> str:
     """Scrub known secret/credential patterns from *value* and return the result."""
     return _SECRET_RE.sub(_REDACTED, value)
+
+
+def redact_structured_value(key: str | None, value: str) -> str:
+    """Scrub *value* as it appeared under *key* inside a structured record.
+
+    redact_string() replaces a whole ``key: value`` span with REDACTED, which is
+    fine inside free text but destroys the surrounding syntax when the span is a
+    JSON member rather than message content. This renders the pair the way a dict
+    repr would, so the key-name patterns still fire, but collapses only the value
+    so the caller's structure survives.
+    """
+    scrubbed: Final = redact_string(value)
+    if scrubbed != value or key is None:
+        return scrubbed
+    rendered: Final = f"'{key}': '{value}'"
+    return _REDACTED if redact_string(rendered) != rendered else value

litellm/litellm_core_utils/streaming_chunk_builder_utils.py

@@ -3,7 +3,9 @@ import time
 from collections.abc import Iterator, Mapping, Sequence
 from itertools import groupby
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, TypedDict, Union, cast
+from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypedDict, Union, cast
+
+from typing_extensions import ReadOnly, Required
 
 from litellm._logging import verbose_logger
 from litellm.types.llms.openai import (
@@ -14,6 +16,9 @@ from litellm.types.utils import (
     CacheCreationTokenDetails,
     ChatCompletionAudioResponse,
     ChatCompletionCustomToolCallPayload,
+    ChatCompletionDeltaCustomToolCall,
+    ChatCompletionDeltaCustomToolCallPayload,
+    ChatCompletionDeltaToolCall,
     ChatCompletionMessageCustomToolCall,
     ChatCompletionMessageToolCall,
     Choices,
@@ -25,6 +30,7 @@ from litellm.types.utils import (
     ModelResponseStream,
     PromptTokensDetailsWrapper,
     ServerToolUse,
+    StreamingChoices,
     Usage,
 )
 from litellm.utils import print_verbose, token_counter
@@ -79,6 +85,51 @@ class _AudioChunk(TypedDict):
     choices: Sequence[_AudioChoice]
 
 
+_ChunkHiddenParams: TypeAlias = dict[str, object]
+
+
+class _BaseChunk(TypedDict, total=False):
+    id: ReadOnly[str]
+    object: ReadOnly[str]
+    created: ReadOnly[int]
+    model: ReadOnly[str]
+    system_fingerprint: ReadOnly[str | None]
+    choices: ReadOnly[Required[Sequence[StreamingChoices]]]
+    _hidden_params: ReadOnly[_ChunkHiddenParams]
+
+
+class _ToolCallFunctionFragment(TypedDict, total=False):
+    name: ReadOnly[str]
+    arguments: ReadOnly[str]
+    provider_specific_fields: ReadOnly[dict[str, object]]
+
+
+class _ToolCallCustomFragment(TypedDict, total=False):
+    name: ReadOnly[str]
+    input: ReadOnly[str]
+
+
+class _ToolCallFragment(TypedDict, total=False):
+    index: ReadOnly[int]
+    id: ReadOnly[str | None]
+    type: ReadOnly[str | None]
+    function: ReadOnly[_ToolCallFunctionFragment | Function | None]
+    custom: ReadOnly[_ToolCallCustomFragment | None]
+    provider_specific_fields: ReadOnly[dict[str, object] | None]
+
+
+class _ToolCallDelta(TypedDict, total=False):
+    tool_calls: ReadOnly[Sequence[_ToolCallFragment | ChatCompletionDeltaToolCall | ChatCompletionDeltaCustomToolCall]]
+
+
+class _ToolCallChoice(TypedDict, total=False):
+    delta: ReadOnly[_ToolCallDelta]
+
+
+class _ToolCallChunk(TypedDict):
+    choices: ReadOnly[Sequence[_ToolCallChoice]]
+
+
 class _UsageBearingChunk(TypedDict, total=False):
     usage: Usage | None
     _hidden_params: Mapping[str, str]
@@ -158,7 +209,7 @@ class ChunkProcessor:
         return chunks
 
     def update_model_response_with_hidden_params(
-        self, model_response: ModelResponse, chunk: Mapping[str, dict[str, object]] | None = None
+        self, model_response: ModelResponse, chunk: "_BaseChunk | None" = None
     ) -> ModelResponse:
         if chunk is None:
             return model_response
@@ -214,18 +265,18 @@ class ChunkProcessor:
             )
 
     @staticmethod
-    def _get_chunk_id(chunks: Sequence[Mapping[str, str]]) -> str:
+    def _get_chunk_id(chunks: Sequence["_BaseChunk"]) -> str:
         """
         Chunks:
         [{"id": ""}, {"id": "1"}, {"id": "1"}]
         """
         for chunk in chunks:
-            if chunk.get("id"):
-                return chunk["id"]
+            if chunk_id := chunk.get("id"):
+                return chunk_id
         return ""
 
     @staticmethod
-    def _get_model_from_chunks(chunks: Sequence[Mapping[str, str]], first_chunk_model: str) -> str:
+    def _get_model_from_chunks(chunks: Sequence["_BaseChunk"], first_chunk_model: str) -> str:
         """
         Get the actual model from chunks, preferring a model that differs from the first chunk.
 
@@ -241,7 +292,7 @@ class ChunkProcessor:
         # Fall back to first chunk's model if no different model found
         return first_chunk_model
 
-    def build_base_response(self, chunks: list[dict[str, Any]]) -> ModelResponse:
+    def build_base_response(self, chunks: Sequence["_BaseChunk"]) -> ModelResponse:
         chunk = self.first_chunk
         id: Final = ChunkProcessor._get_chunk_id(chunks)
         object: Final = chunk["object"]
@@ -292,7 +343,7 @@ class ChunkProcessor:
 
     @staticmethod
     def _iter_tool_call_fragments(
-        tool_call_chunks: Sequence[Mapping[str, Any]],
+        tool_call_chunks: Sequence["_ToolCallChunk"],
     ) -> Iterator[tuple[int, str, str]]:
         for chunk in tool_call_chunks:
             for choice in chunk["choices"]:
@@ -306,21 +357,21 @@ class ChunkProcessor:
                         index = tool_call.get("index", 0)
                         function = tool_call.get("function")
                         if isinstance(function, dict):
-                            if function.get("arguments"):
-                                yield index, "arguments", function["arguments"]
-                        elif getattr(function, "arguments", None):
-                            yield index, "arguments", function.arguments
+                            if fragment_arguments := function.get("arguments"):
+                                yield index, "arguments", fragment_arguments
+                        elif function_arguments := getattr(function, "arguments", None):
+                            yield index, "arguments", function_arguments
                         custom = tool_call.get("custom")
-                        if isinstance(custom, dict) and custom.get("input"):
-                            yield index, "custom_input", custom["input"]
+                        if isinstance(custom, dict) and (custom_input := custom.get("input")):
+                            yield index, "custom_input", custom_input
                     else:
                         index = getattr(tool_call, "index", 0)
                         function = getattr(tool_call, "function", None)
-                        if getattr(function, "arguments", None):
-                            yield index, "arguments", function.arguments
+                        if object_arguments := getattr(function, "arguments", None):
+                            yield index, "arguments", object_arguments
                         custom = getattr(tool_call, "custom", None)
-                        if getattr(custom, "input", None):
-                            yield index, "custom_input", custom.input
+                        if object_custom_input := getattr(custom, "input", None):
+                            yield index, "custom_input", object_custom_input
 
     @staticmethod
     def _join_fragments_by_index_and_field(
@@ -337,7 +388,7 @@ class ChunkProcessor:
         )
 
     def get_combined_tool_content(
-        self, tool_call_chunks: Sequence[Mapping[str, Any]]
+        self, tool_call_chunks: Sequence["_ToolCallChunk"]
     ) -> list[
         ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall
     ]:  # mutable-ok: assigned verbatim to Message.tool_calls, a list field
@@ -364,7 +415,7 @@ class ChunkProcessor:
                         has_function = "function" in tool_call and tool_call["function"] is not None
                         has_custom = "custom" in tool_call and tool_call["custom"] is not None
                     else:
-                        has_function = hasattr(tool_call, "function") and tool_call.function is not None
+                        has_function = getattr(tool_call, "function", None) is not None
                         has_custom = getattr(tool_call, "custom", None) is not None
 
                     if not has_function and not has_custom:
@@ -387,61 +438,67 @@ class ChunkProcessor:
 
                     # Extract id, type, and function data (handle both dict and object)
                     if isinstance(tool_call, dict):
-                        if tool_call.get("id"):
-                            tool_call_map[index]["id"] = tool_call["id"]
-                        if tool_call.get("type"):
-                            tool_call_map[index]["type"] = tool_call["type"]
+                        if fragment_id := tool_call.get("id"):
+                            tool_call_map[index]["id"] = fragment_id
+                        if fragment_type := tool_call.get("type"):
+                            tool_call_map[index]["type"] = fragment_type
 
                         function = tool_call.get("function", {})
                         if isinstance(function, dict):
-                            if function.get("name"):
-                                tool_call_map[index]["name"] = function["name"]
+                            if fragment_name := function.get("name"):
+                                tool_call_map[index]["name"] = fragment_name
                         else:
                             # function is an object
-                            if hasattr(function, "name") and function.name:
-                                tool_call_map[index]["name"] = function.name
+                            if function_name := getattr(function, "name", None):
+                                tool_call_map[index]["name"] = function_name
 
                         custom = tool_call.get("custom")
                         if isinstance(custom, dict):
-                            if custom.get("name"):
-                                tool_call_map[index]["custom_name"] = custom["name"]
+                            if custom_name := custom.get("name"):
+                                tool_call_map[index]["custom_name"] = custom_name
                     else:
                         # tool_call is an object
                         if hasattr(tool_call, "id") and tool_call.id:
                             tool_call_map[index]["id"] = tool_call.id
                         if hasattr(tool_call, "type") and tool_call.type:
                             tool_call_map[index]["type"] = tool_call.type
-                        if hasattr(tool_call, "function"):
-                            if hasattr(tool_call.function, "name") and tool_call.function.name:
-                                tool_call_map[index]["name"] = tool_call.function.name
+                        if object_function_name := getattr(getattr(tool_call, "function", None), "name", None):
+                            tool_call_map[index]["name"] = object_function_name
 
-                        custom = getattr(tool_call, "custom", None)
-                        if custom is not None:
-                            if getattr(custom, "name", None):
-                                tool_call_map[index]["custom_name"] = custom.name
+                        object_custom: ChatCompletionDeltaCustomToolCallPayload | None = getattr(
+                            tool_call, "custom", None
+                        )
+                        if object_custom is not None:
+                            if getattr(object_custom, "name", None):
+                                tool_call_map[index]["custom_name"] = object_custom.name
 
                     # Preserve provider_specific_fields from streaming chunks
-                    provider_fields = None
+                    provider_fields: object = None
                     if isinstance(tool_call, dict):
                         provider_fields = tool_call.get("provider_specific_fields")
-                        if not provider_fields and isinstance(tool_call.get("function"), dict):
-                            provider_fields = tool_call["function"].get("provider_specific_fields")
+                        if not provider_fields and isinstance(fragment_function := tool_call.get("function"), dict):
+                            provider_fields = fragment_function.get("provider_specific_fields")
                     else:
-                        if hasattr(tool_call, "provider_specific_fields") and tool_call.provider_specific_fields:
-                            provider_fields = tool_call.provider_specific_fields
-                        elif (
-                            hasattr(tool_call, "function")
-                            and hasattr(tool_call.function, "provider_specific_fields")
-                            and tool_call.function.provider_specific_fields
-                        ):
-                            provider_fields = tool_call.function.provider_specific_fields
+                        object_provider_fields: object = getattr(tool_call, "provider_specific_fields", None)
+                        if object_provider_fields:
+                            provider_fields = object_provider_fields
+                        else:
+                            function_provider_fields: object = getattr(
+                                getattr(tool_call, "function", None),
+                                "provider_specific_fields",
+                                None,
+                            )
+                            if function_provider_fields:
+                                provider_fields = function_provider_fields
 
                     if provider_fields:
                         # Merge provider_specific_fields if multiple chunks have them
-                        if tool_call_map[index]["provider_specific_fields"] is None:
-                            tool_call_map[index]["provider_specific_fields"] = {}
+                        merged_provider_fields = tool_call_map[index]["provider_specific_fields"]
+                        if merged_provider_fields is None:
+                            merged_provider_fields = {}
+                            tool_call_map[index]["provider_specific_fields"] = merged_provider_fields
                         if isinstance(provider_fields, dict):
-                            tool_call_map[index]["provider_specific_fields"].update(provider_fields)
+                            merged_provider_fields.update(provider_fields)
 
         joined_fragments: Final = self._join_fragments_by_index_and_field(
             self._iter_tool_call_fragments(tool_call_chunks)
@@ -762,19 +819,14 @@ class ChunkProcessor:
                         server_tool_use = usage_chunk.server_tool_use
                     else:
                         server_tool_use = ServerToolUse.model_validate(usage_chunk.server_tool_use)
-                if (
-                    usage_chunk_dict["prompt_tokens_details"] is not None
-                    and getattr(
+                if usage_chunk_dict["prompt_tokens_details"] is not None:
+                    chunk_web_search_requests: int | None = getattr(
                         usage_chunk_dict["prompt_tokens_details"],
                         "web_search_requests",
                         None,
                     )
-                    is not None
-                ):
-                    web_search_requests = getattr(
-                        usage_chunk_dict["prompt_tokens_details"],
-                        "web_search_requests",
-                    )
+                    if chunk_web_search_requests is not None:
+                        web_search_requests = chunk_web_search_requests
 
                 prompt_tokens_details = usage_chunk_dict["prompt_tokens_details"] or prompt_tokens_details
 

litellm/litellm_core_utils/streaming_handler.py

@@ -6,7 +6,7 @@ import logging
 import threading
 import time
 import traceback
-from collections.abc import AsyncIterator, Callable, Iterator, Mapping, Sequence
+from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
 from dataclasses import dataclass
 from typing import Any, Final, NoReturn, Protocol, TypeVar, cast
 
@@ -155,6 +155,33 @@ class _TextCompletionChoiceLike(Protocol):
     finish_reason: str | None
 
 
+class _VertexFunctionCallLike(Protocol):
+    name: str
+    args: Mapping[str, Iterable[object]]
+
+
+class _VertexPartLike(Protocol):
+    function_call: _VertexFunctionCallLike
+
+
+class _VertexContentLike(Protocol):
+    parts: Sequence[_VertexPartLike]
+
+
+class _VertexFinishReasonLike(Protocol):
+    name: str
+
+
+class _VertexCandidateLike(Protocol):
+    content: _VertexContentLike
+    finish_reason: _VertexFinishReasonLike
+
+
+class _VertexChunkLike(Protocol):
+    text: str
+    candidates: Sequence[_VertexCandidateLike]
+
+
 class CustomStreamWrapper:
     def __init__(
         self,
@@ -291,13 +318,13 @@ class CustomStreamWrapper:
         that has since taken over the same Task/thread's context.
         """
         try:
-            logging_obj: Final = getattr(self, "logging_obj", None)
+            logging_obj: Final[object | None] = getattr(self, "logging_obj", None)
             if logging_obj is None:
                 return
             method_name: Final = (
                 "_restore_correlation_context_if_unclaimed" if guarded else "_restore_correlation_context"
             )
-            restore: Final = getattr(logging_obj, method_name, None)
+            restore: Final[Callable[[], object] | None] = getattr(logging_obj, method_name, None)
             if restore is not None:
                 restore()
         except Exception as restore_error:  # noqa: BLE001  # best-effort cleanup; must not raise into the caller
@@ -1261,18 +1288,18 @@ class CustomStreamWrapper:
                         raise Exception("An unknown error occurred with the stream")
                     self.received_finish_reason = "stop"
         elif self.custom_llm_provider == "vertex_ai" and not isinstance(chunk, ModelResponseStream):
-            chunk = cast(Any, chunk)
+            vertex_chunk: Final = cast(_VertexChunkLike, chunk)
             import proto
 
-            if hasattr(chunk, "candidates") is True:
+            if hasattr(vertex_chunk, "candidates") is True:
                 try:
                     try:
-                        completion_obj["content"] = chunk.text
+                        completion_obj["content"] = vertex_chunk.text
                     except Exception as e:
                         original_exception: Final = e
                         if "Part has no text." in str(e):
                             ## check for function calling
-                            function_call: Final = chunk.candidates[0].content.parts[0].function_call
+                            function_call: Final = vertex_chunk.candidates[0].content.parts[0].function_call
 
                             args_dict: Final = {}
 
@@ -1311,15 +1338,15 @@ class CustomStreamWrapper:
                         else:
                             raise original_exception
                     if (
-                        hasattr(chunk.candidates[0], "finish_reason")
-                        and chunk.candidates[0].finish_reason.name != "FINISH_REASON_UNSPECIFIED"
+                        hasattr(vertex_chunk.candidates[0], "finish_reason")
+                        and vertex_chunk.candidates[0].finish_reason.name != "FINISH_REASON_UNSPECIFIED"
                     ):  # every non-final chunk in vertex ai has this
-                        self.received_finish_reason = map_finish_reason(chunk.candidates[0].finish_reason.name)
+                        self.received_finish_reason = map_finish_reason(vertex_chunk.candidates[0].finish_reason.name)
                 except Exception:
-                    if chunk.candidates[0].finish_reason.name == "SAFETY":
-                        raise Exception(f"The response was blocked by VertexAI. {chunk}")
+                    if vertex_chunk.candidates[0].finish_reason.name == "SAFETY":
+                        raise Exception(f"The response was blocked by VertexAI. {vertex_chunk}")
             else:
-                completion_obj["content"] = str(chunk)
+                completion_obj["content"] = str(vertex_chunk)
         elif self.custom_llm_provider == "petals":
             if self.completion_stream is None or len(self.completion_stream) == 0:
                 if self.received_finish_reason is not None:
@@ -1357,13 +1384,14 @@ class CustomStreamWrapper:
             if response_obj["is_finished"]:
                 self.received_finish_reason = response_obj["finish_reason"]
             if response_obj["usage"] is not None:
+                _text_completion_usage: Final[Usage] = response_obj["usage"]
                 setattr(
                     model_response,
                     "usage",
                     litellm.Usage(
-                        prompt_tokens=response_obj["usage"].prompt_tokens,
-                        completion_tokens=response_obj["usage"].completion_tokens,
-                        total_tokens=response_obj["usage"].total_tokens,
+                        prompt_tokens=_text_completion_usage.prompt_tokens,
+                        completion_tokens=_text_completion_usage.completion_tokens,
+                        total_tokens=_text_completion_usage.total_tokens,
                     ),
                 )
         elif self.custom_llm_provider == "text-completion-codestral":
@@ -1395,15 +1423,17 @@ class CustomStreamWrapper:
             if response_obj["is_finished"]:
                 self.received_finish_reason = response_obj["finish_reason"]
         elif self.custom_llm_provider == "cached_response":
-            chunk = cast(ModelResponseStream, chunk)
-            chunk_finish_reason: Final = chunk.choices[0].finish_reason
+            cached_chunk: Final = cast(ModelResponseStream, chunk)
+            chunk_finish_reason: Final = cached_chunk.choices[0].finish_reason
             response_obj = {
-                "text": chunk.choices[0].delta.content,
+                "text": cached_chunk.choices[0].delta.content,
                 "is_finished": chunk_finish_reason is not None,
                 "finish_reason": chunk_finish_reason,
-                "original_chunk": chunk,
+                "original_chunk": cached_chunk,
                 "tool_calls": (
-                    chunk.choices[0].delta.tool_calls if hasattr(chunk.choices[0].delta, "tool_calls") else None
+                    cached_chunk.choices[0].delta.tool_calls
+                    if hasattr(cached_chunk.choices[0].delta, "tool_calls")
+                    else None
                 ),
             }
 
@@ -1411,11 +1441,11 @@ class CustomStreamWrapper:
             if response_obj["tool_calls"] is not None:
                 completion_obj["tool_calls"] = response_obj["tool_calls"]
             print_verbose(f"completion obj content: {completion_obj['content']}")
-            if hasattr(chunk, "id"):
-                model_response.id = chunk.id
-                self.response_id = chunk.id
-            if hasattr(chunk, "system_fingerprint"):
-                self.system_fingerprint = chunk.system_fingerprint
+            if hasattr(cached_chunk, "id"):
+                model_response.id = cached_chunk.id
+                self.response_id = cached_chunk.id
+            if hasattr(cached_chunk, "system_fingerprint"):
+                self.system_fingerprint = cached_chunk.system_fingerprint
             if response_obj["is_finished"]:
                 self.received_finish_reason = response_obj["finish_reason"]
         else:  # openai / azure chat model
@@ -1563,6 +1593,7 @@ class CustomStreamWrapper:
                     if self.stream_options is not None and self.stream_options["include_usage"] is True:
                         model_response.choices = []
                         return model_response
+                    self._record_usage_only_chunk(model_response=model_response)
                     return
             ## CHECK FOR TOOL USE
 
@@ -1789,6 +1820,16 @@ class CustomStreamWrapper:
             model_response.choices[0].finish_reason = "tool_calls"
         return model_response
 
+    def _record_usage_only_chunk(self, model_response: "ModelResponseStream") -> None:
+        """
+        Keep provider usage-only chunks (e.g. OpenRouter's post-finish chunk, which carries a
+        provider-reported cost) available to cost tracking. They are never returned to the
+        caller; ``stream_options.include_usage`` only controls what the caller sees.
+        """
+        if getattr(model_response, "usage", None) is None:
+            return
+        self.chunks.append(model_response.model_copy(update={"choices": []}))
+
     @staticmethod
     def _propagate_usage_cost_to_hidden_params(
         response: "ModelResponse",
@@ -2310,16 +2351,16 @@ class CustomStreamWrapper:
         def _normalize_status_code(exc: Exception) -> int | None:
             """Best-effort status_code extraction."""
             try:
-                code: Final = getattr(exc, "status_code", None)
+                code: Final[int | str | None] = getattr(exc, "status_code", None)
                 if code is not None:
                     return int(code)
             except Exception:
                 pass
 
-            response: Final = getattr(exc, "response", None)
+            response: Final[object | None] = getattr(exc, "response", None)
             if response is not None:
                 try:
-                    status_code: Final = getattr(response, "status_code", None)
+                    status_code: Final[int | str | None] = getattr(response, "status_code", None)
                     if status_code is not None:
                         return int(status_code)
                 except Exception:

litellm/llms/anthropic/chat/guardrail_translation/handler.py

@@ -13,7 +13,7 @@ Pattern Overview:
 """
 
 import json
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from copy import deepcopy
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, Final, cast
@@ -61,6 +61,7 @@ if TYPE_CHECKING:
         ModifyResponseException,
     )
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+    from litellm.proxy._types import UserAPIKeyAuth
     from litellm.types.llms.anthropic_messages.anthropic_response import (
         AnthropicMessagesResponse,
     )
@@ -123,7 +124,7 @@ class AnthropicMessagesHandler(BaseTranslation):
 
     @staticmethod
     def _build_streaming_usage_response(
-        responses_so_far: list[Any],
+        responses_so_far: list[object],
         request_data: dict | None,
     ) -> ModelResponse | None:
         chunks: Final = tuple(response for response in responses_so_far if isinstance(response, (str, bytes)))
@@ -141,7 +142,7 @@ class AnthropicMessagesHandler(BaseTranslation):
         self,
         exc: "ModifyResponseException",
         stream_started: bool = False,
-        responses_so_far: list[Any] | None = None,
+        responses_so_far: list[object] | None = None,
     ) -> list[bytes]:
         """
         Build an Anthropic SSE sequence delivering the guardrail block message
@@ -184,7 +185,7 @@ class AnthropicMessagesHandler(BaseTranslation):
         )
         return list(FakeAnthropicMessagesStreamIterator(response=block_response))
 
-    def _block_continuation_chunks(self, exc: "ModifyResponseException", responses_so_far: list[Any]) -> list[bytes]:
+    def _block_continuation_chunks(self, exc: "ModifyResponseException", responses_so_far: list[object]) -> list[bytes]:
         """Continue an already-started message: close the open content block,
         append the block message as a new text block, then end the message --
         without a second message_start."""
@@ -234,7 +235,7 @@ class AnthropicMessagesHandler(BaseTranslation):
 
     @staticmethod
     def _content_block_state(
-        responses_so_far: list[Any],
+        responses_so_far: list[object],
     ) -> tuple[int | None, int | None]:
         """From the SSE chunks already sent to the client, return (open
         content-block index or None, highest content-block index seen or None).
@@ -260,7 +261,7 @@ class AnthropicMessagesHandler(BaseTranslation):
         return open_index, max_index
 
     @staticmethod
-    def _iter_sse_events(item: Any) -> list[dict]:
+    def _iter_sse_events(item: object) -> list[dict[str, object]]:
         """Yield the event-data dicts in one stream chunk.
 
         Handles both formats this stream can carry (see
@@ -271,14 +272,16 @@ class AnthropicMessagesHandler(BaseTranslation):
             return [item]
         if not isinstance(item, (bytes, bytearray)):
             return []
-        events: Final[list[dict]] = []
+        events: Final[list[dict[str, object]]] = []
         for block in item.decode("utf-8", errors="replace").split("\n\n"):
             for line in block.split("\n"):
                 line = line.strip()
                 if not line.startswith("data:"):
                     continue
                 try:
-                    parsed = json.loads(line[len("data:") :].strip())
+                    parsed: str | int | float | bool | None | Sequence[object] | Mapping[str, object] = json.loads(
+                        line[len("data:") :].strip()
+                    )
                 except json.JSONDecodeError:
                     continue
                 if isinstance(parsed, dict):
@@ -315,7 +318,7 @@ class AnthropicMessagesHandler(BaseTranslation):
         self,
         data: dict,
         guardrail_to_apply: "CustomGuardrail",
-        litellm_logging_obj: Any | None = None,
+        litellm_logging_obj: "LiteLLMLoggingObj | None" = None,
     ) -> Any:
         """
         Process input messages by applying guardrails to text content.
@@ -467,8 +470,8 @@ class AnthropicMessagesHandler(BaseTranslation):
 
     @staticmethod
     def _openai_system_message_to_anthropic(
-        message: dict[str, Any],
-    ) -> dict[str, Any] | None:  # mutable-ok: API message payload
+        message: dict[str, object],
+    ) -> dict[str, object] | None:  # mutable-ok: API message payload
         """Convert an OpenAI system message to the client's Anthropic-shaped entry."""
         content: Final = message.get("content")
         if isinstance(content, str):
@@ -477,14 +480,14 @@ class AnthropicMessagesHandler(BaseTranslation):
             )  # mutable-ok: API message payload
         if not isinstance(content, list):
             return None
-        blocks: Final[list[dict[str, Any]]] = []  # mutable-ok: API message payload
+        blocks: Final[list[dict[str, object]]] = []  # mutable-ok: API message payload
         for block in content:
             if not isinstance(block, dict) or block.get("type") != "text":
                 continue
             text = block.get("text")
             if not isinstance(text, str) or not text:
                 continue
-            anthropic_block: dict[str, Any] = {  # mutable-ok: API message payload
+            anthropic_block: dict[str, object] = {  # mutable-ok: API message payload
                 "type": "text",
                 "text": text,
             }  # mutable-ok: API message payload
@@ -496,6 +499,39 @@ class AnthropicMessagesHandler(BaseTranslation):
             {"role": "system", "content": blocks} if blocks else None  # mutable-ok: API message payload
         )  # mutable-ok: API message payload
 
+    @staticmethod
+    def _fold_leading_systems_into_top_level(
+        data: dict[str, object],  # mutable-ok: API message payload
+        leading_systems: Sequence[object],
+        include_existing_system: bool,
+    ) -> None:
+        """Deliver leading system rows through Anthropic's top-level system param, which rejects them in messages."""
+        existing: Final = data.get("system") if include_existing_system else None
+        existing_blocks: Final[list[object]] = (  # mutable-ok: API message payload
+            [{"type": "text", "text": existing}]
+            if isinstance(existing, str) and existing
+            else list(existing)
+            if isinstance(existing, list)
+            else []
+        )
+        converted_rows: Final = tuple(
+            AnthropicMessagesHandler._openai_system_message_to_anthropic(message)
+            for message in leading_systems
+            if isinstance(message, dict)
+        )
+        folded: Final[list[object]] = existing_blocks + [  # mutable-ok: API message payload
+            block
+            for row in converted_rows
+            if row is not None
+            for block in (
+                [{"type": "text", "text": row["content"]}] if isinstance(row["content"], str) else row["content"]
+            )
+        ]
+        if folded:
+            data["system"] = folded  # rebind-ok: write-back mutates the request payload in place
+        else:
+            data.pop("system", None)
+
     @staticmethod
     def _is_hoisted_top_level_system(message: object, hoisted_system_message: object) -> bool:
         """Match the hoisted prompt by identity, or by value after serialization."""
@@ -572,9 +608,24 @@ class AnthropicMessagesHandler(BaseTranslation):
                 )
 
         ordered: Final = AnthropicMessagesHandler._defer_systems_inside_tool_exchanges(structured_messages)
+        leading_count: Final = next(
+            (index for index, message in enumerate(ordered) if not _is_system(message)),
+            len(ordered),
+        )
+        leading_systems: Final = ordered[:leading_count]
+        hoisted_in_leading: Final = any(
+            AnthropicMessagesHandler._is_hoisted_top_level_system(message, hoisted_system_message)
+            for message in leading_systems
+        )
+        if leading_systems and not (leading_count == 1 and hoisted_in_leading):
+            AnthropicMessagesHandler._fold_leading_systems_into_top_level(
+                data,
+                leading_systems,
+                include_existing_system=hoisted_system_message is None,
+            )
         run: Final[list] = []  # mutable-ok: API message payload
-        hoisted_dropped = False  # rebind-ok: flips once the hoisted prompt is dropped
-        for message in ordered:
+        hoisted_dropped = hoisted_in_leading  # rebind-ok: flips once the hoisted prompt is dropped
+        for message in ordered[leading_count:]:
             if not _is_system(message):
                 run.append(message)
                 continue
@@ -602,7 +653,7 @@ class AnthropicMessagesHandler(BaseTranslation):
 
     @staticmethod
     def _extract_midturn_system_text(
-        message: dict[str, Any],  # mutable-ok: API message payload
+        message: Mapping[str, object],
         msg_idx: int,
     ) -> ExtractedInput:
         """Match the adapter's filtering so positional guardrail write-back stays aligned."""
@@ -636,7 +687,7 @@ class AnthropicMessagesHandler(BaseTranslation):
     @classmethod
     def _extract_input_text_and_images(
         cls,
-        message: dict[str, Any],
+        message: Mapping[str, object],
         msg_idx: int,
         skip_system_message: bool = False,
         skip_tool_message: bool = False,
@@ -707,7 +758,7 @@ class AnthropicMessagesHandler(BaseTranslation):
     @classmethod
     def _extract_tool_result(
         cls,
-        content_item: Mapping[str, Any],
+        content_item: Mapping[str, object],
         msg_idx: int,
         content_idx: int,
     ) -> ExtractedInput:
@@ -736,7 +787,7 @@ class AnthropicMessagesHandler(BaseTranslation):
         )
 
     @staticmethod
-    def _image_sources(block: Mapping[str, Any]) -> tuple[str, ...]:
+    def _image_sources(block: Mapping[str, object]) -> tuple[str, ...]:
         source: Final = block.get("source")
         if not isinstance(source, Mapping):
             return ()
@@ -746,7 +797,7 @@ class AnthropicMessagesHandler(BaseTranslation):
 
     async def _apply_guardrail_responses_to_input(
         self,
-        messages: list[dict[str, Any]],
+        messages: list[dict[str, object]],
         responses: list[str],
         scanned: tuple[ScannedText, ...],
     ) -> None:
@@ -788,10 +839,10 @@ class AnthropicMessagesHandler(BaseTranslation):
         self,
         response: "AnthropicMessagesResponse",
         guardrail_to_apply: "CustomGuardrail",
-        litellm_logging_obj: Any | None = None,
-        user_api_key_dict: Any | None = None,
+        litellm_logging_obj: "LiteLLMLoggingObj | None" = None,
+        user_api_key_dict: "UserAPIKeyAuth | None" = None,
         request_data: dict | None = None,
-    ) -> Any:
+    ) -> "AnthropicMessagesResponse":
         """
         Process output response by applying guardrails to text content and tool calls.
 
@@ -869,8 +920,8 @@ class AnthropicMessagesHandler(BaseTranslation):
         self,
         responses_so_far: list[Any],
         guardrail_to_apply: "CustomGuardrail",
-        litellm_logging_obj: Any | None = None,
-        user_api_key_dict: Any | None = None,
+        litellm_logging_obj: "LiteLLMLoggingObj | None" = None,
+        user_api_key_dict: "UserAPIKeyAuth | None" = None,
         request_data: dict | None = None,
     ) -> list[Any]:
         """
@@ -950,8 +1001,8 @@ class AnthropicMessagesHandler(BaseTranslation):
     def _prepare_request_data(
         self,
         request_data: dict | None,
-        response: Any,
-        user_api_key_dict: Any | None,
+        response: object,
+        user_api_key_dict: "UserAPIKeyAuth | None",
         key: str,
     ) -> dict:
         """Ensure request_data has the response/responses_so_far key and metadata."""
@@ -968,7 +1019,7 @@ class AnthropicMessagesHandler(BaseTranslation):
         return request_data
 
     @staticmethod
-    def _get_response_content(response: Any) -> list[Any]:
+    def _get_response_content(response: object) -> list[Any]:
         """Extract content list from a dict or object response."""
         if isinstance(response, dict):
             return response.get("content", []) or []
@@ -986,10 +1037,10 @@ class AnthropicMessagesHandler(BaseTranslation):
     ) -> None:
         """Extract text, images, and tool calls from content blocks."""
         for content_idx, content_block in enumerate(response_content):
-            block_dict: dict[str, Any] = {}
+            block_dict: dict[str, object] = {}
             if isinstance(content_block, dict):
                 block_type = content_block.get("type")
-                block_dict = cast(dict[str, Any], content_block)
+                block_dict = cast(dict[str, object], content_block)
             elif hasattr(content_block, "type"):
                 block_type = getattr(content_block, "type", None)
                 if hasattr(content_block, "model_dump"):
@@ -1017,7 +1068,7 @@ class AnthropicMessagesHandler(BaseTranslation):
         texts_to_check: list[str],
         images_to_check: list[str],
         tool_calls_to_check: list["ChatCompletionToolCallChunk"],
-        response: Any,
+        response: object,
     ) -> "GenericGuardrailAPIInputs":
         """Build GenericGuardrailAPIInputs with optional images, tool calls, model."""
         inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check)
@@ -1212,7 +1263,7 @@ class AnthropicMessagesHandler(BaseTranslation):
 
     def _extract_output_text_and_images(
         self,
-        content_block: dict[str, Any],
+        content_block: dict[str, object],
         content_idx: int,
         texts_to_check: list[str],
         images_to_check: list[str],
@@ -1282,7 +1333,7 @@ class AnthropicMessagesHandler(BaseTranslation):
             # Handle both dict and Pydantic object content blocks
             if isinstance(content_block, dict):
                 if content_block.get("type") == "text":
-                    cast(dict[str, Any], content_block)["text"] = guardrail_response
+                    cast(dict[str, object], content_block)["text"] = guardrail_response
             elif hasattr(content_block, "type") and getattr(content_block, "type", None) == "text":
                 # Update Pydantic object's text attribute
                 if hasattr(content_block, "text"):

litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py

@@ -624,6 +624,12 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
                     return self.chunk_queue.popleft()
 
                 if processed_chunk["type"] == "content_block_delta" and not self._delta_has_content(processed_chunk):
+                    # A tool_use block opens with empty arguments (Bedrock Converse's
+                    # ``contentBlockStart``, OpenAI's ``arguments: ""``), so flush the
+                    # block start queued above instead of waiting for the next upstream
+                    # chunk, which on a trailing-burst provider is the whole generation.
+                    if self.chunk_queue:
+                        return self.chunk_queue.popleft()
                     continue
 
                 if processed_chunk["type"] == "message_delta" and self.sent_content_block_finish is False:
@@ -847,6 +853,9 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
                     if processed_chunk["type"] == "content_block_delta" and not self._delta_has_content(
                         processed_chunk
                     ):
+                        # See ``__next__``: flush the queued block start (issue #32004).
+                        if self.chunk_queue:
+                            return self.chunk_queue.popleft()
                         continue
 
                     if processed_chunk["type"] == "message_delta" and self.sent_content_block_finish is False:

litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py

@@ -84,6 +84,7 @@ from litellm.types.llms.anthropic import (
     AnthropicResponseContentBlockText,
     AnthropicResponseContentBlockThinking,
     AnthropicResponseContentBlockToolUse,
+    AnthropicThinkingParam,
     AppliedEdit,
     ContentBlockDelta,
     ContentJsonBlockDelta,
@@ -305,7 +306,7 @@ class LiteLLMAnthropicMessagesAdapter:
                 target["cache_control"] = cache_control
             else:
                 # Fallback for non-dict objects (shouldn't happen in practice)
-                cast(dict[str, Any], target)["cache_control"] = cache_control
+                cast(dict[str, object], target)["cache_control"] = cache_control
 
     def translatable_anthropic_params(self) -> list[str]:
         """
@@ -323,7 +324,7 @@ class LiteLLMAnthropicMessagesAdapter:
             "stop_sequences",
         ]
 
-    def _is_web_search_tool(self, tool: dict[str, Any]) -> bool:
+    def _is_web_search_tool(self, tool: Mapping[str, object]) -> bool:
         """
         Check if a tool is an Anthropic web search tool.
 
@@ -498,7 +499,7 @@ class LiteLLMAnthropicMessagesAdapter:
                             assistant_message_str = str(content)
                         elif isinstance(content, dict):
                             if content.get("type") == "text":
-                                text_block: dict[str, Any] = {
+                                text_block: dict[str, object] = {
                                     "type": "text",
                                     "text": content.get("text", ""),
                                 }
@@ -513,10 +514,12 @@ class LiteLLMAnthropicMessagesAdapter:
                                     "name": tool_name,
                                     "arguments": json.dumps(content.get("input", {})),
                                 }
-                                signature = self._extract_signature_from_tool_use_content(cast(dict[str, Any], content))
+                                signature = self._extract_signature_from_tool_use_content(
+                                    cast(dict[str, object], content)
+                                )
 
                                 if signature:
-                                    provider_specific_fields: dict[str, Any] = (
+                                    provider_specific_fields: dict[str, object] = (
                                         function_chunk.get("provider_specific_fields") or {}
                                     )
                                     provider_specific_fields["thought_signature"] = signature
@@ -575,7 +578,7 @@ class LiteLLMAnthropicMessagesAdapter:
 
     @staticmethod
     def translate_anthropic_thinking_to_reasoning_effort(
-        thinking: dict[str, Any],
+        thinking: AnthropicThinkingParam,
     ) -> str | None:
         """
         Translate Anthropic's thinking parameter to OpenAI's reasoning_effort.
@@ -632,9 +635,9 @@ class LiteLLMAnthropicMessagesAdapter:
 
     @staticmethod
     def translate_thinking_for_model(
-        thinking: dict[str, Any],
+        thinking: AnthropicThinkingParam,
         model: str,
-    ) -> dict[str, Any]:
+    ) -> dict[str, object]:
         """
         Translate Anthropic thinking parameter based on the target model.
 
@@ -670,7 +673,7 @@ class LiteLLMAnthropicMessagesAdapter:
     @staticmethod
     def _apply_reasoning_summary_wrapping(
         reasoning_effort: str,
-        thinking: dict[str, Any],
+        thinking: Mapping[str, object],
     ) -> Any:
         """
         Apply the reasoning_effort/summary wrapping rules shared by every
@@ -731,6 +734,7 @@ class LiteLLMAnthropicMessagesAdapter:
             "input_schema",
             "description",
             "cache_control",
+            "strict",
             "type",
         ]
 
@@ -760,6 +764,8 @@ class LiteLLMAnthropicMessagesAdapter:
                 function_chunk["parameters"] = tool["input_schema"]
             if "description" in tool:
                 function_chunk["description"] = tool["description"]
+            if "strict" in tool:
+                function_chunk["strict"] = bool(tool["strict"])
 
             for k, v in tool.items():
                 if k not in mapped_tool_params:  # pass additional computer kwargs
@@ -770,7 +776,7 @@ class LiteLLMAnthropicMessagesAdapter:
 
         return new_tools, tool_name_mapping
 
-    def translate_anthropic_output_format_to_openai(self, output_format: Any) -> dict[str, Any] | None:
+    def translate_anthropic_output_format_to_openai(self, output_format: Any) -> dict[str, object] | None:
         """
         Translate Anthropic's output_format to OpenAI's response_format.
 
@@ -889,7 +895,7 @@ class LiteLLMAnthropicMessagesAdapter:
             model_name: Final = anthropic_message_request.get("model", "")
             for block in system_content:
                 if isinstance(block, dict) and block.get("type") == "text":
-                    text_block: dict[str, Any] = {
+                    text_block: dict[str, object] = {
                         "type": "text",
                         "text": block.get("text", ""),
                     }
@@ -959,7 +965,7 @@ class LiteLLMAnthropicMessagesAdapter:
         web_search_tools: Final[list[AllAnthropicToolsValues]] = []
         regular_tools: Final[list[AllAnthropicToolsValues]] = []
         for tool in tools:
-            cast_tool = cast(dict[str, Any], tool)
+            cast_tool = cast(dict[str, object], tool)
             if self._is_web_search_tool(cast_tool):
                 web_search_tools.append(cast(AllAnthropicToolsValues, tool))
             else:
@@ -1007,7 +1013,7 @@ class LiteLLMAnthropicMessagesAdapter:
                         new_kwargs["output_config"] = effort_config  # rebind-ok: out-param store like thinking above
             return
 
-        reasoning_effort = self.translate_anthropic_thinking_to_reasoning_effort(cast(dict[str, Any], thinking))
+        reasoning_effort = self.translate_anthropic_thinking_to_reasoning_effort(cast(AnthropicThinkingParam, thinking))
         if not reasoning_effort:
             return
 
@@ -1020,7 +1026,7 @@ class LiteLLMAnthropicMessagesAdapter:
                 reasoning_effort = output_config["effort"]
 
         new_kwargs["reasoning_effort"] = self._apply_reasoning_summary_wrapping(
-            reasoning_effort, cast(dict[str, Any], thinking)
+            reasoning_effort, cast(dict[str, object], thinking)
         )
 
     def _translate_output_format_to_openai(
@@ -1040,7 +1046,7 @@ class LiteLLMAnthropicMessagesAdapter:
 
         ``output_format`` takes precedence when both are provided.
         """
-        output_format: Any = anthropic_message_request.get("output_format")
+        output_format: object = anthropic_message_request.get("output_format")
         if not output_format:
             output_config: Final = anthropic_message_request.get("output_config")
             if isinstance(output_config, dict):
@@ -1407,7 +1413,7 @@ class LiteLLMAnthropicMessagesAdapter:
                 if THOUGHT_SIGNATURE_SEPARATOR in raw_id:
                     parts = raw_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)
                     thought_sig = parts[1] if len(parts) > 1 else None
-                tool_block: dict[str, Any] = {
+                tool_block: dict[str, object] = {
                     "type": "tool_use",
                     "id": normalize_anthropic_tool_use_id(raw_id),
                     "name": tool_name,

litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py

@@ -16,7 +16,7 @@ How it works:
 
 import uuid
 from collections.abc import AsyncIterator
-from typing import Any, Final
+from typing import TYPE_CHECKING, Any, Final
 
 import litellm
 import litellm.constants as _c
@@ -28,6 +28,9 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
     AnthropicMessagesResponse,
 )
 
+if TYPE_CHECKING:
+    from litellm.router import Router
+
 ADVISOR_MAX_USES: Final[int] = _c.ADVISOR_MAX_USES
 ADVISOR_NATIVE_PROVIDERS: Final[frozenset] = _c.ADVISOR_NATIVE_PROVIDERS
 ADVISOR_TOOL_DESCRIPTION: Final[str] = _c.ADVISOR_TOOL_DESCRIPTION
@@ -97,6 +100,14 @@ class AdvisorOrchestrationHandler(MessagesInterceptor):
 
         parent_request_id: Final[str] = str(kwargs.pop("litellm_call_id", None) or uuid.uuid4())
         metadata_base: Final[dict] = dict(kwargs.pop("metadata", None) or {})
+        advisor_metadata: Final = {
+            **metadata_base,
+            "advisor_sub_call": True,
+            "parent_request_id": parent_request_id,
+        }
+        advisor_router: Final = (
+            None if (advisor_api_key or advisor_api_base) else _resolve_advisor_router(advisor_model)
+        )
         iteration = 0
 
         while True:
@@ -138,20 +149,27 @@ class AdvisorOrchestrationHandler(MessagesInterceptor):
 
             # --- Advisor sub-call (always non-streaming, no tools) ---
             try:
-                advisor_response: AnthropicMessagesResponse = await _call_messages_handler(
-                    model=advisor_model,
-                    messages=advisor_messages,
-                    tools=None,
-                    stream=False,
-                    max_tokens=max_tokens,
-                    custom_llm_provider=None,  # let litellm resolve from model name
-                    metadata={
-                        **metadata_base,
-                        "advisor_sub_call": True,
-                        "parent_request_id": parent_request_id,
-                    },
-                    api_key=advisor_api_key,
-                    api_base=advisor_api_base,
+                advisor_response: AnthropicMessagesResponse = (
+                    await advisor_router.aanthropic_messages(
+                        model=advisor_model,
+                        messages=advisor_messages,
+                        tools=None,
+                        stream=False,
+                        max_tokens=max_tokens,
+                        metadata=advisor_metadata,
+                    )
+                    if advisor_router is not None
+                    else await _call_messages_handler(
+                        model=advisor_model,
+                        messages=advisor_messages,
+                        tools=None,
+                        stream=False,
+                        max_tokens=max_tokens,
+                        custom_llm_provider=None,
+                        metadata=advisor_metadata,
+                        api_key=advisor_api_key,
+                        api_base=advisor_api_base,
+                    )
                 )
             except Exception as advisor_sub_call_exception:
                 mark_advisor_orchestration_failure(advisor_sub_call_exception)
@@ -284,6 +302,11 @@ def _build_advisor_context(
 
     tool_use blocks are excluded because Anthropic requires tool_use to be
     immediately followed by tool_result — not the advisor question.
+
+    In-sequence system rows (e.g. Claude Code SessionStart hook output) are
+    excluded: they are executor-directed, and a trailing one becomes invalid
+    once the question turn is appended after it (a system row must precede an
+    assistant message or end the array).
     """
     question: Final = (advisor_use_block.get("input") or {}).get("question") or (
         "Please provide guidance on the current task."
@@ -295,7 +318,7 @@ def _build_advisor_context(
         for block in raw_content
         if isinstance(block, dict) and block.get("type") == "text"
     ]
-    result: Final = list(messages)
+    result: Final = [m for m in messages if m.get("role") != "system"]
     if executor_text_blocks:
         result.append({"role": "assistant", "content": executor_text_blocks})
     result.append({"role": "user", "content": question})
@@ -357,6 +380,24 @@ def _inject_max_uses_error(
     ]
 
 
+def _resolve_advisor_router(advisor_model: str) -> "Router | None":
+    """Return the proxy router when it serves ``advisor_model`` directly or via a wildcard.
+
+    Returns ``None`` for SDK callers (no proxy router) and for advisor models the router
+    doesn't know about, so those keep resolving through ``litellm.anthropic_messages()``
+    provider inference.
+    """
+    try:
+        from litellm.proxy.proxy_server import llm_router
+    except (ImportError, ModuleNotFoundError):
+        return None
+    if llm_router is None:
+        return None
+    if llm_router.is_recognized_model(advisor_model) or llm_router.pattern_router.route(advisor_model):
+        return llm_router
+    return None
+
+
 async def _call_messages_handler(
     model: str,
     messages: list[dict],

litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py

@@ -3,7 +3,7 @@
 import json
 import traceback
 from collections import deque
-from collections.abc import AsyncIterator
+from collections.abc import AsyncIterator, Mapping
 from typing import Any, Final
 
 from litellm import verbose_logger
@@ -68,6 +68,19 @@ class AnthropicResponsesStreamWrapper:
         self._current_block_index += 1
         return self._current_block_index
 
+    def _open_block(self, item_id: str | None, content_block: Mapping[str, Any]) -> int:
+        block_idx = self._next_block_index()
+        if item_id:
+            self._item_id_to_block_index[item_id] = block_idx
+        self._chunk_queue.append(
+            {
+                "type": "content_block_start",
+                "index": block_idx,
+                "content_block": content_block,
+            }
+        )
+        return block_idx
+
     def _process_event(self, event: Any) -> None:
         """Convert one Responses API event into zero or more Anthropic chunks queued for emission."""
         event_type = getattr(event, "type", None)
@@ -93,47 +106,22 @@ class AnthropicResponsesStreamWrapper:
             item_id = getattr(item, "id", None) or (item.get("id") if isinstance(item, dict) else None)
 
             if item_type == "message":
-                block_idx = self._next_block_index()
-                if item_id:
-                    self._item_id_to_block_index[item_id] = block_idx
-                self._chunk_queue.append(
-                    {
-                        "type": "content_block_start",
-                        "index": block_idx,
-                        "content_block": {"type": "text", "text": ""},
-                    }
-                )
+                self._open_block(item_id, {"type": "text", "text": ""})
             elif item_type == "function_call":
                 call_id: Final = (
                     getattr(item, "call_id", None) or (item.get("call_id") if isinstance(item, dict) else None) or ""
                 )
                 name = getattr(item, "name", None) or (item.get("name") if isinstance(item, dict) else None) or ""
-                block_idx = self._next_block_index()
                 if item_id:
-                    self._item_id_to_block_index[item_id] = block_idx
                     self._pending_tool_ids[item_id] = call_id
-                self._chunk_queue.append(
+                self._open_block(
+                    item_id,
                     {
-                        "type": "content_block_start",
-                        "index": block_idx,
-                        "content_block": {
-                            "type": "tool_use",
-                            "id": call_id,
-                            "name": name,
-                            "input": {},
-                        },
-                    }
-                )
-            elif item_type == "reasoning":
-                block_idx = self._next_block_index()
-                if item_id:
-                    self._item_id_to_block_index[item_id] = block_idx
-                self._chunk_queue.append(
-                    {
-                        "type": "content_block_start",
-                        "index": block_idx,
-                        "content_block": {"type": "thinking", "thinking": ""},
-                    }
+                        "type": "tool_use",
+                        "id": call_id,
+                        "name": name,
+                        "input": {},
+                    },
                 )
             return
 
@@ -146,16 +134,7 @@ class AnthropicResponsesStreamWrapper:
                 # Some providers (e.g. LMStudio) skip response.output_item.added,
                 # so no text block is open yet; synthesize content_block_start
                 # instead of emitting a delta with index -1
-                block_idx = self._next_block_index()
-                if item_id:
-                    self._item_id_to_block_index[item_id] = block_idx
-                self._chunk_queue.append(
-                    {
-                        "type": "content_block_start",
-                        "index": block_idx,
-                        "content_block": {"type": "text", "text": ""},
-                    }
-                )
+                block_idx = self._open_block(item_id, {"type": "text", "text": ""})
             self._chunk_queue.append(
                 {
                     "type": "content_block_delta",
@@ -169,11 +148,11 @@ class AnthropicResponsesStreamWrapper:
         if event_type == "response.reasoning_summary_text.delta":
             item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None)
             delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "")
-            block_idx = (
-                self._item_id_to_block_index.get(item_id, self._current_block_index)
-                if item_id
-                else self._current_block_index
-            )
+            block_idx = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index
+            if block_idx < 0:
+                if not delta:
+                    return
+                block_idx = self._open_block(item_id, {"type": "thinking", "thinking": ""})
             self._chunk_queue.append(
                 {
                     "type": "content_block_delta",
@@ -207,11 +186,9 @@ class AnthropicResponsesStreamWrapper:
             item_id = (
                 getattr(item, "id", None) or (item.get("id") if isinstance(item, dict) else None) if item else None
             )
-            block_idx = (
-                self._item_id_to_block_index.get(item_id, self._current_block_index)
-                if item_id
-                else self._current_block_index
-            )
+            block_idx = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index
+            if block_idx < 0:
+                return
             self._chunk_queue.append(
                 {
                     "type": "content_block_stop",

litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py

@@ -266,7 +266,13 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
             if (isinstance(tool_type, str) and tool_type.startswith("web_search")) or tool_name == "web_search":
                 result.append({"type": "web_search_preview"})
                 continue
-            func_tool: dict[str, Any] = {"type": "function", "name": tool_name}
+            # Responses turns strict mode on when `strict` is omitted, silently rewriting
+            # `required` to every property. Anthropic tools are non-strict unless asked.
+            func_tool: dict[str, Any] = {
+                "type": "function",
+                "name": tool_name,
+                "strict": bool(tool_dict.get("strict")),
+            }
             if "description" in tool_dict:
                 func_tool["description"] = tool_dict["description"]
             if "input_schema" in tool_dict:

litellm/llms/azure/chat/gpt_transformation.py

@@ -112,6 +112,16 @@ class AzureOpenAIConfig(BaseConfig):
             "store",
         ]
 
+    @classmethod
+    def requires_max_completion_tokens(cls, model: str) -> bool:
+        """Whether Azure rejects the legacy ``max_tokens`` key for this deployment.
+
+        Deliberately wider than ``AzureOpenAIGPT5Config.is_model_gpt_5_model``: the whole gpt-5
+        name family needs the rename, including the ``gpt-5-chat*`` models that are excluded from
+        the reasoning path by https://github.com/BerriAI/litellm/issues/13781.
+        """
+        return "gpt-5" in model or "gpt5_series" in model
+
     def _is_response_format_supported_model(self, model: str) -> bool:
         """
         Determines if the model supports response_format.
@@ -160,6 +170,7 @@ class AzureOpenAIConfig(BaseConfig):
         api_version: str = "",
     ) -> dict:
         supported_openai_params: Final = self.get_supported_openai_params(model)
+        renames_max_tokens: Final = self.requires_max_completion_tokens(model)
         api_version_times: Final = api_version.split("-")
 
         if len(api_version_times) >= 3:
@@ -172,7 +183,9 @@ class AzureOpenAIConfig(BaseConfig):
             api_version_day = None
 
         for param, value in non_default_params.items():
-            if param == "tool_choice":
+            if param == "max_tokens" and renames_max_tokens:
+                optional_params.setdefault("max_completion_tokens", value)
+            elif param == "tool_choice":
                 """
                 This parameter requires API version 2023-12-01-preview or later
 

litellm/llms/azure_ai/agents/handler.py

@@ -22,10 +22,11 @@ import asyncio
 import json
 import time
 import uuid
-from collections.abc import AsyncIterator, Callable
-from typing import TYPE_CHECKING, Any, Final
+from collections.abc import AsyncIterator, Awaitable, Mapping
+from typing import TYPE_CHECKING, Any, Final, Protocol, TypeAlias, TypedDict
 
 import httpx
+from typing_extensions import ReadOnly
 
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.url_utils import encode_url_path_segment
@@ -33,7 +34,11 @@ from litellm.llms.azure_ai.agents.transformation import (
     AzureAIAgentsConfig,
     AzureAIAgentsError,
 )
-from litellm.types.utils import ModelResponse
+from litellm.types.llms.openai import (
+    ChatCompletionAnnotation,
+    ChatCompletionAnnotationURLCitation,
+)
+from litellm.types.utils import ModelResponse, ModelResponseStream
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@@ -46,6 +51,69 @@ else:
     AsyncHTTPHandler = Any
 
 
+class _AzureRawAnnotation(TypedDict, total=False):
+    type: ReadOnly[str]
+    text: ReadOnly[str]
+    start_index: ReadOnly[int]
+    end_index: ReadOnly[int]
+    url_citation: ReadOnly[ChatCompletionAnnotationURLCitation]
+
+
+_TransformedAnnotation: TypeAlias = ChatCompletionAnnotation | _AzureRawAnnotation
+
+
+class _AzureText(TypedDict, total=False):
+    value: ReadOnly[str]
+    annotations: ReadOnly[list[_AzureRawAnnotation]]
+
+
+class _AzureContentItem(TypedDict, total=False):
+    type: ReadOnly[str]
+    text: ReadOnly[_AzureText]
+
+
+class _AzureMessage(TypedDict, total=False):
+    role: ReadOnly[str]
+    content: ReadOnly[list[_AzureContentItem]]
+
+
+class _AzureMessagesData(TypedDict, total=False):
+    data: ReadOnly[list[_AzureMessage]]
+
+
+class _CreatedObject(TypedDict):
+    id: ReadOnly[str]
+
+
+class _RunError(TypedDict, total=False):
+    message: ReadOnly[str]
+
+
+class _RunStatus(TypedDict, total=False):
+    status: ReadOnly[str]
+    last_error: ReadOnly[_RunError]
+
+
+class _SSEDelta(TypedDict, total=False):
+    content: ReadOnly[list[_AzureContentItem]]
+
+
+class _SSEEventData(TypedDict, total=False):
+    id: ReadOnly[str]
+    content: ReadOnly[list[_AzureContentItem]]
+    delta: ReadOnly[_SSEDelta]
+
+
+class _SyncAgentRequest(Protocol):
+    def __call__(self, method: str, url: str, json_data: Mapping[str, object] | None = None) -> httpx.Response: ...
+
+
+class _AsyncAgentRequest(Protocol):
+    def __call__(
+        self, method: str, url: str, json_data: Mapping[str, object] | None = None
+    ) -> Awaitable[httpx.Response]: ...
+
+
 class AzureAIAgentsHandler:
     """
     Handler for Azure AI Agent Service.
@@ -89,7 +157,9 @@ class AzureAIAgentsHandler:
     # -------------------------------------------------------------------------
     # Response Helpers
     # -------------------------------------------------------------------------
-    def _extract_content_from_messages(self, messages_data: dict) -> tuple[str, list[dict[str, Any]] | None]:
+    def _extract_content_from_messages(
+        self, messages_data: _AzureMessagesData
+    ) -> tuple[str, list[_TransformedAnnotation] | None]:
         """Extract assistant content and annotations from the messages response.
 
         Returns (content, annotations) where annotations is a list of
@@ -108,8 +178,8 @@ class AzureAIAgentsHandler:
 
     def _transform_annotations(
         self,
-        raw_annotations: list[dict[str, Any]] | None,
-    ) -> list[dict[str, Any]] | None:
+        raw_annotations: list[_AzureRawAnnotation] | None,
+    ) -> list[_TransformedAnnotation] | None:
         """Transform Azure AI Foundry annotations to OpenAI-compatible format.
 
         Azure AI returns annotations like:
@@ -123,11 +193,11 @@ class AzureAIAgentsHandler:
         if not raw_annotations:
             return None
 
-        result: Final[list[dict[str, Any]]] = []
+        result: Final[list[_TransformedAnnotation]] = []
         for ann in raw_annotations:
             ann_type = ann.get("type")
             if ann_type == "url_citation":
-                url_citation = dict(ann.get("url_citation", {}))
+                url_citation: ChatCompletionAnnotationURLCitation = {**ann.get("url_citation", {})}
                 # Azure puts start/end_index at annotation level; OpenAI
                 # expects them inside url_citation
                 if "start_index" in ann and "start_index" not in url_citation:
@@ -147,8 +217,8 @@ class AzureAIAgentsHandler:
         content: str,
         model_response: ModelResponse,
         thread_id: str,
-        messages: list[dict[str, Any]],
-        annotations: list[dict[str, Any]] | None = None,
+        messages: list[dict[str, object]],
+        annotations: list[_TransformedAnnotation] | None = None,
     ) -> ModelResponse:
         """Build the ModelResponse from agent output."""
         from litellm.types.utils import Choices, Message, Usage
@@ -201,7 +271,7 @@ class AzureAIAgentsHandler:
         api_key: str,
         optional_params: dict,
         headers: dict | None,
-    ) -> tuple:
+    ) -> tuple[dict[str, str], str, str, str | None, str]:
         """Prepare common parameters for completion.
 
         Azure Foundry Agents API uses Bearer token authentication:
@@ -241,7 +311,7 @@ class AzureAIAgentsHandler:
     def completion(
         self,
         model: str,
-        messages: list[dict[str, Any]],
+        messages: list[dict[str, object]],
         api_base: str,
         api_key: str,
         model_response: ModelResponse,
@@ -266,7 +336,7 @@ class AzureAIAgentsHandler:
             api_base,
         ) = self._prepare_completion_params(model, api_base, api_key, optional_params, headers)
 
-        def make_request(method: str, url: str, json_data: dict | None = None) -> httpx.Response:
+        def make_request(method: str, url: str, json_data: Mapping[str, object] | None = None) -> httpx.Response:
             if method == "GET":
                 return client.get(url=url, headers=headers)
             return client.post(
@@ -290,14 +360,14 @@ class AzureAIAgentsHandler:
 
     def _execute_agent_flow_sync(
         self,
-        make_request: Callable,
+        make_request: _SyncAgentRequest,
         api_base: str,
         api_version: str,
         agent_id: str,
         thread_id: str | None,
-        messages: list[dict[str, Any]],
+        messages: list[dict[str, object]],
         optional_params: dict,
-    ) -> tuple[str, str, list[dict[str, Any]] | None]:
+    ) -> tuple[str, str, list[_TransformedAnnotation] | None]:
         """Execute the agent flow synchronously. Returns (thread_id, content, annotations)."""
 
         # Step 1: Create thread if not provided
@@ -305,7 +375,8 @@ class AzureAIAgentsHandler:
             verbose_logger.debug("Creating thread at: %s", self._build_thread_url(api_base, api_version))
             response = make_request("POST", self._build_thread_url(api_base, api_version), {})
             self._check_response(response, [200, 201], "Failed to create thread")
-            thread_id = response.json()["id"]
+            thread_data: Final[_CreatedObject] = response.json()
+            thread_id = thread_data["id"]
             verbose_logger.debug("Created thread: %s", thread_id)
 
         # At this point thread_id is guaranteed to be a string
@@ -325,7 +396,8 @@ class AzureAIAgentsHandler:
 
         response = make_request("POST", self._build_runs_url(api_base, thread_id, api_version), run_payload)
         self._check_response(response, [200, 201], "Failed to create run")
-        run_id: Final = response.json()["id"]
+        run_data: Final[_CreatedObject] = response.json()
+        run_id: Final = run_data["id"]
         verbose_logger.debug("Created run: %s", run_id)
 
         # Step 4: Poll for completion
@@ -334,13 +406,15 @@ class AzureAIAgentsHandler:
             response = make_request("GET", status_url)
             self._check_response(response, [200], "Failed to get run status")
 
-            status = response.json().get("status")
+            status_data: _RunStatus = response.json()
+            status = status_data.get("status")
             verbose_logger.debug("Run status: %s", status)
 
             if status == "completed":
                 break
             elif status in ["failed", "cancelled", "expired"]:
-                error_msg = response.json().get("last_error", {}).get("message", "Unknown error")
+                error_data: _RunStatus = response.json()
+                error_msg = error_data.get("last_error", {}).get("message", "Unknown error")
                 raise AzureAIAgentsError(status_code=500, message=f"Run {status}: {error_msg}")
 
             time.sleep(self.config.POLL_INTERVAL_SECONDS)
@@ -351,7 +425,8 @@ class AzureAIAgentsHandler:
         response = make_request("GET", self._build_list_messages_url(api_base, thread_id, api_version))
         self._check_response(response, [200], "Failed to get messages")
 
-        content, annotations = self._extract_content_from_messages(response.json())
+        messages_data: Final[_AzureMessagesData] = response.json()
+        content, annotations = self._extract_content_from_messages(messages_data)
         return thread_id, content, annotations
 
     # -------------------------------------------------------------------------
@@ -360,7 +435,7 @@ class AzureAIAgentsHandler:
     async def acompletion(
         self,
         model: str,
-        messages: list[dict[str, Any]],
+        messages: list[dict[str, object]],
         api_base: str,
         api_key: str,
         model_response: ModelResponse,
@@ -389,7 +464,7 @@ class AzureAIAgentsHandler:
             api_base,
         ) = self._prepare_completion_params(model, api_base, api_key, optional_params, headers)
 
-        async def make_request(method: str, url: str, json_data: dict | None = None) -> httpx.Response:
+        async def make_request(method: str, url: str, json_data: Mapping[str, object] | None = None) -> httpx.Response:
             if method == "GET":
                 return await client.get(url=url, headers=headers)
             return await client.post(
@@ -413,14 +488,14 @@ class AzureAIAgentsHandler:
 
     async def _execute_agent_flow_async(
         self,
-        make_request: Callable,
+        make_request: _AsyncAgentRequest,
         api_base: str,
         api_version: str,
         agent_id: str,
         thread_id: str | None,
-        messages: list[dict[str, Any]],
+        messages: list[dict[str, object]],
         optional_params: dict,
-    ) -> tuple[str, str, list[dict[str, Any]] | None]:
+    ) -> tuple[str, str, list[_TransformedAnnotation] | None]:
         """Execute the agent flow asynchronously. Returns (thread_id, content, annotations)."""
 
         # Step 1: Create thread if not provided
@@ -428,7 +503,8 @@ class AzureAIAgentsHandler:
             verbose_logger.debug("Creating thread at: %s", self._build_thread_url(api_base, api_version))
             response = await make_request("POST", self._build_thread_url(api_base, api_version), {})
             self._check_response(response, [200, 201], "Failed to create thread")
-            thread_id = response.json()["id"]
+            thread_data: Final[_CreatedObject] = response.json()
+            thread_id = thread_data["id"]
             verbose_logger.debug("Created thread: %s", thread_id)
 
         # At this point thread_id is guaranteed to be a string
@@ -448,7 +524,8 @@ class AzureAIAgentsHandler:
 
         response = await make_request("POST", self._build_runs_url(api_base, thread_id, api_version), run_payload)
         self._check_response(response, [200, 201], "Failed to create run")
-        run_id: Final = response.json()["id"]
+        run_data: Final[_CreatedObject] = response.json()
+        run_id: Final = run_data["id"]
         verbose_logger.debug("Created run: %s", run_id)
 
         # Step 4: Poll for completion
@@ -457,13 +534,15 @@ class AzureAIAgentsHandler:
             response = await make_request("GET", status_url)
             self._check_response(response, [200], "Failed to get run status")
 
-            status = response.json().get("status")
+            status_data: _RunStatus = response.json()
+            status = status_data.get("status")
             verbose_logger.debug("Run status: %s", status)
 
             if status == "completed":
                 break
             elif status in ["failed", "cancelled", "expired"]:
-                error_msg = response.json().get("last_error", {}).get("message", "Unknown error")
+                error_data: _RunStatus = response.json()
+                error_msg = error_data.get("last_error", {}).get("message", "Unknown error")
                 raise AzureAIAgentsError(status_code=500, message=f"Run {status}: {error_msg}")
 
             await asyncio.sleep(self.config.POLL_INTERVAL_SECONDS)
@@ -474,7 +553,8 @@ class AzureAIAgentsHandler:
         response = await make_request("GET", self._build_list_messages_url(api_base, thread_id, api_version))
         self._check_response(response, [200], "Failed to get messages")
 
-        content, annotations = self._extract_content_from_messages(response.json())
+        messages_data: Final[_AzureMessagesData] = response.json()
+        content, annotations = self._extract_content_from_messages(messages_data)
         return thread_id, content, annotations
 
     # -------------------------------------------------------------------------
@@ -483,7 +563,7 @@ class AzureAIAgentsHandler:
     async def acompletion_stream(
         self,
         model: str,
-        messages: list[dict[str, Any]],
+        messages: list[dict[str, object]],
         api_base: str,
         api_key: str,
         logging_obj: LiteLLMLoggingObj,
@@ -491,7 +571,7 @@ class AzureAIAgentsHandler:
         litellm_params: dict,
         timeout: float,
         headers: dict | None = None,
-    ) -> AsyncIterator:
+    ) -> AsyncIterator[ModelResponseStream]:
         """Execute async streaming completion using Azure Agent Service with native SSE."""
         import litellm
         from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
@@ -505,12 +585,12 @@ class AzureAIAgentsHandler:
         ) = self._prepare_completion_params(model, api_base, api_key, optional_params, headers)
 
         # Build payload for create-thread-and-run with streaming
-        thread_messages: Final = []
+        thread_messages: Final[list[dict[str, object]]] = []
         for msg in messages:
             if msg.get("role") in ["user", "system"]:
                 thread_messages.append({"role": "user", "content": msg.get("content", "")})
 
-        payload: Final[dict[str, Any]] = {
+        payload: Final[dict[str, object]] = {
             "assistant_id": agent_id,
             "stream": True,
         }
@@ -552,14 +632,14 @@ class AzureAIAgentsHandler:
         self,
         response: httpx.Response,
         model: str,
-    ) -> AsyncIterator:
+    ) -> AsyncIterator[ModelResponseStream]:
         """Process SSE stream and yield OpenAI-compatible streaming chunks."""
         from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
 
         response_id: Final = f"chatcmpl-{uuid.uuid4().hex[:8]}"
         created: Final = int(time.time())
         thread_id = None
-        collected_annotations: list[dict[str, Any]] | None = None
+        collected_annotations: list[_TransformedAnnotation] | None = None
 
         current_event = None
 
@@ -597,7 +677,7 @@ class AzureAIAgentsHandler:
                     return
 
                 try:
-                    data = json.loads(data_str)
+                    data: _SSEEventData = json.loads(data_str)
                 except json.JSONDecodeError:
                     continue
 

litellm/llms/azure_ai/chat/transformation.py

@@ -1,3 +1,4 @@
+import copy
 import enum
 import re
 from typing import Any, Final, cast
@@ -11,6 +12,7 @@ from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     _audio_or_image_in_message_content,
     convert_content_list_to_str,
+    filter_value_from_dict,
 )
 from litellm.llms.azure.common_utils import BaseAzureLLM
 from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
@@ -28,6 +30,9 @@ class AzureFoundryErrorStrings(str, enum.Enum):
     SET_EXTRA_PARAMETERS_TO_PASS_THROUGH = "Set extra-parameters to 'pass-through'"
 
 
+NON_OPENAI_SPEC_MESSAGE_FIELDS: Final = ("thinking_blocks", "provider_specific_fields", "cache_control")
+
+
 class AzureAIStudioConfig(OpenAIConfig):
     def get_supported_openai_params(self, model: str) -> list:
         model_supports_tool_choice = True  # azure ai supports this by default
@@ -167,10 +172,23 @@ class AzureAIStudioConfig(OpenAIConfig):
     ) -> list:
         """
         - Azure AI Studio doesn't support content as a list. This handles:
-            1. Transforms list content to a string.
-            2. If message contains an image or audio, send as is (user-intended)
+            1. Strips message fields that are not part of the OpenAI chat-completions
+               schema (thinking_blocks, provider_specific_fields, cache_control).
+               Azure AI Foundry backends set additionalProperties=false and reject
+               these with "Extra inputs are not permitted", which breaks multi-turn
+               Anthropic-format clients that echo thinking blocks back as history.
+            2. Transforms list content to a string.
+            3. If message contains an image or audio, send as is (user-intended)
+
+        Operates on a deep copy so the caller's messages keep their thinking blocks
+        and provider metadata, which a fallback to another provider still needs.
         """
-        for message in messages:
+        stripped_messages: Final = copy.deepcopy(messages)
+        for message in stripped_messages:
+            message_dict = cast(dict, message)  # cast-ok: TypedDict is a runtime dict stripped on our copy
+            for field in NON_OPENAI_SPEC_MESSAGE_FIELDS:
+                filter_value_from_dict(message_dict, field)
+
             # Do nothing if the message contains an image or audio
             if _audio_or_image_in_message_content(message):
                 continue
@@ -178,7 +196,7 @@ class AzureAIStudioConfig(OpenAIConfig):
             texts = convert_content_list_to_str(message=message)
             if texts:
                 message["content"] = texts
-        return messages
+        return stripped_messages
 
     def _is_azure_openai_model(self, model: str, api_base: str | None) -> bool:
         try:

litellm/llms/azure_ai/ocr/document_intelligence/transformation.py

@@ -11,6 +11,7 @@ The operation location must be polled until the analysis completes.
 import asyncio
 import re
 import time
+from collections.abc import Mapping
 from typing import Any, Final
 from urllib.parse import quote
 
@@ -23,15 +24,19 @@ from litellm.constants import (
     AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI,
     AZURE_OPERATION_POLLING_TIMEOUT,
 )
+from litellm.exceptions import UnsupportedParamsError
 from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin, encode_url_path_segment
 from litellm.llms.base_llm.ocr.transformation import (
+    OCR_REQUEST_FORMAT_PARAM,
     BaseOCRConfig,
     DocumentType,
     OCRPage,
     OCRPageDimensions,
     OCRRequestData,
+    OCRRequestFormat,
     OCRResponse,
     OCRUsageInfo,
+    parse_ocr_request_format,
 )
 from litellm.secret_managers.main import get_secret_str
 
@@ -97,8 +102,12 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
         comma-separated string. Other Mistral-specific params (e.g.
         `include_image_base64`) are not supported by Azure DI and are
         ignored during transformation.
+
+        `req_format` selects the response shape: "litellm" (default) returns
+        the normalized OCR schema, "native" returns Azure DI's own analyze
+        operation payload as-is.
         """
-        return ["pages", "features"]
+        return ["pages", "features", OCR_REQUEST_FORMAT_PARAM]
 
     def map_ocr_params(
         self,
@@ -117,14 +126,27 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
         """
         pages: Final = non_default_params.get("pages")
         features: Final = non_default_params.get("features")
+        request_format: Final = non_default_params.get(OCR_REQUEST_FORMAT_PARAM)
         normalized_pages: Final = self._normalize_pages_param(pages) if pages is not None else ""
         normalized_features: Final = self._normalize_features_param(features) if features is not None else ""
         return {
             **optional_params,
             **({"pages": normalized_pages} if normalized_pages else {}),
             **({"features": normalized_features} if normalized_features else {}),
+            **(
+                {OCR_REQUEST_FORMAT_PARAM: self._parse_request_format(request_format, model)}
+                if request_format is not None
+                else {}
+            ),
         }
 
+    @staticmethod
+    def _parse_request_format(request_format: object, model: str) -> OCRRequestFormat:
+        try:
+            return parse_ocr_request_format(request_format)
+        except ValueError as e:
+            raise UnsupportedParamsError(message=f"{e}", model=model, llm_provider="azure_ai") from e
+
     @staticmethod
     def _normalize_pages_param(pages: Any) -> str:
         """
@@ -594,14 +616,33 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
         poll_headers = {"Ocp-Apim-Subscription-Key": raw_response.request.headers.get("Ocp-Apim-Subscription-Key", "")}
         return operation_url, poll_headers
 
-    def _transform_completed_response(self, model: str, raw_response: httpx.Response) -> OCRResponse:
+    @staticmethod
+    def _get_request_format(optional_params: object) -> OCRRequestFormat:
+        if not isinstance(optional_params, dict):
+            return "litellm"
+        request_format: Final = optional_params.get(OCR_REQUEST_FORMAT_PARAM)
+        if request_format is None:
+            return "litellm"
+        return parse_ocr_request_format(request_format)
+
+    def _transform_completed_response(
+        self,
+        model: str,
+        raw_response: httpx.Response,
+        request_format: OCRRequestFormat,
+    ) -> OCRResponse:
         """
         Transform a completed Azure Document Intelligence analyze operation
         into the Mistral OCR response shape, preserving Azure-native
         `analyzeResult` fields (`content`, `tables`, `keyValuePairs`) as
         top-level response fields.
+
+        When `request_format` is "native", the untouched Azure operation
+        payload is attached to the response's hidden params so the proxy can
+        return it verbatim while cost tracking still reads `usage_info`.
         """
-        operation: Final = AzureDocumentIntelligenceOperation.model_validate(raw_response.json())
+        raw_operation: Final[Mapping[str, object]] = raw_response.json()
+        operation: Final = AzureDocumentIntelligenceOperation.model_validate(raw_operation)
 
         verbose_logger.debug("Azure Document Intelligence response status: %s", operation.status)
 
@@ -614,7 +655,7 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
         mistral_pages: Final = [self._transform_azure_page(azure_page) for azure_page in analyze_result.pages]
         usage_info: Final = OCRUsageInfo(pages_processed=len(mistral_pages), doc_size_bytes=None)
 
-        return OCRResponse(
+        response: Final = OCRResponse(
             pages=mistral_pages,
             model=model,
             usage_info=usage_info,
@@ -624,6 +665,11 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
             keyValuePairs=analyze_result.keyValuePairs,
         )
 
+        if request_format == "native":
+            response.set_provider_native_response(raw_operation)
+
+        return response
+
     def transform_ocr_response(
         self,
         model: str,
@@ -681,8 +727,12 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
         Returns:
             OCRResponse in Mistral format
         """
+        request_format: Final = self._get_request_format(kwargs.get("optional_params"))
+
         if raw_response.status_code != 202:
-            return self._transform_completed_response(model=model, raw_response=raw_response)
+            return self._transform_completed_response(
+                model=model, raw_response=raw_response, request_format=request_format
+            )
 
         verbose_logger.debug("Azure DI returned 202 Accepted, polling operation...")
         operation_url, poll_headers = self._get_polling_target(raw_response)
@@ -691,7 +741,9 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
             headers=poll_headers,
             timeout_secs=AZURE_OPERATION_POLLING_TIMEOUT,
         )
-        return self._transform_completed_response(model=model, raw_response=completed_response)
+        return self._transform_completed_response(
+            model=model, raw_response=completed_response, request_format=request_format
+        )
 
     async def async_transform_ocr_response(
         self,
@@ -714,8 +766,12 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
         Returns:
             OCRResponse in Mistral format
         """
+        request_format: Final = self._get_request_format(kwargs.get("optional_params"))
+
         if raw_response.status_code != 202:
-            return self._transform_completed_response(model=model, raw_response=raw_response)
+            return self._transform_completed_response(
+                model=model, raw_response=raw_response, request_format=request_format
+            )
 
         verbose_logger.debug("Azure DI returned 202 Accepted, polling operation (async)...")
         operation_url, poll_headers = self._get_polling_target(raw_response)
@@ -724,4 +780,6 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
             headers=poll_headers,
             timeout_secs=AZURE_OPERATION_POLLING_TIMEOUT,
         )
-        return self._transform_completed_response(model=model, raw_response=completed_response)
+        return self._transform_completed_response(
+            model=model, raw_response=completed_response, request_format=request_format
+        )

litellm/llms/base_llm/ocr/transformation.py

@@ -2,7 +2,8 @@
 Base OCR transformation configuration.
 """
 
-from typing import TYPE_CHECKING, Any
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Any, Final, Literal
 
 import httpx
 from pydantic import PrivateAttr
@@ -21,6 +22,26 @@ else:
 # File-type inputs are preprocessed to this format in litellm/ocr/main.py.
 DocumentType = dict[str, str]
 
+OCRRequestFormat = Literal["litellm", "native"]
+
+OCR_REQUEST_FORMATS: Final[tuple[OCRRequestFormat, ...]] = ("litellm", "native")
+
+OCR_REQUEST_FORMAT_PARAM: Final = "req_format"
+
+OCR_REQUEST_FORMAT_HEADER: Final = "x-req-format"
+
+PROVIDER_NATIVE_RESPONSE_KEY: Final = "provider_native_response"
+
+
+def parse_ocr_request_format(value: object) -> OCRRequestFormat:
+    if value == "litellm":
+        return "litellm"
+    if value == "native":
+        return "native"
+    raise ValueError(
+        f"Invalid `{OCR_REQUEST_FORMAT_PARAM}`: {value!r}. Expected one of {', '.join(OCR_REQUEST_FORMATS)}."
+    )
+
 
 class OCRPageDimensions(LiteLLMPydanticObjectBase):
     """Page dimensions from OCR response."""
@@ -80,6 +101,15 @@ class OCRResponse(LiteLLMPydanticObjectBase):
     # Define private attributes using PrivateAttr
     _hidden_params: dict = PrivateAttr(default_factory=dict)
 
+    def set_provider_native_response(self, native_response: Mapping[str, object]) -> None:
+        """Keep the provider's own response payload alongside the normalized one."""
+        self._hidden_params[PROVIDER_NATIVE_RESPONSE_KEY] = native_response
+
+    def get_provider_native_response(self) -> Mapping[str, object] | None:
+        """The provider's own response payload, when `req_format=native` was requested."""
+        native_response: Final = self._hidden_params.get(PROVIDER_NATIVE_RESPONSE_KEY)
+        return native_response if isinstance(native_response, dict) else None
+
 
 class OCRRequestData(LiteLLMPydanticObjectBase):
     """OCR request data structure."""

litellm/llms/base_llm/vector_store/transformation.py

@@ -1,5 +1,6 @@
 from abc import abstractmethod
-from typing import TYPE_CHECKING, Any
+from collections.abc import Mapping, Sequence
+from typing import TYPE_CHECKING, Any, NoReturn
 
 import httpx
 
@@ -154,3 +155,75 @@ class BaseVectorStoreConfig:
         response: VectorStoreSearchResponse,
     ) -> tuple[float, float]:
         return 0.0, 0.0
+
+
+class BaseDirectVectorStoreConfig(BaseVectorStoreConfig):
+    """
+    Base config for vector store providers whose datastore has no HTTP API
+    (e.g. Valkey over RESP). Instead of transforming to an httpx request, the
+    config executes the search itself via (a)execute_search_vector_store_request.
+    """
+
+    @abstractmethod
+    def execute_search_vector_store_request(
+        self,
+        vector_store_id: str,
+        query: str | Sequence[str],
+        vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams,
+        litellm_logging_obj: LiteLLMLoggingObj,
+        litellm_params: Mapping[str, object],
+        timeout: float | httpx.Timeout | None = None,
+    ) -> VectorStoreSearchResponse:
+        pass
+
+    @abstractmethod
+    async def aexecute_search_vector_store_request(
+        self,
+        vector_store_id: str,
+        query: str | Sequence[str],
+        vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams,
+        litellm_logging_obj: LiteLLMLoggingObj,
+        litellm_params: Mapping[str, object],
+        timeout: float | httpx.Timeout | None = None,
+    ) -> VectorStoreSearchResponse:
+        pass
+
+    def transform_search_vector_store_request(
+        self,
+        vector_store_id: str,
+        query: str | Sequence[str],
+        vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams,
+        api_base: str,
+        litellm_logging_obj: LiteLLMLoggingObj,
+        litellm_params: Mapping[str, object],
+        extra_body: Mapping[str, object] | None = None,
+    ) -> NoReturn:
+        raise NotImplementedError("Direct vector store providers execute the search themselves; no HTTP request shape")
+
+    def transform_search_vector_store_response(
+        self, response: httpx.Response, litellm_logging_obj: LiteLLMLoggingObj
+    ) -> NoReturn:
+        raise NotImplementedError("Direct vector store providers execute the search themselves; no HTTP response shape")
+
+    def transform_create_vector_store_request(
+        self,
+        vector_store_create_optional_params: VectorStoreCreateOptionalRequestParams,
+        api_base: str,
+    ) -> NoReturn:
+        raise NotImplementedError
+
+    def transform_create_vector_store_response(self, response: httpx.Response) -> NoReturn:
+        raise NotImplementedError
+
+    def get_complete_url(
+        self,
+        api_base: str | None,
+        litellm_params: Mapping[str, object],
+    ) -> str:
+        return api_base or ""
+
+    def get_auth_credentials(self, litellm_params: Mapping[str, object]) -> BaseVectorStoreAuthCredentials:
+        return BaseVectorStoreAuthCredentials()
+
+    def get_vector_store_endpoints_by_type(self) -> VectorStoreIndexEndpoints:
+        return VectorStoreIndexEndpoints(read=[], write=[])  # mutable-ok: the TypedDict declares list fields

litellm/llms/bedrock/batches/handler.py

@@ -1,11 +1,14 @@
 from datetime import datetime
-from typing import Any, Final, cast
+from typing import TYPE_CHECKING, Any, Final, cast
 
 from openai.types.batch import BatchRequestCounts
 from openai.types.batch import Metadata as OpenAIBatchMetadata
 
 from litellm.types.utils import LiteLLMBatch
 
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+
 # AWS Bedrock model-invocation-job statuses → OpenAI Batch statuses.
 # Mirrors the mapping used by `BedrockBatchesConfig.transform_create_batch_response`
 # so create / retrieve return consistent statuses.
@@ -22,6 +25,8 @@ _BEDROCK_MIJ_STATUS_TO_OPENAI: Final = {
     "Expired": "expired",
 }
 
+_CANCEL_IDEMPOTENT_STATUSES: Final = frozenset({"cancelling", "cancelled", "completed", "failed", "expired"})
+
 
 def _extract_region_from_bedrock_arn(arn: str) -> str | None:
     """ARN shape: ``arn:aws:bedrock:<region>:<account>:<type>/<id>``"""
@@ -82,6 +87,81 @@ class BedrockBatchesHandler:
     E.g. Twelve Labs Embedding Async Invoke
     """
 
+    @staticmethod
+    def cancel_batch(
+        batch_id: str,
+        aws_region_name: str | None = None,
+        logging_obj: "LiteLLMLoggingObj | None" = None,
+        aws_access_key_id: str | None = None,
+        aws_secret_access_key: str | None = None,
+        aws_session_token: str | None = None,
+        aws_session_name: str | None = None,
+        aws_profile_name: str | None = None,
+        aws_role_name: str | None = None,
+        aws_web_identity_token: str | None = None,
+        aws_sts_endpoint: str | None = None,
+        aws_external_id: str | None = None,
+        **kwargs: object,  # kwargs-ok: litellm.cancel_batch forwards arbitrary user kwargs verbatim
+    ) -> "LiteLLMBatch":
+        try:
+            import boto3
+            from botocore.exceptions import ClientError
+        except ImportError as exc:
+            raise ImportError("Missing boto3/botocore to call bedrock. Run 'pip install boto3'.") from exc
+
+        region: Final = aws_region_name or _extract_region_from_bedrock_arn(batch_id) or "us-east-1"
+
+        from litellm.llms.bedrock.batches.transformation import BedrockBatchesConfig
+
+        creds: Final = BedrockBatchesConfig().get_credentials(
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+            aws_session_token=aws_session_token,
+            aws_region_name=region,
+            aws_session_name=aws_session_name,
+            aws_profile_name=aws_profile_name,
+            aws_role_name=aws_role_name,
+            aws_web_identity_token=aws_web_identity_token,
+            aws_sts_endpoint=aws_sts_endpoint,
+            aws_external_id=aws_external_id,
+        )
+
+        client: Final = boto3.client(
+            "bedrock",
+            region_name=region,
+            aws_access_key_id=creds.access_key,
+            aws_secret_access_key=creds.secret_key,
+            aws_session_token=creds.token,
+        )
+
+        def job_status() -> "LiteLLMBatch":
+            return BedrockBatchesHandler._handle_model_invocation_job_status(
+                batch_id=batch_id,
+                aws_region_name=region,
+                logging_obj=logging_obj,
+                aws_access_key_id=aws_access_key_id,
+                aws_secret_access_key=aws_secret_access_key,
+                aws_session_token=aws_session_token,
+                aws_session_name=aws_session_name,
+                aws_profile_name=aws_profile_name,
+                aws_role_name=aws_role_name,
+                aws_web_identity_token=aws_web_identity_token,
+                aws_sts_endpoint=aws_sts_endpoint,
+                aws_external_id=aws_external_id,
+            )
+
+        try:
+            client.stop_model_invocation_job(jobIdentifier=batch_id)
+        except ClientError as e:
+            if e.response.get("Error", {}).get("Code") not in ("ValidationException", "ConflictException"):
+                raise
+            current_batch: Final = job_status()
+            if current_batch.status not in _CANCEL_IDEMPOTENT_STATUSES:
+                raise
+            return current_batch
+
+        return job_status()
+
     @staticmethod
     def _handle_async_invoke_status(batch_id: str, aws_region_name: str, logging_obj=None, **kwargs) -> "LiteLLMBatch":
         """

litellm/llms/bedrock/chat/converse_transformation.py

@@ -6,6 +6,7 @@ import copy
 import json
 import time
 import types
+from collections.abc import Mapping
 from typing import Final, Literal, cast, overload
 
 import httpx
@@ -39,6 +40,12 @@ from litellm.llms.anthropic.chat.transformation import (
     AnthropicConfig,
 )
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
+from litellm.llms.bedrock.request_metadata import (
+    bedrock_request_metadata_headers,
+    bedrock_request_metadata_is_owned,
+    merge_bedrock_invoke_headers,
+    resolve_bedrock_request_metadata,
+)
 from litellm.types.llms.bedrock import *
 from litellm.types.llms.openai import (
     AllMessageValues,
@@ -1652,6 +1659,13 @@ class AmazonConverseConfig(BaseConfig):
             user_continue_message=litellm_params.pop("user_continue_message", None),
         )
 
+        request_metadata: Final = resolve_bedrock_request_metadata(
+            litellm_params=litellm_params, caller_metadata=_data.get("requestMetadata")
+        )
+        if bedrock_request_metadata_is_owned():
+            _data.pop("requestMetadata", None)
+            if request_metadata is not None:
+                _data["requestMetadata"] = request_metadata
         data: Final[RequestObject] = {"messages": bedrock_messages, **_data}
 
         return data
@@ -1705,6 +1719,13 @@ class AmazonConverseConfig(BaseConfig):
             user_continue_message=litellm_params.pop("user_continue_message", None),
         )
 
+        request_metadata: Final = resolve_bedrock_request_metadata(
+            litellm_params=litellm_params, caller_metadata=_data.get("requestMetadata")
+        )
+        if bedrock_request_metadata_is_owned():
+            _data.pop("requestMetadata", None)
+            if request_metadata is not None:
+                _data["requestMetadata"] = request_metadata
         data: Final[RequestObject] = {"messages": bedrock_messages, **_data}
 
         return data
@@ -1770,7 +1791,43 @@ class AmazonConverseConfig(BaseConfig):
                 thinking_blocks_list.append(_redacted_block)
         return thinking_blocks_list
 
-    def _transform_usage(
+    @staticmethod
+    def is_converse_usage_shape(usage_object: Mapping[str, object]) -> bool:
+        """Converse-family models report camelCase token counts, not Anthropic's snake_case."""
+        return "inputTokens" in usage_object or "outputTokens" in usage_object
+
+    @staticmethod
+    def _usage_count(usage_object: Mapping[str, object], *keys: str) -> int:
+        for key in keys:
+            value = usage_object.get(key)
+            if isinstance(value, (int, float)) and not isinstance(value, bool):
+                return int(value)
+        return 0
+
+    def usage_from_batch_output(self, usage_object: Mapping[str, object]) -> Usage:
+        """Read a Converse-shaped usage block out of a batch output line.
+
+        Batch output omits fields the live API always sends, so the block is
+        completed before going through the same transform, keeping a batch and an
+        equivalent non-batch call in agreement on tokens.
+        """
+        input_tokens: Final = self._usage_count(usage_object, "inputTokens")
+        output_tokens: Final = self._usage_count(usage_object, "outputTokens")
+        cache_read: Final = self._usage_count(usage_object, "cacheReadInputTokens", "cacheReadInputTokenCount")
+        cache_write: Final = self._usage_count(usage_object, "cacheWriteInputTokens", "cacheWriteInputTokenCount")
+        return self.transform_usage(
+            ConverseTokenUsageBlock(
+                inputTokens=input_tokens,
+                outputTokens=output_tokens,
+                totalTokens=self._usage_count(usage_object, "totalTokens") or input_tokens + output_tokens,
+                cacheReadInputTokenCount=cache_read,
+                cacheReadInputTokens=cache_read,
+                cacheWriteInputTokenCount=cache_write,
+                cacheWriteInputTokens=cache_write,
+            )
+        )
+
+    def transform_usage(
         self,
         usage: ConverseTokenUsageBlock,
         reasoning_content: str | None = None,
@@ -2191,7 +2248,7 @@ class AmazonConverseConfig(BaseConfig):
             chat_completion_message["tool_calls"] = filtered_tools
 
         ## CALCULATING USAGE - bedrock returns usage in the headers
-        usage: Final = self._transform_usage(
+        usage: Final = self.transform_usage(
             completion_response["usage"],
             reasoning_content=chat_completion_message.get("reasoning_content"),
         )
@@ -2258,7 +2315,8 @@ class AmazonConverseConfig(BaseConfig):
     ) -> dict:
         if api_key:
             headers["Authorization"] = f"Bearer {api_key}"
-        return headers
+        owned_names, metadata_headers = bedrock_request_metadata_headers(litellm_params)
+        return merge_bedrock_invoke_headers(headers, (), metadata_headers, owned_names)
 
     def should_fake_stream(
         self,

litellm/llms/bedrock/chat/invoke_handler.py

@@ -559,7 +559,7 @@ class AWSEventStreamDecoder:
             elif "stopReason" in chunk_data:
                 finish_reason = map_finish_reason(chunk_data.get("stopReason", "stop"))
             elif "usage" in chunk_data:
-                usage = converse_config._transform_usage(chunk_data.get("usage", {}))
+                usage = converse_config.transform_usage(chunk_data.get("usage", {}))
 
             model_response_provider_specific_fields: Final = {}
             if "trace" in chunk_data:

litellm/llms/bedrock/chat/invoke_transformations/amazon_openai_transformation.py

@@ -13,6 +13,10 @@ import httpx
 
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
 from litellm.llms.bedrock.common_utils import BedrockError
+from litellm.llms.bedrock.request_metadata import (
+    bedrock_request_metadata_headers,
+    merge_bedrock_invoke_headers,
+)
 from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
 from litellm.passthrough.utils import CommonUtils
 from litellm.types.llms.openai import AllMessageValues
@@ -169,9 +173,12 @@ class AmazonBedrockOpenAIConfig(OpenAIGPTConfig, BaseAWSLLM):
         """
         Validate the environment and return headers.
 
-        For Bedrock, we don't need Bearer token auth since we use AWS SigV4.
+        For Bedrock, we don't need Bearer token auth since we use AWS SigV4. This path signs the
+        same ``/model/{id}/invoke`` endpoint as ``AmazonInvokeConfig``, so it owns the request
+        metadata header on the same terms rather than letting a caller supply it.
         """
-        return headers
+        owned_names, metadata_headers = bedrock_request_metadata_headers(litellm_params)
+        return merge_bedrock_invoke_headers(headers, (), metadata_headers, owned_names)
 
     def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BedrockError:
         """Return the appropriate error class for Bedrock."""

litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py

@@ -20,6 +20,10 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
 from litellm.llms.bedrock.chat.invoke_handler import make_call, make_sync_call
 from litellm.llms.bedrock.common_utils import BedrockError
+from litellm.llms.bedrock.request_metadata import (
+    bedrock_request_metadata_headers,
+    merge_bedrock_invoke_headers,
+)
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
     HTTPHandler,
@@ -417,15 +421,13 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
         api_base: str | None = None,
     ) -> dict:
         raw_guardrail_config: Final = optional_params.pop("guardrailConfig", None)
-        if raw_guardrail_config is None:
-            return headers
-        existing_header_names: Final = frozenset(name.lower() for name in headers)
-        guardrail_headers: Final = {
-            name: value
-            for name, value in _bedrock_invoke_guardrail_headers(raw_guardrail_config).items()
-            if name.lower() not in existing_header_names
-        }
-        return {**headers, **guardrail_headers}
+        guardrail_headers: Final = (
+            ()
+            if raw_guardrail_config is None
+            else tuple(_bedrock_invoke_guardrail_headers(raw_guardrail_config).items())
+        )
+        owned_names, metadata_headers = bedrock_request_metadata_headers(litellm_params)
+        return merge_bedrock_invoke_headers(headers, guardrail_headers, metadata_headers, owned_names)
 
     def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException:
         return BedrockError(status_code=status_code, message=error_message)

litellm/llms/bedrock/files/transformation.py

@@ -3,6 +3,7 @@ import json
 import os
 import time
 from collections.abc import Iterable, Mapping, MutableMapping, Sequence
+from contextlib import suppress
 from functools import cache
 from itertools import chain
 from types import MappingProxyType
@@ -64,6 +65,10 @@ from ..common_utils import BedrockError, merge_bedrock_aws_request_params, resol
 # Same pattern as the `upload_url` handoff in `transform_create_file_request`.
 S3_SIGNED_GET_HEADERS_PARAM: Final = "_s3_signed_get_headers"
 
+# litellm_params key carrying the size of the body uploaded to S3, handed from
+# `transform_create_file_request` to `transform_create_file_response`.
+UPLOAD_CONTENT_LENGTH_PARAM: Final = "_s3_upload_content_length"
+
 
 def _frozen_mapping(items: Iterable[tuple[str, object]]) -> Mapping[str, object]:
     return MappingProxyType(dict(items))
@@ -145,11 +150,12 @@ class _BedrockS3RequestParams(BaseModel):
 
 
 class _TrustedS3ModelCredentials(BaseModel):
-    """The S3 bucket the server trusts file ids against, from the deployment snapshot."""
+    """The S3 buckets the server trusts file ids against, from the deployment snapshot."""
 
     model_config = ConfigDict(extra="ignore")
 
     s3_bucket_name: str | None = None
+    s3_output_bucket_name: str | None = None
 
 
 def extract_s3_uri_from_file_id(file_id: str) -> str:
@@ -175,6 +181,18 @@ def extract_s3_uri_from_file_id(file_id: str) -> str:
     raise ValueError("file_id must be a managed LiteLLM S3 file id")
 
 
+_S3_BUCKET_REQUIRED_ERROR: Final = "S3 bucket_name is required. Set 's3_bucket_name' in proxy config or AWS_S3_BUCKET_NAME for Bedrock file content retrieval."
+
+
+def _trusted_s3_model_credentials(litellm_params: Mapping[str, object]) -> _TrustedS3ModelCredentials:
+    trusted_model_credentials: Final = litellm_params.get("_litellm_internal_model_credentials")
+    if not isinstance(trusted_model_credentials, MappingProxyType):
+        return _TrustedS3ModelCredentials()
+    snapshot: Final[dict[str, object]] = {}
+    snapshot.update(trusted_model_credentials)  # any-ok: untyped snapshot
+    return _TrustedS3ModelCredentials.model_validate(snapshot)
+
+
 def get_configured_s3_bucket_name(litellm_params: Mapping[str, object]) -> str:
     """
     Resolve the server-configured S3 bucket for Bedrock file operations.
@@ -183,20 +201,62 @@ def get_configured_s3_bucket_name(litellm_params: Mapping[str, object]) -> str:
     environment; never a request-supplied param, since the bucket is what
     `validate_managed_cloud_file_id` checks file ids against.
     """
-    trusted_model_credentials: Final = litellm_params.get("_litellm_internal_model_credentials")
-    bucket_name: str | None = None
-    if isinstance(trusted_model_credentials, MappingProxyType):
-        snapshot: Final[dict[str, object]] = {}
-        snapshot.update(trusted_model_credentials)  # any-ok: untyped snapshot
-        bucket_name = _TrustedS3ModelCredentials.model_validate(snapshot).s3_bucket_name
-    bucket_name = bucket_name or os.getenv("AWS_S3_BUCKET_NAME")
+    bucket_name: Final = _trusted_s3_model_credentials(litellm_params).s3_bucket_name or os.getenv("AWS_S3_BUCKET_NAME")
     if not bucket_name:
-        raise ValueError(
-            "S3 bucket_name is required. Set 's3_bucket_name' in proxy config or AWS_S3_BUCKET_NAME for Bedrock file content retrieval."
-        )
+        raise ValueError(_S3_BUCKET_REQUIRED_ERROR)
     return bucket_name
 
 
+def get_configured_s3_bucket_names(litellm_params: Mapping[str, object]) -> tuple[str, ...]:
+    """
+    Resolve the server-configured S3 buckets a Bedrock file id may live in.
+
+    Bedrock batch outputs land in ``s3_output_bucket_name`` when it differs from
+    the input bucket, so retrieval validates against both. Same trust rules as
+    ``get_configured_s3_bucket_name``: only the immutable credential snapshot or
+    the environment, never a request param.
+    """
+    trusted: Final = _trusted_s3_model_credentials(litellm_params)
+    input_bucket: Final = trusted.s3_bucket_name or os.getenv("AWS_S3_BUCKET_NAME")
+    output_bucket: Final = trusted.s3_output_bucket_name or os.getenv("AWS_S3_OUTPUT_BUCKET_NAME")
+    buckets: Final = tuple(dict.fromkeys(bucket for bucket in (input_bucket, output_bucket) if bucket))
+    if not buckets:
+        raise ValueError(_S3_BUCKET_REQUIRED_ERROR)
+    return buckets
+
+
+def _validate_file_id_against_configured_buckets(
+    s3_uri: str,
+    configured_bucket_names: tuple[str, ...],
+    allow_legacy_cloud_file_ids: bool,
+) -> tuple[str, str]:
+    def validate_against(configured_bucket_name: str) -> tuple[str, str]:
+        return validate_managed_cloud_file_id(
+            file_id=s3_uri,
+            scheme="s3://",
+            configured_bucket_name=configured_bucket_name,
+            allowed_object_prefixes=BEDROCK_MANAGED_S3_PREFIXES,
+            allow_legacy_cloud_file_ids=allow_legacy_cloud_file_ids,
+        )
+
+    for candidate_bucket_name in configured_bucket_names[:-1]:
+        with suppress(ValueError):
+            return validate_against(candidate_bucket_name)
+    return validate_against(configured_bucket_names[-1])
+
+
+def _uploaded_object_size(litellm_params: Mapping[str, object], raw_response: Response) -> int:
+    """
+    S3 answers PutObject with an empty body, so the stored object size comes from the
+    signed request recorded by `transform_create_file_request`, not the response headers.
+    """
+    uploaded_size: Final = litellm_params.get(UPLOAD_CONTENT_LENGTH_PARAM)
+    if isinstance(uploaded_size, int):
+        return uploaded_size
+    response_content_length: Final = raw_response.headers.get("Content-Length", "0")
+    return int(response_content_length) if response_content_length.isdigit() else 0
+
+
 class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
     """
     Config for Bedrock Files - handles S3 uploads for Bedrock batch processing
@@ -924,6 +984,8 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
         )
 
         litellm_params["upload_url"] = api_base
+        upload_content_length: Final = len(file_content.encode("utf-8"))
+        litellm_params[UPLOAD_CONTENT_LENGTH_PARAM] = upload_content_length  # rebind-ok: same handoff as upload_url
 
         # Return a dict that tells the HTTP handler exactly what to do
         return {
@@ -1081,12 +1143,6 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
         """
         Transform S3 File upload response into OpenAI-style FileObject
         """
-        # For S3 uploads, we typically get an ETag and other metadata
-        response_headers: Final = raw_response.headers
-        # Extract S3 object information from the response
-        # S3 PUT object returns ETag and other metadata in headers
-        content_length: Final[str] = response_headers.get("Content-Length", "0")
-
         # Use the actual upload URL that was used for the S3 upload
         upload_url: Final = litellm_params.get("upload_url")
         file_id: str = ""
@@ -1101,7 +1157,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
             filename=filename,
             created_at=int(time.time()),  # Current timestamp
             status="uploaded",
-            bytes=int(content_length) if content_length.isdigit() else 0,
+            bytes=_uploaded_object_size(litellm_params=litellm_params, raw_response=raw_response),
             object="file",
         )
 
@@ -1174,11 +1230,9 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
             raise ValueError("file_id is required for Bedrock file content retrieval")
 
         s3_uri: Final = extract_s3_uri_from_file_id(file_id)
-        bucket_name, object_key = validate_managed_cloud_file_id(
-            file_id=s3_uri,
-            scheme="s3://",
-            configured_bucket_name=get_configured_s3_bucket_name(litellm_params),
-            allowed_object_prefixes=BEDROCK_MANAGED_S3_PREFIXES,
+        bucket_name, object_key = _validate_file_id_against_configured_buckets(
+            s3_uri=s3_uri,
+            configured_bucket_names=get_configured_s3_bucket_names(litellm_params),
             allow_legacy_cloud_file_ids=should_allow_legacy_cloud_file_ids(litellm_params),
         )
 

litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py

@@ -1,4 +1,5 @@
 from collections.abc import AsyncIterator
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, cast
 
 import httpx
@@ -37,6 +38,10 @@ from litellm.llms.bedrock.common_utils import (
     normalize_tool_input_schema_types_for_bedrock_invoke,
     pop_bedrock_invoke_output_config_format,
 )
+from litellm.llms.bedrock.request_metadata import (
+    bedrock_request_metadata_headers,
+    merge_bedrock_invoke_headers,
+)
 from litellm.types.llms.anthropic import (
     ANTHROPIC_BETA_HEADER_VALUES,
     ANTHROPIC_TOOL_SEARCH_BETA_HEADER,
@@ -89,7 +94,8 @@ class AmazonAnthropicClaudeMessagesConfig(
         api_key: str | None = None,
         api_base: str | None = None,
     ) -> tuple[dict, str | None]:
-        return headers, api_base
+        owned_names, metadata_headers = bedrock_request_metadata_headers(litellm_params)
+        return merge_bedrock_invoke_headers(headers, (), metadata_headers, owned_names), api_base
 
     def sign_request(
         self,
@@ -956,13 +962,32 @@ class AmazonAnthropicClaudeMessagesStreamDecoder(AWSEventStreamDecoder):
         Bedrock returns usage metrics using camelCase keys. Convert these to
         the Anthropic `/v1/messages` specification so callers receive a
         consistent response shape when streaming.
+
+        Token counts already present in the chunk's own Anthropic usage block
+        win over the invocationMetrics-derived ones, and cache token fields
+        (``cache_read_input_tokens`` / ``cache_creation_input_tokens`` on
+        ``message_stop.usage``, or ``cacheReadInputTokenCount`` /
+        ``cacheWriteInputTokenCount`` inside the invocation metrics) are
+        preserved: ``invocationMetrics.inputTokenCount`` excludes cache reads
+        and writes, so replacing the whole usage block with input/output counts
+        alone drops the cache breakdown, ``_promote_message_stop_usage`` has
+        nothing left to promote, and cache tokens end up billed at $0.
         """
         amazon_bedrock_invocation_metrics: Final = chunk_data.pop("amazon-bedrock-invocationMetrics", {})
         if amazon_bedrock_invocation_metrics:
-            anthropic_usage: Final = {}
-            if "inputTokenCount" in amazon_bedrock_invocation_metrics:
-                anthropic_usage["input_tokens"] = amazon_bedrock_invocation_metrics["inputTokenCount"]
-            if "outputTokenCount" in amazon_bedrock_invocation_metrics:
-                anthropic_usage["output_tokens"] = amazon_bedrock_invocation_metrics["outputTokenCount"]
-            chunk_data["usage"] = anthropic_usage
+            existing_usage: Final = chunk_data.get("usage")
+            preserved_usage: Final = existing_usage if isinstance(existing_usage, dict) else MappingProxyType({})
+            metrics_usage: Final = MappingProxyType(
+                {
+                    anthropic_key: amazon_bedrock_invocation_metrics[metrics_key]
+                    for anthropic_key, metrics_key in (
+                        ("input_tokens", "inputTokenCount"),
+                        ("output_tokens", "outputTokenCount"),
+                        ("cache_read_input_tokens", "cacheReadInputTokenCount"),
+                        ("cache_creation_input_tokens", "cacheWriteInputTokenCount"),
+                    )
+                    if metrics_key in amazon_bedrock_invocation_metrics
+                }
+            )
+            chunk_data["usage"] = {**metrics_usage, **preserved_usage}
         return chunk_data

litellm/llms/bedrock/request_metadata.py

@@ -0,0 +1,199 @@
+"""
+Resolve AWS Bedrock ``requestMetadata`` from LiteLLM proxy identity and caller metadata.
+
+Bedrock attaches request metadata to CloudTrail records and to the dimension AWS Cost
+Explorer groups on, so everything here is opt-in: nothing is forwarded unless the operator
+sets ``litellm.bedrock_request_metadata_fields`` (``litellm_settings`` on the proxy).
+
+Two properties are load-bearing for that billing record and are asserted by the tests:
+proxy identity is resolved first so it can never be evicted by caller-supplied pairs, and the
+whole ``user_api_key_`` prefix is reserved so a caller cannot write a proxy-authoritative
+looking key. Values that break Bedrock's constraints are dropped rather than sanitised or
+rejected, because an operator flipping this setting on must not turn a working request into a
+400 and a silently rewritten attribution key is worse than an absent one.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Mapping
+from typing import Final
+
+import litellm
+
+BEDROCK_REQUEST_METADATA_HEADER: Final = "X-Amzn-Bedrock-Request-Metadata"
+BEDROCK_REQUEST_METADATA_MAX_PAIRS: Final = 16
+BEDROCK_REQUEST_METADATA_IDENTITY_PREFIX: Final = "user_api_key_"
+BEDROCK_REQUEST_METADATA_CLIENT_FIELD: Final = "spend_logs_metadata"
+
+_METADATA_PARAM_NAMES: Final[tuple[str, ...]] = ("metadata", "litellm_metadata")
+_KEY_PATTERN: Final = re.compile(r"^[a-zA-Z0-9\s:_@$#=/+,.-]{1,256}$")
+_VALUE_PATTERN: Final = re.compile(r"^[a-zA-Z0-9\s:_@$#=/+,.-]{0,256}$")
+_OWNED_HEADER_NAMES: Final[frozenset[str]] = frozenset((BEDROCK_REQUEST_METADATA_HEADER.lower(),))
+
+
+def _is_forwardable(key: str, value: str) -> bool:
+    return _KEY_PATTERN.match(key) is not None and _VALUE_PATTERN.match(value) is not None
+
+
+def _text_pairs(source: object) -> tuple[tuple[str, str], ...]:
+    if not isinstance(source, Mapping):
+        return ()
+    return tuple((key, value) for key, value in source.items() if isinstance(key, str) and isinstance(value, str))
+
+
+def _allowed_fields() -> tuple[str, ...]:
+    """
+    The operator allow-list, deduplicated so a field repeated in config cannot consume a second
+    reserved slot and shrink the client budget for nothing. First occurrence wins, which keeps
+    the operator's declared precedence intact.
+    """
+    configured: Final[object] = litellm.bedrock_request_metadata_fields
+    if not isinstance(configured, (list, tuple)):
+        return ()
+    fields: Final = tuple(str(field) for field in configured)
+    return tuple(field for index, field in enumerate(fields) if field not in fields[:index])
+
+
+def _metadata_sources(litellm_params: Mapping[str, object] | None) -> tuple[Mapping[str, object], ...]:
+    """``metadata`` on /v1/chat/completions, ``litellm_metadata`` on the LITELLM_METADATA_ROUTES."""
+    if litellm_params is None:
+        return ()
+    return tuple(
+        source
+        for name in _METADATA_PARAM_NAMES
+        for source in (litellm_params.get(name),)
+        if isinstance(source, Mapping)
+    )
+
+
+def _identity_pairs(
+    sources: tuple[Mapping[str, object], ...],
+    allowed_fields: tuple[str, ...],
+) -> tuple[tuple[str, str], ...]:
+    return tuple(
+        (field, value)
+        for field in allowed_fields
+        if field.startswith(BEDROCK_REQUEST_METADATA_IDENTITY_PREFIX)
+        for value in (_first_text(sources, field),)
+        if value is not None and _is_forwardable(field, value)
+    )[:BEDROCK_REQUEST_METADATA_MAX_PAIRS]
+
+
+def _first_text(sources: tuple[Mapping[str, object], ...], field: str) -> str | None:
+    return next((value for source in sources if isinstance(value := source.get(field), str)), None)
+
+
+def _client_pairs(
+    sources: tuple[Mapping[str, object], ...],
+    allowed_fields: tuple[str, ...],
+    caller_metadata: object,
+    budget: int,
+) -> tuple[tuple[str, str], ...]:
+    spend_logs_pairs: Final = (
+        tuple(pair for source in sources for pair in _text_pairs(source.get(BEDROCK_REQUEST_METADATA_CLIENT_FIELD)))
+        if BEDROCK_REQUEST_METADATA_CLIENT_FIELD in allowed_fields
+        else ()
+    )
+    candidates: Final = tuple(
+        (key, value)
+        for key, value in (*_text_pairs(caller_metadata), *spend_logs_pairs)
+        if not key.startswith(BEDROCK_REQUEST_METADATA_IDENTITY_PREFIX) and _is_forwardable(key, value)
+    )
+    return tuple(
+        pair
+        for index, pair in enumerate(candidates)
+        if pair[0] not in tuple(earlier for earlier, _ in candidates[:index])
+    )[:budget]
+
+
+def resolve_bedrock_request_metadata(
+    litellm_params: Mapping[str, object] | None,
+    caller_metadata: object = None,
+) -> dict[str, str] | None:
+    """
+    Resolve the ``requestMetadata`` pairs to send to Bedrock, or ``None`` when the feature is
+    off or nothing survives Bedrock's constraints. The result is a plain dict because it is
+    written straight onto the Converse body, which Bedrock types as ``dict[str, str]``.
+
+    ``caller_metadata`` is any ``requestMetadata`` the caller passed explicitly. It has already
+    been validated (and rejected with a 400) by the Converse transformation, so it is only
+    filtered here for the reserved identity prefix and the remaining slot budget.
+    """
+    allowed_fields: Final = _allowed_fields()
+    if not allowed_fields:
+        return None
+    sources: Final = _metadata_sources(litellm_params)
+    identity: Final = _identity_pairs(sources, allowed_fields)
+    client: Final = _client_pairs(
+        sources=sources,
+        allowed_fields=allowed_fields,
+        caller_metadata=caller_metadata,
+        budget=BEDROCK_REQUEST_METADATA_MAX_PAIRS - len(identity),
+    )
+    resolved: Final = {key: value for key, value in (*identity, *client)}
+    return resolved or None
+
+
+def bedrock_request_metadata_is_owned() -> bool:
+    """
+    Whether the proxy OWNS the request-metadata field and header name for this request.
+
+    Ownership follows the operator's opt-in alone, never whether anything resolved, because a
+    caller can suppress the resolver by omitting the allow-listed fields or by sending values
+    that all fail Bedrock's rules. Owned-but-empty has to mean "absent on the wire" rather than
+    "fall back to whatever the caller supplied", or the reserved-prefix guarantee is bypassable
+    by anyone who can make the resolver produce nothing.
+    """
+    return bool(_allowed_fields())
+
+
+def bedrock_request_metadata_headers(
+    litellm_params: Mapping[str, object] | None,
+) -> tuple[frozenset[str], tuple[tuple[str, str], ...]]:
+    """
+    The signed ``X-Amzn-Bedrock-Request-Metadata`` header for the Invoke paths, which have no
+    body field for request metadata.
+
+    Returns the header names the proxy OWNS and, separately, the pairs to send. Ownership is
+    reported whenever forwarding is enabled, including when nothing resolves, because a caller
+    can suppress the resolver (omit the allow-listed fields, or send values that all fail
+    Bedrock's rules) and an owned-but-empty result must still evict the caller's header rather
+    than fall back to it.
+    """
+    if not bedrock_request_metadata_is_owned():
+        return frozenset(), ()
+    resolved: Final = resolve_bedrock_request_metadata(litellm_params)
+    if resolved is None:
+        return _OWNED_HEADER_NAMES, ()
+    return _OWNED_HEADER_NAMES, ((BEDROCK_REQUEST_METADATA_HEADER, json.dumps(resolved, separators=(",", ":"))),)
+
+
+def merge_bedrock_invoke_headers(
+    headers: dict[str, str],
+    caller_owned: tuple[tuple[str, str], ...],
+    proxy_owned: tuple[tuple[str, str], ...],
+    proxy_owned_names: frozenset[str],
+) -> dict[str, str]:
+    """
+    Merge the ``X-Amzn-*`` headers the Invoke paths derive from params.
+
+    ``caller_owned`` (the guardrail headers) defers to a header the caller already set, which is
+    the long-standing behaviour for those. ``proxy_owned_names`` are dropped from the caller's
+    headers unconditionally and re-supplied only from ``proxy_owned``, because those names carry
+    proxy-authenticated identity into an AWS billing record that the caller must not be able to
+    write. Names are compared case-insensitively so a caller cannot leave a second spelling in
+    the dict and let the transport pick the winner.
+    """
+    if not caller_owned and not proxy_owned and not proxy_owned_names:
+        return headers
+    existing_names: Final = frozenset(name.lower() for name in headers)
+    return {
+        name: value
+        for name, value in (
+            *((n, v) for n, v in headers.items() if n.lower() not in proxy_owned_names),
+            *((n, v) for n, v in caller_owned if n.lower() not in existing_names),
+            *proxy_owned,
+        )
+    }

litellm/llms/custom_httpx/llm_http_handler.py

@@ -2,7 +2,7 @@ import asyncio
 import json
 import os
 import ssl
-from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping
+from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
 from contextlib import asynccontextmanager
 from functools import lru_cache
 from types import ModuleType
@@ -49,13 +49,17 @@ from litellm.llms.base_llm.image_generation.transformation import (
     BaseImageGenerationConfig,
 )
 from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig, OCRResponse
+from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig
 from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
 from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig
 from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
 from litellm.llms.base_llm.search.transformation import BaseSearchConfig, SearchResponse
 from litellm.llms.base_llm.skills.transformation import BaseSkillsAPIConfig
 from litellm.llms.base_llm.text_to_speech.transformation import BaseTextToSpeechConfig
-from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig
+from litellm.llms.base_llm.vector_store.transformation import (
+    BaseDirectVectorStoreConfig,
+    BaseVectorStoreConfig,
+)
 from litellm.llms.base_llm.vector_store_files.transformation import (
     BaseVectorStoreFilesConfig,
 )
@@ -69,6 +73,7 @@ from litellm.llms.custom_httpx.http_handler import (
 from litellm.responses.streaming_iterator import (
     BaseResponsesAPIStreamingIterator,
     MockResponsesAPIStreamingIterator,
+    ProjectQuotaCallback,
     ResponsesAPIStreamingIterator,
     ResponsesWebSocketStreaming,
     SyncResponsesAPIStreamingIterator,
@@ -252,6 +257,27 @@ def _has_pre_call_deployment_hook(logging_obj: LiteLLMLoggingObj) -> bool:
     return False
 
 
+def _collect_ws_project_quota_callbacks() -> tuple[ProjectQuotaCallback, ...]:
+    """Duck-type discover proxy hooks exposing per-frame project ITPM/OTPM
+    enforcement, so the Responses WebSocket loop can charge every
+    ``response.create`` frame, not just the connection's first one.
+
+    Uses duck-typing on ``litellm.callbacks`` (rather than importing the
+    proxy hook directly) to avoid a layering violation (SDK importing from
+    the proxy layer).
+    """
+    import litellm as _litellm
+
+    callbacks: Final = cast(  # cast-ok: callback registry is inspected before protocol use
+        Sequence[object], _litellm.callbacks
+    )
+    return tuple(
+        cast(ProjectQuotaCallback, callback)  # cast-ok: required callback method is callable
+        for callback in callbacks
+        if callable(getattr(callback, "enforce_project_io_token_quota_for_frame", None))
+    )
+
+
 class BaseLLMHTTPHandler:
     async def _make_common_async_call(
         self,
@@ -892,6 +918,7 @@ class BaseLLMHTTPHandler:
         )
         if provider_config is None:
             raise ValueError(f"Provider {custom_llm_provider} does not support embedding")
+        embedding_extra_body: Final[Mapping[str, object] | None] = optional_params.pop("extra_body", None)
         # get config from model, custom llm provider
         headers = provider_config.validate_environment(
             api_key=api_key,
@@ -916,6 +943,8 @@ class BaseLLMHTTPHandler:
             optional_params=optional_params,
             headers=headers,
         )
+        if embedding_extra_body:
+            data.update(embedding_extra_body)
 
         # Some providers (e.g. OCI) require request signing after the body is built.
         # The default BaseConfig.sign_request returns (headers, None) — a no-op for
@@ -1555,12 +1584,14 @@ class BaseLLMHTTPHandler:
         model: str,
         response: httpx.Response,
         logging_obj: LiteLLMLoggingObj,
+        optional_params: Mapping[str, object],
     ) -> OCRResponse:
         """Shared logic for transforming OCR responses."""
         return provider_config.transform_ocr_response(
             model=model,
             raw_response=response,
             logging_obj=logging_obj,
+            optional_params=optional_params,
         )
 
     def ocr(
@@ -1636,6 +1667,7 @@ class BaseLLMHTTPHandler:
             model=model,
             response=response,
             logging_obj=logging_obj,
+            optional_params=optional_params,
         )
 
     async def async_ocr(
@@ -1698,6 +1730,7 @@ class BaseLLMHTTPHandler:
             model=model,
             raw_response=response,
             logging_obj=logging_obj,
+            optional_params=optional_params,
         )
 
     def search(
@@ -5930,10 +5963,10 @@ class BaseLLMHTTPHandler:
         self,
         api_base: str,
         api_key: str,
-        request_data: dict[str, Any],
+        request_data: dict[str, object],
         logging_obj: LiteLLMLoggingObj,
         timeout: float | httpx.Timeout,
-        provider_config: Any | None = None,
+        provider_config: BaseRealtimeHTTPConfig | None = None,
         model: str | None = None,
         extra_headers: dict[str, object] | None = None,
         client: HTTPHandler | AsyncHTTPHandler | None = None,
@@ -5963,10 +5996,10 @@ class BaseLLMHTTPHandler:
         self,
         api_base: str,
         api_key: str,
-        request_data: dict[str, Any],
+        request_data: dict[str, object],
         logging_obj: LiteLLMLoggingObj,
         timeout: float | httpx.Timeout,
-        provider_config: Any | None = None,
+        provider_config: BaseRealtimeHTTPConfig | None = None,
         model: str | None = None,
         extra_headers: dict[str, object] | None = None,
         client: HTTPHandler | AsyncHTTPHandler | None = None,
@@ -5992,7 +6025,7 @@ class BaseLLMHTTPHandler:
         endpoint: Literal["client_secrets", "transcription_sessions"],
         api_base: str,
         api_key: str,
-        request_data: dict[str, Any],
+        request_data: dict[str, object],
         logging_obj: LiteLLMLoggingObj,
         timeout: float | httpx.Timeout,
         provider_config: Any | None = None,
@@ -6168,6 +6201,8 @@ class BaseLLMHTTPHandler:
         - Uses ManagedResponsesWebSocketHandler which makes HTTP streaming calls
         - Forwards events over the websocket connection
         """
+        _ws_quota_callbacks: Final = _collect_ws_project_quota_callbacks()
+
         if responses_api_provider_config is None or not responses_api_provider_config.supports_native_websocket():
             from litellm.responses.streaming_iterator import (
                 ManagedResponsesWebSocketHandler,
@@ -6184,6 +6219,7 @@ class BaseLLMHTTPHandler:
                 timeout=timeout,
                 custom_llm_provider=custom_llm_provider,
                 first_message=first_message,
+                quota_callbacks=_ws_quota_callbacks,
                 **kwargs,
             )
             await handler.run()
@@ -6304,6 +6340,7 @@ class BaseLLMHTTPHandler:
                     first_message=first_message,
                     guardrail_callbacks=_ws_guardrail_callbacks,
                     output_guardrail_callbacks=_ws_output_guardrail_callbacks,
+                    quota_callbacks=_ws_quota_callbacks,
                     authorized_model=model,
                 )
                 await streaming.bidirectional_forward()
@@ -9396,6 +9433,27 @@ class BaseLLMHTTPHandler:
             )
 
     ###### VECTOR STORE HANDLER ######
+    @staticmethod
+    def _pre_call_direct_vector_store_search(
+        logging_obj: LiteLLMLoggingObj,
+        custom_llm_provider: str,
+        vector_store_id: str,
+        query: str | Sequence[str],
+    ) -> None:
+        """Direct providers have no HTTP request to echo, and an empty api_base makes the debug
+        logger fall back to dumping model_call_details, which holds stored provider credentials."""
+        endpoint: Final = f"{custom_llm_provider}://{vector_store_id}"
+        logging_obj.pre_call(
+            input="",
+            api_key="",
+            additional_args={  # mutable-ok: pre_call's additional_args contract is a dict
+                "query": query,
+                "vector_store_id": vector_store_id,
+                "api_base": endpoint,
+                "request_str": f"direct vector store search: {endpoint}",
+            },
+        )
+
     async def async_vector_store_search_handler(
         self,
         vector_store_id: str,
@@ -9411,6 +9469,22 @@ class BaseLLMHTTPHandler:
         client: HTTPHandler | AsyncHTTPHandler | None = None,
         _is_async: bool = False,
     ) -> VectorStoreSearchResponse:
+        if isinstance(vector_store_provider_config, BaseDirectVectorStoreConfig):
+            self._pre_call_direct_vector_store_search(
+                logging_obj=logging_obj,
+                custom_llm_provider=custom_llm_provider,
+                vector_store_id=vector_store_id,
+                query=query,
+            )
+            return await vector_store_provider_config.aexecute_search_vector_store_request(
+                vector_store_id=vector_store_id,
+                query=query,
+                vector_store_search_optional_params=vector_store_search_optional_params,
+                litellm_logging_obj=logging_obj,
+                litellm_params=dict(litellm_params),  # mutable-ok: snapshot GenericLiteLLMParams into the Mapping shape
+                timeout=timeout,
+            )
+
         if client is None or not isinstance(client, AsyncHTTPHandler):
             async_httpx_client = get_async_httpx_client(
                 llm_provider=litellm.LlmProviders(custom_llm_provider),
@@ -9524,6 +9598,22 @@ class BaseLLMHTTPHandler:
                 client=client,
             )
 
+        if isinstance(vector_store_provider_config, BaseDirectVectorStoreConfig):
+            self._pre_call_direct_vector_store_search(
+                logging_obj=logging_obj,
+                custom_llm_provider=custom_llm_provider,
+                vector_store_id=vector_store_id,
+                query=query,
+            )
+            return vector_store_provider_config.execute_search_vector_store_request(
+                vector_store_id=vector_store_id,
+                query=query,
+                vector_store_search_optional_params=vector_store_search_optional_params,
+                litellm_logging_obj=logging_obj,
+                litellm_params=dict(litellm_params),  # mutable-ok: snapshot GenericLiteLLMParams into the Mapping shape
+                timeout=timeout,
+            )
+
         if client is None or not isinstance(client, HTTPHandler):
             sync_httpx_client = _get_httpx_client(params={"ssl_verify": litellm_params.get("ssl_verify", None)})
         else:
@@ -11077,7 +11167,7 @@ class BaseLLMHTTPHandler:
         client: HTTPHandler | AsyncHTTPHandler | None = None,
         stream: bool = False,
         litellm_metadata: dict[str, object] | None = None,
-        system_instruction: Any | None = None,
+        system_instruction: object | None = None,
     ) -> Any:
         """
         Handles Google GenAI generate content requests.
@@ -11208,7 +11298,7 @@ class BaseLLMHTTPHandler:
         client: AsyncHTTPHandler | None = None,
         stream: bool = False,
         litellm_metadata: dict[str, object] | None = None,
-        system_instruction: Any | None = None,
+        system_instruction: object | None = None,
     ) -> Any:
         """
         Async version of the generate content handler.

litellm/llms/fireworks_ai/common_utils.py

@@ -29,9 +29,12 @@ def get_fireworks_session_id(litellm_params: dict) -> str | None:
     return None
 
 
+AZURE_FOUNDRY_FIREWORKS_MODEL_ID_PREFIX: Final = "FW-"
+
+
 def resolve_fireworks_resource_name(model: str) -> str:
     stripped: Final = model.removeprefix("fireworks_ai/")
-    if stripped.startswith("accounts/") or "#" in stripped:
+    if stripped.startswith(("accounts/", AZURE_FOUNDRY_FIREWORKS_MODEL_ID_PREFIX)) or "#" in stripped:
         return stripped
     if stripped.startswith(("routers/", "models/")):
         return f"accounts/fireworks/{stripped}"

litellm/llms/tinyfish/search/transformation.py

@@ -14,6 +14,7 @@ import httpx
 from pydantic import TypeAdapter, ValidationError
 
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.core_helpers import process_response_headers
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.base_llm.search.transformation import (
@@ -22,7 +23,7 @@ from litellm.llms.base_llm.search.transformation import (
 )
 from litellm.secret_managers.main import get_secret_str
 
-_UrlEncodableParams: Final = TypeAdapter(dict[str, str | int | bool])
+_UrlEncodableParams: Final = TypeAdapter(dict[str, str | int | float | bool])
 _StrList: Final = TypeAdapter(list[str])
 _StrFrozenSet: Final = TypeAdapter(frozenset[str])
 
@@ -94,16 +95,16 @@ class TinyfishSearchConfig(BaseSearchConfig):
         TinyFish equivalents:
         - ``query`` (str or list[str]) → ``query`` (list joined by spaces)
         - ``country`` → ``location``
-        - ``search_domain_filter`` (list[str]) → folded into the query as
-          ``(<query>) (site:a OR site:b ...)`` (TinyFish has no first-class
-          field today; see ML-2084 for the planned ``include_domains``)
+        - ``search_domain_filter`` (list[str]) → folded into the query using
+          search operators
         - ``max_results`` → not sent on the wire; stashed on
           ``self._caller_max_results`` for client-side response truncation
           (TinyFish doesn't honor it server-side)
         - ``max_tokens_per_page`` → silently dropped (no TinyFish equivalent)
 
         Any other ``optional_params`` keys are forwarded to TinyFish as-is.
-        dict/list values are JSON-encoded so they survive ``urlencode``.
+        dict and list values are JSON-encoded so structured payloads survive
+        ``urlencode``.
 
         Returns:
             ``{_TINYFISH_PARAMS_KEY: <dict of querystring entries>}``.
@@ -144,14 +145,12 @@ class TinyfishSearchConfig(BaseSearchConfig):
         supported_perplexity: Final = _StrFrozenSet.validate_python(raw_supported)
         for param, value in optional_params.items():
             if param not in supported_perplexity and param not in request_data:
-                # `fetch` expects a JSON-encoded object on the wire; accept the
-                # natural Python dict form and serialize here so callers don't
-                # have to pre-stringify.
-                if isinstance(value, dict):
+                # Serialize dicts/lists as JSON so structured params survive urlencode.
+                if isinstance(value, (dict, list)):
                     value = json.dumps(value, separators=(",", ":"))
                 # `urlencode` would render Python bool as "True"/"False"
-                # (capitalized). ux-labs validators require lowercase
-                # "true"/"false" (e.g. `include_thumbnail`); normalize here.
+                # (capitalized). TinyFish Search's bool params require lowercase
+                # "true"/"false" strings on the wire; normalize here.
                 elif isinstance(value, bool):
                     value = "true" if value else "false"
                 request_data[param] = value
@@ -167,17 +166,35 @@ class TinyfishSearchConfig(BaseSearchConfig):
         """
         Transform a TinyFish response to LiteLLM's unified ``SearchResponse``.
 
-        Mappings (per-result):
-        - ``title`` → ``SearchResult.title`` (defaults to ``""`` if missing/null)
-        - ``url`` → ``SearchResult.url`` (defaults to ``""``)
-        - ``snippet`` → ``SearchResult.snippet`` (defaults to ``""``)
-        - all other per-result fields (``position``, ``site_name``,
-          ``thumbnail_url``, ``fetch``, ``fetch_error``, ...) ride through as
-          extras on ``SearchResult`` via its ``extra="allow"`` config.
+        Per-result field handling:
+        - ``title``, ``url``, ``snippet`` are declared on ``SearchResult`` and
+          populated by ``SearchResponse.model_validate`` when present. Missing
+          or ``None`` values are defaulted to ``""`` beforehand by
+          ``_default_missing_result_fields`` so a degraded result flows through
+          instead of failing the whole call.
+        - All undeclared per-result fields (``position``, ``site_name``, and
+          any others TinyFish returns) ride through as extras via
+          ``SearchResult``'s ``extra="allow"`` config — accessible as
+          attributes on the result object or enumerable via
+          ``result.model_extra``.
 
-        Top-level ``parameter_warnings`` (see ML-2085) is read when present and
-        each entry is re-fired via ``verbose_logger.warning``. Absent or
-        malformed entries are silently skipped — never throws.
+        Top-level ``parameter_warnings`` is read when present and each entry
+        is re-fired via ``verbose_logger.warning``. Absent or malformed
+        entries are silently skipped — never throws.
+
+        Top-level extras (``query``, ``total_results``, ``page``, and any
+        future TinyFish additions) ride through via
+        ``SearchResponse.extra="allow"``. The validated response is returned
+        in place after truncating ``results`` to the caller's ``max_results``,
+        so every field pydantic populated survives regardless of which
+        storage bucket (declared attribute or ``__pydantic_extra__``) holds it.
+
+        TinyFish response headers (e.g. ``x-request-id``, ``retry-after``,
+        ``x-ratelimit-limit`` — httpx normalizes header names to lowercase)
+        are stashed on ``response._hidden_params["headers"]`` (raw) and
+        ``response._hidden_params["additional_headers"]`` (sanitized via
+        ``process_response_headers``) so callers can correlate a search with
+        server-side logs.
 
         Error paths routed through ``self._wrap_error`` for uniform
         ``"TinyFish Search: <msg>. See <docs> for details."`` wrapping:
@@ -223,7 +240,12 @@ class TinyfishSearchConfig(BaseSearchConfig):
         _emit_parameter_warnings(parsed)
 
         max_results: Final = self._caller_max_results or _TINYFISH_RESULT_CAP
-        return SearchResponse(results=list(parsed.results[:max_results]))
+        parsed.results = parsed.results[:max_results]
+        raw_headers: Final = dict(raw_response.headers)
+        hidden: Final = parsed._hidden_params  # pyright: ignore[reportPrivateUsage]  # sole hidden-params channel
+        hidden["headers"] = raw_headers
+        hidden["additional_headers"] = process_response_headers(raw_headers)
+        return parsed
 
     def _wrap_error(
         self,
@@ -243,9 +265,9 @@ class TinyfishSearchConfig(BaseSearchConfig):
         carry the ``TinyFish Search:`` prefix — the bare error already names
         the host in the URL, so attribution is implicit there.
         """
-        # ux-labs frontend wraps every error body as {"error": {"code", "message", "details"?}}.
+        # TinyFish Search wraps every error body as {"error": {"code", "message", "details"?}}.
         # Best-effort unwrap to surface the inner message; fall back to the raw body
-        # for non-ux-labs responses (CDN HTML pages, other JSON envelopes, plain text).
+        # for other envelope shapes (CDN HTML pages, other JSON envelopes, plain text).
         inner_message = error_message
         try:
             body: Final[object] = json.loads(error_message)  # any-ok: json.loads -> Any
@@ -290,7 +312,7 @@ def _default_missing_result_fields(raw_json: object) -> None:
 
 
 def _emit_parameter_warnings(parsed: SearchResponse) -> None:
-    """Re-fire TinyFish-side ``parameter_warnings`` (see ML-2085) as warnings.
+    """Re-fire TinyFish-side ``parameter_warnings`` as warnings.
 
     Defensive: skip silently on any shape we don't recognize so a malformed
     entry (or an early/partial rollout of the field) never throws.

litellm/llms/valkey/common_utils.py

@@ -0,0 +1,18 @@
+"""Shared helpers for Valkey integrations (semantic cache, vector stores)."""
+
+import struct
+from collections.abc import Sequence
+from typing import Final
+from urllib.parse import quote
+
+
+def build_valkey_url(host: str, port: str, password: str | None = None, ssl: bool = False) -> str:
+    """Deliberately reads no environment: callers of the vector store control the
+    host, so an env-sourced password would be sent to a caller-chosen server."""
+    credentials: Final = f":{quote(password, safe='')}@" if password else ""
+    scheme: Final = "rediss" if ssl else "redis"
+    return f"{scheme}://{credentials}{host}:{port}"
+
+
+def pack_vector(embedding: Sequence[float]) -> bytes:
+    return struct.pack(f"<{len(embedding)}f", *embedding)

litellm/llms/valkey/vector_stores/__init__.py

@@ -0,0 +1,3 @@
+from litellm.llms.valkey.vector_stores.transformation import ValkeyVectorStoreConfig
+
+__all__ = ("ValkeyVectorStoreConfig",)

litellm/llms/valkey/vector_stores/transformation.py

@@ -0,0 +1,299 @@
+"""
+Valkey vector store provider.
+
+Valkey's vector search (the valkey-search module) speaks RESP only, no HTTP
+API, so this config extends BaseDirectVectorStoreConfig and executes the
+FT.SEARCH KNN query itself via redis-py instead of shaping an httpx request.
+Documents are HASHes indexed by an FT index named after the vector_store_id.
+"""
+
+from collections.abc import Awaitable, Callable, Mapping, Sequence
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final, NoReturn
+
+import httpx
+from pydantic import BaseModel, ConfigDict
+
+import litellm
+from litellm.llms.base_llm.vector_store.transformation import BaseDirectVectorStoreConfig
+from litellm.llms.valkey.common_utils import build_valkey_url, pack_vector
+from litellm.types.utils import EmbeddingResponse
+from litellm.types.vector_stores import (
+    VectorStoreCreateOptionalRequestParams,
+    VectorStoreResultContent,
+    VectorStoreSearchOptionalRequestParams,
+    VectorStoreSearchResponse,
+    VectorStoreSearchResult,
+)
+
+if TYPE_CHECKING:
+    from redis import Redis
+    from redis.asyncio import Redis as AsyncRedis
+    from redis.commands.search.document import Document
+    from redis.commands.search.query import Query
+    from redis.commands.search.result import Result
+
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+
+DEFAULT_VALKEY_PORT: Final = 6379
+DEFAULT_SOCKET_CONNECT_TIMEOUT_SECONDS: Final = 5.0
+DEFAULT_SOCKET_TIMEOUT_SECONDS: Final = 30.0
+DEFAULT_MAX_NUM_RESULTS: Final = 10
+MIN_MAX_NUM_RESULTS: Final = 1
+MAX_MAX_NUM_RESULTS: Final = 50
+DEFAULT_EMBEDDING_FIELD_NAME: Final = "embedding"
+DEFAULT_TEXT_FIELD_NAME: Final = "text"
+DISTANCE_FIELD_NAME: Final = "vector_distance"
+
+_EMPTY_EMBEDDING_CONFIG: Final = MappingProxyType({})
+_REDIS_INSTALL_HINT: Final = (
+    "The Valkey vector store requires the 'redis' package. Run 'pip install redis' to install it."
+)
+_SEARCH_ONLY_MESSAGE: Final = "Valkey vector store is search-only; create indexes with FT.CREATE directly"
+
+
+def _import_sync_redis() -> "type[Redis]":
+    try:
+        from redis import Redis as SyncRedisClient
+    except ImportError as e:
+        raise ValueError(_REDIS_INSTALL_HINT) from e
+    return SyncRedisClient
+
+
+def _import_async_redis() -> "type[AsyncRedis]":
+    try:
+        from redis.asyncio import Redis as AsyncRedisClient
+    except ImportError as e:
+        raise ValueError(_REDIS_INSTALL_HINT) from e
+    return AsyncRedisClient
+
+
+def _import_query() -> "type[Query]":
+    try:
+        from redis.commands.search.query import Query as RedisQuery
+    except ImportError as e:
+        raise ValueError(_REDIS_INSTALL_HINT) from e
+    return RedisQuery
+
+
+class _ValkeySearchParams(BaseModel):
+    """Typed view over the vector store's litellm_params; unrelated keys are ignored."""
+
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    litellm_embedding_model: str | None = None
+    litellm_embedding_config: Mapping[str, object] | None = None
+    valkey_host: str | None = None
+    valkey_port: int | None = None
+    valkey_password: str | None = None
+    valkey_ssl: bool | None = None
+    valkey_text_field: str | None = None
+    valkey_embedding_field: str | None = None
+
+    @property
+    def text_field(self) -> str:
+        return self.valkey_text_field or DEFAULT_TEXT_FIELD_NAME
+
+    @property
+    def embedding_field(self) -> str:
+        return self.valkey_embedding_field or DEFAULT_EMBEDDING_FIELD_NAME
+
+    def require_embedding_model(self) -> str:
+        if not self.litellm_embedding_model:
+            raise ValueError(
+                "litellm_embedding_model is required in litellm_params for the Valkey vector store. "
+                "Example: litellm_params['litellm_embedding_model'] = 'openai/text-embedding-3-small'"
+            )
+        return self.litellm_embedding_model
+
+    def connection_url(self) -> str:
+        if not self.valkey_host:
+            raise ValueError(
+                "valkey_host is required in litellm_params for the Valkey vector store. "
+                "Set it on the vector store's litellm_params, e.g. valkey_host: my-valkey.example.com"
+            )
+        return build_valkey_url(
+            host=self.valkey_host,
+            port=str(self.valkey_port or DEFAULT_VALKEY_PORT),
+            password=self.valkey_password,
+            ssl=bool(self.valkey_ssl),
+        )
+
+
+class ValkeyVectorStoreConfig(BaseDirectVectorStoreConfig):
+    def __init__(
+        self,
+        sync_client: "Redis | None" = None,
+        async_client: "AsyncRedis | None" = None,
+        embedding_fn: Callable[..., EmbeddingResponse] | None = None,
+        aembedding_fn: Callable[..., Awaitable[EmbeddingResponse]] | None = None,
+    ) -> None:
+        super().__init__()
+        self.sync_client = sync_client
+        self.async_client = async_client
+        self.embedding_fn = embedding_fn if embedding_fn is not None else litellm.embedding
+        self.aembedding_fn = aembedding_fn if aembedding_fn is not None else litellm.aembedding
+
+    @staticmethod
+    def _query_text(query: str | Sequence[str]) -> str:
+        if isinstance(query, str):
+            return query
+        if not query:
+            raise ValueError("query must not be empty")
+        return " ".join(query)
+
+    @staticmethod
+    def _socket_timeouts(timeout: float | httpx.Timeout | None) -> tuple[float, float]:
+        if isinstance(timeout, httpx.Timeout):
+            return (
+                timeout.connect or DEFAULT_SOCKET_CONNECT_TIMEOUT_SECONDS,
+                timeout.read or DEFAULT_SOCKET_TIMEOUT_SECONDS,
+            )
+        if timeout is not None:
+            return (min(float(timeout), DEFAULT_SOCKET_CONNECT_TIMEOUT_SECONDS), float(timeout))
+        return (DEFAULT_SOCKET_CONNECT_TIMEOUT_SECONDS, DEFAULT_SOCKET_TIMEOUT_SECONDS)
+
+    @staticmethod
+    def _knn_limit(vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams) -> int:
+        requested: Final = vector_store_search_optional_params.get("max_num_results")
+        if requested is None:
+            return DEFAULT_MAX_NUM_RESULTS
+        if not MIN_MAX_NUM_RESULTS <= requested <= MAX_MAX_NUM_RESULTS:
+            raise ValueError(
+                f"max_num_results must be between {MIN_MAX_NUM_RESULTS} and {MAX_MAX_NUM_RESULTS}, got {requested}"
+            )
+        return requested
+
+    @classmethod
+    def _knn_query(
+        cls,
+        vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams,
+        embedding_field: str,
+        text_field: str,
+    ) -> "Query":
+        if vector_store_search_optional_params.get("filters") is not None:
+            raise ValueError("Valkey vector store does not support the filters parameter yet")
+        k: Final = cls._knn_limit(vector_store_search_optional_params)
+        query_cls: Final = _import_query()
+        knn_expr: Final = f"*=>[KNN {k} @{embedding_field} $vec AS {DISTANCE_FIELD_NAME}]"
+        # valkey-search rejects SORTBY on the KNN distance alias, so results are
+        # re-ordered client-side in _to_response instead.
+        return query_cls(knn_expr).return_fields(text_field, DISTANCE_FIELD_NAME).paging(0, k).dialect(2)
+
+    @staticmethod
+    def _to_result(doc: "Document", text_field: str) -> VectorStoreSearchResult:
+        content: Final = [  # mutable-ok: VectorStoreSearchResult declares a list of content parts
+            VectorStoreResultContent(text=str(getattr(doc, text_field, "")), type="text")
+        ]
+        return VectorStoreSearchResult(
+            score=1.0 - float(getattr(doc, DISTANCE_FIELD_NAME)),
+            content=content,
+            file_id=getattr(doc, "id", None),
+            filename=getattr(doc, "id", None),
+        )
+
+    @classmethod
+    def _to_response(cls, search_result: "Result", query_text: str, text_field: str) -> VectorStoreSearchResponse:
+        docs: Final = getattr(search_result, "docs", None) or ()
+        data: Final = sorted(
+            (cls._to_result(doc, text_field) for doc in docs),
+            key=lambda result: result.get("score") or 0.0,
+            reverse=True,
+        )
+        return VectorStoreSearchResponse(
+            object="vector_store.search_results.page",
+            search_query=query_text,
+            data=data,
+        )
+
+    def execute_search_vector_store_request(
+        self,
+        vector_store_id: str,
+        query: str | Sequence[str],
+        vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams,
+        litellm_logging_obj: "LiteLLMLoggingObj",
+        litellm_params: Mapping[str, object],
+        timeout: float | httpx.Timeout | None = None,
+    ) -> VectorStoreSearchResponse:
+        params: Final = _ValkeySearchParams.model_validate(litellm_params)
+        query_text: Final = self._query_text(query)
+        knn: Final = self._knn_query(
+            vector_store_search_optional_params,
+            embedding_field=params.embedding_field,
+            text_field=params.text_field,
+        )
+        embedding_response: Final = self.embedding_fn(
+            model=params.require_embedding_model(),
+            input=[query_text],  # mutable-ok: litellm.embedding's input contract is a list
+            **(params.litellm_embedding_config or _EMPTY_EMBEDDING_CONFIG),
+        )
+        vec_params: Final = {"vec": pack_vector(embedding_response.data[0]["embedding"])}  # mutable-ok: redis-py API
+
+        if self.sync_client is not None:
+            raw: Final = self.sync_client.ft(vector_store_id).search(knn, query_params=vec_params)
+            return self._to_response(raw, query_text, params.text_field)
+
+        connect_timeout, op_timeout = self._socket_timeouts(timeout)
+        client: Final = _import_sync_redis().from_url(
+            params.connection_url(),
+            socket_connect_timeout=connect_timeout,
+            socket_timeout=op_timeout,
+        )
+        try:
+            raw_result: Final = client.ft(vector_store_id).search(knn, query_params=vec_params)
+            return self._to_response(raw_result, query_text, params.text_field)
+        finally:
+            client.close()
+
+    async def aexecute_search_vector_store_request(
+        self,
+        vector_store_id: str,
+        query: str | Sequence[str],
+        vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams,
+        litellm_logging_obj: "LiteLLMLoggingObj",
+        litellm_params: Mapping[str, object],
+        timeout: float | httpx.Timeout | None = None,
+    ) -> VectorStoreSearchResponse:
+        params: Final = _ValkeySearchParams.model_validate(litellm_params)
+        query_text: Final = self._query_text(query)
+        knn: Final = self._knn_query(
+            vector_store_search_optional_params,
+            embedding_field=params.embedding_field,
+            text_field=params.text_field,
+        )
+        embedding_response: Final = await self.aembedding_fn(
+            model=params.require_embedding_model(),
+            input=[query_text],  # mutable-ok: litellm.embedding's input contract is a list
+            **(params.litellm_embedding_config or _EMPTY_EMBEDDING_CONFIG),
+        )
+        vec_params: Final = {"vec": pack_vector(embedding_response.data[0]["embedding"])}  # mutable-ok: redis-py API
+
+        if self.async_client is not None:
+            raw: Final = await self.async_client.ft(vector_store_id).search(  # pyright: ignore[reportGeneralTypeIssues]  # types-redis 4.6 stubs shadow redis 5.3.1 and type the async client's ft() as the sync Search, so search() returns a non-awaitable Result; it is a coroutine at runtime
+                knn, query_params=vec_params
+            )
+            return self._to_response(raw, query_text, params.text_field)
+
+        connect_timeout, op_timeout = self._socket_timeouts(timeout)
+        client: Final = _import_async_redis().from_url(
+            params.connection_url(),
+            socket_connect_timeout=connect_timeout,
+            socket_timeout=op_timeout,
+        )
+        try:
+            raw_result: Final = await client.ft(vector_store_id).search(  # pyright: ignore[reportGeneralTypeIssues]  # types-redis 4.6 stubs shadow redis 5.3.1 and type the async client's ft() as the sync Search, so search() returns a non-awaitable Result; it is a coroutine at runtime
+                knn, query_params=vec_params
+            )
+            return self._to_response(raw_result, query_text, params.text_field)
+        finally:
+            await client.aclose()
+
+    def transform_create_vector_store_request(
+        self,
+        vector_store_create_optional_params: VectorStoreCreateOptionalRequestParams,
+        api_base: str,
+    ) -> NoReturn:
+        raise NotImplementedError(_SEARCH_ONLY_MESSAGE)
+
+    def transform_create_vector_store_response(self, response: httpx.Response) -> NoReturn:
+        raise NotImplementedError(_SEARCH_ONLY_MESSAGE)

litellm/main.py

@@ -420,6 +420,8 @@ async def acompletion(
     verbosity: Literal["low", "medium", "high"] | None = None,
     safety_identifier: str | None = None,
     service_tier: str | None = None,
+    store: bool | None = None,
+    prompt_cache_key: str | None = None,
     # set api_base, api_version, api_key
     base_url: str | None = None,
     api_version: str | None = None,
@@ -585,6 +587,8 @@ async def acompletion(
         "verbosity": verbosity,
         "safety_identifier": safety_identifier,
         "service_tier": service_tier,
+        "store": store,
+        "prompt_cache_key": prompt_cache_key,
         "extra_headers": extra_headers,
         "acompletion": True,  # assuming this is a required parameter
         "thinking": thinking,
@@ -4930,6 +4934,8 @@ def completion(
     extra_headers: dict | None = None,
     safety_identifier: str | None = None,
     service_tier: str | None = None,
+    store: bool | None = None,
+    prompt_cache_key: str | None = None,
     # soon to be deprecated params by OpenAI
     functions: list | None = None,
     function_call: str | None = None,
@@ -5058,6 +5064,8 @@ def completion(
                 verbosity=verbosity,
                 safety_identifier=safety_identifier,
                 service_tier=service_tier,
+                store=store,
+                prompt_cache_key=prompt_cache_key,
                 base_url=base_url,
                 api_version=api_version,
                 api_key=api_key,
@@ -5367,6 +5375,8 @@ def completion(
             ),
             "safety_identifier": safety_identifier,
             "service_tier": service_tier,
+            "store": store,
+            "prompt_cache_key": prompt_cache_key,
             "allowed_openai_params": kwargs.get("allowed_openai_params"),
             "base_model": base_model,
         }

litellm/model_prices_and_context_window_backup.json

@@ -10052,6 +10052,21 @@
         "output_cost_per_second": 0.0066027,
         "supports_tool_choice": true
     },
+    "bedrock/guardrails": {
+        "guardrail_cost_per_unit": {
+            "automatedReasoningPolicyUnits": 0.00017,
+            "contentPolicyImageUnits": 0.00075,
+            "contentPolicyUnits": 0.00015,
+            "contextualGroundingPolicyUnits": 0.0001,
+            "sensitiveInformationPolicyFreeUnits": 0.0,
+            "sensitiveInformationPolicyUnits": 0.0001,
+            "topicPolicyUnits": 0.00015,
+            "wordPolicyUnits": 0.0
+        },
+        "litellm_provider": "bedrock",
+        "mode": "guardrail",
+        "source": "https://aws.amazon.com/bedrock/pricing/"
+    },
     "bedrock/ap-northeast-1/1-month-commitment/anthropic.claude-instant-v1": {
         "input_cost_per_second": 0.01475,
         "litellm_provider": "bedrock",
@@ -14441,6 +14456,25 @@
         "supports_tool_choice": true,
         "supports_output_config": true
     },
+    "databricks/databricks-claude-opus-4-6": {
+        "input_cost_per_token": 5.00003e-06,
+        "input_dbu_cost_per_token": 7.1429e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 2.5000010000000002e-05,
+        "output_dbu_cost_per_token": 0.000357143,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+        "supports_assistant_prefill": true,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },
     "databricks/databricks-claude-sonnet-4": {
         "input_cost_per_token": 2.9999900000000002e-06,
         "input_dbu_cost_per_token": 4.2857e-05,
@@ -14498,6 +14532,25 @@
         "supports_reasoning": true,
         "supports_tool_choice": true
     },
+    "databricks/databricks-claude-sonnet-4-6": {
+        "input_cost_per_token": 2.9999900000000002e-06,
+        "input_dbu_cost_per_token": 4.2857e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 64000,
+        "max_tokens": 64000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.5000020000000002e-05,
+        "output_dbu_cost_per_token": 0.000214286,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+        "supports_assistant_prefill": true,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },
     "databricks/databricks-gemini-2-5-flash": {
         "input_cost_per_token": 3.0001999999999996e-07,
         "input_dbu_cost_per_token": 4.285999999999999e-06,
@@ -14532,6 +14585,74 @@
         "supports_function_calling": true,
         "supports_tool_choice": true
     },
+    "databricks/databricks-gemini-3-1-flash-lite": {
+        "input_cost_per_token": 3.1248e-07,
+        "input_dbu_cost_per_token": 4.464e-06,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.87502e-06,
+        "output_dbu_cost_per_token": 2.6786e-05,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+        "supports_function_calling": true,
+        "supports_tool_choice": true
+    },
+    "databricks/databricks-gemini-3-1-pro": {
+        "input_cost_per_token": 2.49998e-06,
+        "input_dbu_cost_per_token": 3.5714e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.5000020000000002e-05,
+        "output_dbu_cost_per_token": 0.000214286,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+        "supports_function_calling": true,
+        "supports_tool_choice": true
+    },
+    "databricks/databricks-gemini-3-flash": {
+        "input_cost_per_token": 6.2503e-07,
+        "input_dbu_cost_per_token": 8.929e-06,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 3.74997e-06,
+        "output_dbu_cost_per_token": 5.3571e-05,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+        "supports_function_calling": true,
+        "supports_tool_choice": true
+    },
+    "databricks/databricks-gemini-3-pro": {
+        "input_cost_per_token": 2.49998e-06,
+        "input_dbu_cost_per_token": 3.5714e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.5000020000000002e-05,
+        "output_dbu_cost_per_token": 0.000214286,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+        "supports_function_calling": true,
+        "supports_tool_choice": true
+    },
     "databricks/databricks-gemma-3-12b": {
         "input_cost_per_token": 1.5000999999999998e-07,
         "input_dbu_cost_per_token": 2.1429999999999996e-06,
@@ -14577,6 +14698,126 @@
         "output_dbu_cost_per_token": 0.000142857,
         "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
     },
+    "databricks/databricks-gpt-5-1-codex-max": {
+        "input_cost_per_token": 1.24999e-06,
+        "input_dbu_cost_per_token": 1.7857e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 9.999990000000002e-06,
+        "output_dbu_cost_per_token": 0.000142857,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
+    "databricks/databricks-gpt-5-1-codex-mini": {
+        "input_cost_per_token": 2.4997e-07,
+        "input_dbu_cost_per_token": 3.571e-06,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.99997e-06,
+        "output_dbu_cost_per_token": 2.8571e-05,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
+    "databricks/databricks-gpt-5-2": {
+        "input_cost_per_token": 1.75e-06,
+        "input_dbu_cost_per_token": 2.5e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.4e-05,
+        "output_dbu_cost_per_token": 0.0002,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
+    "databricks/databricks-gpt-5-2-codex": {
+        "input_cost_per_token": 1.75e-06,
+        "input_dbu_cost_per_token": 2.5e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.4e-05,
+        "output_dbu_cost_per_token": 0.0002,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
+    "databricks/databricks-gpt-5-3-codex": {
+        "input_cost_per_token": 1.75e-06,
+        "input_dbu_cost_per_token": 2.5e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.4e-05,
+        "output_dbu_cost_per_token": 0.0002,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
+    "databricks/databricks-gpt-5-4": {
+        "input_cost_per_token": 2.49998e-06,
+        "input_dbu_cost_per_token": 3.5714e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.5000020000000002e-05,
+        "output_dbu_cost_per_token": 0.000214286,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
+    "databricks/databricks-gpt-5-4-mini": {
+        "input_cost_per_token": 7.4998e-07,
+        "input_dbu_cost_per_token": 1.0714e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 4.50002e-06,
+        "output_dbu_cost_per_token": 6.4286e-05,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
+    "databricks/databricks-gpt-5-4-nano": {
+        "input_cost_per_token": 1.9999e-07,
+        "input_dbu_cost_per_token": 2.857e-06,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.24999e-06,
+        "output_dbu_cost_per_token": 1.7857e-05,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
     "databricks/databricks-gpt-5-mini": {
         "input_cost_per_token": 2.4997000000000006e-07,
         "input_dbu_cost_per_token": 3.571e-06,
@@ -19424,20 +19665,20 @@
         "web_search_billing_unit": "per_query"
     },
     "vertex_ai/gemini-3.6-flash": {
-        "cache_read_input_token_cost": 1.5e-07,
-        "cache_read_input_token_cost_flex": 7.5e-08,
-        "input_cost_per_token": 1.5e-06,
-        "input_cost_per_token_batches": 7.5e-07,
-        "input_cost_per_token_flex": 7.5e-07,
+        "cache_read_input_token_cost": 7.5e-08,
+        "cache_read_input_token_cost_flex": 3.75e-08,
+        "input_cost_per_token": 7.5e-07,
+        "input_cost_per_token_batches": 3.75e-07,
+        "input_cost_per_token_flex": 3.75e-07,
         "litellm_provider": "vertex_ai",
         "max_input_tokens": 1048576,
         "max_output_tokens": 65536,
         "max_tokens": 65536,
         "mode": "chat",
-        "output_cost_per_reasoning_token": 7.5e-06,
-        "output_cost_per_token": 7.5e-06,
-        "output_cost_per_token_batches": 3.75e-06,
-        "output_cost_per_token_flex": 3.75e-06,
+        "output_cost_per_reasoning_token": 3.75e-06,
+        "output_cost_per_token": 3.75e-06,
+        "output_cost_per_token_batches": 1.875e-06,
+        "output_cost_per_token_flex": 1.875e-06,
         "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
         "supported_endpoints": [
             "/v1/chat/completions",
@@ -19467,9 +19708,9 @@
         "supports_vision": true,
         "supports_web_search": true,
         "supports_native_streaming": true,
-        "input_cost_per_token_priority": 2.7e-06,
-        "output_cost_per_token_priority": 1.35e-05,
-        "cache_read_input_token_cost_priority": 2.7e-07,
+        "input_cost_per_token_priority": 1.35e-06,
+        "output_cost_per_token_priority": 6.75e-06,
+        "cache_read_input_token_cost_priority": 1.35e-07,
         "search_context_cost_per_query": {
             "search_context_size_low": 0.014,
             "search_context_size_medium": 0.014,
@@ -21150,20 +21391,20 @@
         "web_search_billing_unit": "per_query"
     },
     "gemini/gemini-3.6-flash": {
-        "cache_read_input_token_cost": 1.5e-07,
-        "cache_read_input_token_cost_flex": 7.5e-08,
-        "input_cost_per_token": 1.5e-06,
-        "input_cost_per_token_batches": 7.5e-07,
-        "input_cost_per_token_flex": 7.5e-07,
+        "cache_read_input_token_cost": 7.5e-08,
+        "cache_read_input_token_cost_flex": 3.75e-08,
+        "input_cost_per_token": 7.5e-07,
+        "input_cost_per_token_batches": 3.75e-07,
+        "input_cost_per_token_flex": 3.75e-07,
         "litellm_provider": "gemini",
         "max_input_tokens": 1048576,
         "max_output_tokens": 65536,
         "max_tokens": 65536,
         "mode": "chat",
-        "output_cost_per_reasoning_token": 7.5e-06,
-        "output_cost_per_token": 7.5e-06,
-        "output_cost_per_token_batches": 3.75e-06,
-        "output_cost_per_token_flex": 3.75e-06,
+        "output_cost_per_reasoning_token": 3.75e-06,
+        "output_cost_per_token": 3.75e-06,
+        "output_cost_per_token_batches": 1.875e-06,
+        "output_cost_per_token_flex": 1.875e-06,
         "rpm": 2000,
         "source": "https://ai.google.dev/pricing/gemini-3",
         "supported_endpoints": [
@@ -21196,9 +21437,9 @@
         "supports_web_search": true,
         "supports_native_streaming": true,
         "tpm": 800000,
-        "input_cost_per_token_priority": 2.7e-06,
-        "output_cost_per_token_priority": 1.35e-05,
-        "cache_read_input_token_cost_priority": 2.7e-07,
+        "input_cost_per_token_priority": 1.35e-06,
+        "output_cost_per_token_priority": 6.75e-06,
+        "cache_read_input_token_cost_priority": 1.35e-07,
         "search_context_cost_per_query": {
             "search_context_size_low": 0.014,
             "search_context_size_medium": 0.014,
@@ -21544,20 +21785,20 @@
         "web_search_billing_unit": "per_query"
     },
     "gemini-3.6-flash": {
-        "cache_read_input_token_cost": 1.5e-07,
-        "cache_read_input_token_cost_flex": 7.5e-08,
-        "input_cost_per_token": 1.5e-06,
-        "input_cost_per_token_batches": 7.5e-07,
-        "input_cost_per_token_flex": 7.5e-07,
+        "cache_read_input_token_cost": 7.5e-08,
+        "cache_read_input_token_cost_flex": 3.75e-08,
+        "input_cost_per_token": 7.5e-07,
+        "input_cost_per_token_batches": 3.75e-07,
+        "input_cost_per_token_flex": 3.75e-07,
         "litellm_provider": "vertex_ai-language-models",
         "max_input_tokens": 1048576,
         "max_output_tokens": 65536,
         "max_tokens": 65536,
         "mode": "chat",
-        "output_cost_per_reasoning_token": 7.5e-06,
-        "output_cost_per_token": 7.5e-06,
-        "output_cost_per_token_batches": 3.75e-06,
-        "output_cost_per_token_flex": 3.75e-06,
+        "output_cost_per_reasoning_token": 3.75e-06,
+        "output_cost_per_token": 3.75e-06,
+        "output_cost_per_token_batches": 1.875e-06,
+        "output_cost_per_token_flex": 1.875e-06,
         "source": "https://ai.google.dev/pricing/gemini-3",
         "supported_endpoints": [
             "/v1/chat/completions",
@@ -21588,9 +21829,9 @@
         "supports_vision": true,
         "supports_web_search": true,
         "supports_native_streaming": true,
-        "input_cost_per_token_priority": 2.7e-06,
-        "output_cost_per_token_priority": 1.35e-05,
-        "cache_read_input_token_cost_priority": 2.7e-07,
+        "input_cost_per_token_priority": 1.35e-06,
+        "output_cost_per_token_priority": 6.75e-06,
+        "cache_read_input_token_cost_priority": 1.35e-07,
         "search_context_cost_per_query": {
             "search_context_size_low": 0.014,
             "search_context_size_medium": 0.014,

litellm/ocr/main.py

@@ -21,7 +21,12 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
 from litellm.llms.azure_ai.ocr.common_utils import (
     is_azure_document_intelligence_model,
 )
-from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig, OCRResponse
+from litellm.llms.base_llm.ocr.transformation import (
+    OCR_REQUEST_FORMAT_PARAM,
+    BaseOCRConfig,
+    OCRResponse,
+    parse_ocr_request_format,
+)
 from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
 from litellm.rust_bridge import ocr as rust_ocr_bridge
 from litellm.types.router import GenericLiteLLMParams
@@ -124,6 +129,24 @@ def _prepare_ocr_request(
     litellm_params: Final = GenericLiteLLMParams.model_validate(kwargs)
 
     supported_params: Final = ocr_provider_config.get_supported_ocr_params(model=model)
+    requested_format: Final = kwargs.get(OCR_REQUEST_FORMAT_PARAM)
+    if requested_format is not None:
+        try:
+            parsed_format: Final = parse_ocr_request_format(requested_format)
+        except ValueError as e:
+            raise litellm.exceptions.UnsupportedParamsError(
+                message=f"{e}", model=model, llm_provider=custom_llm_provider
+            ) from e
+        if OCR_REQUEST_FORMAT_PARAM not in supported_params and parsed_format == "native":
+            raise litellm.exceptions.UnsupportedParamsError(
+                message=(
+                    f"`{OCR_REQUEST_FORMAT_PARAM}='native'` is not supported for provider: {custom_llm_provider}, "
+                    f"model: {model}"
+                ),
+                model=model,
+                llm_provider=custom_llm_provider,
+            )
+
     non_default_params: Final = {}
     for param in supported_params:
         if param in kwargs:
@@ -166,6 +189,8 @@ def _prepare_ocr_request(
 
 
 def _rust_ocr_supported(prepared_request: _PreparedOCRRequest) -> bool:
+    if prepared_request.optional_params.get(OCR_REQUEST_FORMAT_PARAM) == "native":
+        return False
     return prepared_request.custom_llm_provider in _RUST_OCR_PROVIDERS
 
 

litellm/proxy/_experimental/mcp_server/db.py

@@ -4,7 +4,7 @@ import hashlib
 import json
 from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
 from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING, Any, Final, TypedDict, cast
+from typing import TYPE_CHECKING, Any, Final, Protocol, TypedDict, TypeVar, cast
 
 from litellm._logging import verbose_proxy_logger
 from litellm._uuid import uuid
@@ -45,10 +45,47 @@ from litellm.types.mcp import MCPCredentials
 if TYPE_CHECKING:
     from prisma import models as prisma_db_models
     from prisma import types as prisma_db_types
-    from prisma.actions import LiteLLM_MCPUserCredentialsActions, LiteLLM_MCPUserEnvVarsActions
 
     from litellm.types.mcp_server.mcp_server_manager import MCPServer
 
+_RowT = TypeVar("_RowT")
+
+
+class _TableActions(Protocol[_RowT]):
+    async def find_unique(
+        self, where: Mapping[str, object], include: Mapping[str, object] | None = None
+    ) -> _RowT | None: ...
+
+    async def find_many(
+        self,
+        take: int | None = None,
+        where: Mapping[str, object] | None = None,
+        order: Mapping[str, object] | None = None,
+    ) -> list[_RowT]: ...
+
+    async def create(self, data: Mapping[str, object]) -> _RowT: ...
+
+    async def upsert(self, where: Mapping[str, object], data: Mapping[str, object]) -> _RowT: ...
+
+    async def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> _RowT | None: ...
+
+    async def delete(self, where: Mapping[str, object]) -> _RowT | None: ...
+
+    async def delete_many(self, where: Mapping[str, object] | None = None) -> int: ...
+
+
+class _UserEnvVarsTransactionClient(Protocol):
+    litellm_mcpuserenvvars: "_TableActions[prisma_db_models.LiteLLM_MCPUserEnvVars]"
+
+    async def execute_raw(self, query: str, *args: object) -> int: ...
+
+
+class _UserEnvVarsTransaction(Protocol):
+    async def __aenter__(self) -> _UserEnvVarsTransactionClient: ...
+
+    async def __aexit__(self, exc_type: object, exc_value: object, traceback: object) -> bool | None: ...
+
+
 _AUTH_FLOW_SCOPED_FIELDS: Final["frozenset[str]"] = frozenset(
     {
         "issuer",
@@ -434,23 +471,54 @@ def _credentials_blob_to_mutable_dict(blob: str | Mapping[str, object]) -> dict[
     return parsed_blob
 
 
+def _mcp_server_table_actions(
+    prisma_client: PrismaClient,
+) -> "_TableActions[prisma_db_models.LiteLLM_MCPServerTable]":
+    table: Final[_TableActions[prisma_db_models.LiteLLM_MCPServerTable]] = MCPServerRepository(prisma_client).table
+    return table
+
+
+def _verification_token_table_actions(
+    prisma_client: PrismaClient,
+) -> "_TableActions[prisma_db_models.LiteLLM_VerificationToken]":
+    table: Final[_TableActions[prisma_db_models.LiteLLM_VerificationToken]] = VerificationTokenRepository(
+        prisma_client
+    ).table
+    return table
+
+
+def _team_table_actions(
+    prisma_client: PrismaClient,
+) -> "_TableActions[prisma_db_models.LiteLLM_TeamTable]":
+    table: Final[_TableActions[prisma_db_models.LiteLLM_TeamTable]] = TeamRepository(prisma_client).table
+    return table
+
+
+def _oauth_client_table_actions(
+    prisma_client: PrismaClient,
+) -> "_TableActions[prisma_db_models.LiteLLM_MCPServerOAuthClient]":
+    table: Final[_TableActions[prisma_db_models.LiteLLM_MCPServerOAuthClient]] = MCPServerOAuthClientRepository(
+        prisma_client
+    ).table
+    return table
+
+
+def _db_transaction_manager(prisma_client: PrismaClient) -> _UserEnvVarsTransaction:
+    manager: Final[_UserEnvVarsTransaction] = prisma_client.db.tx()
+    return manager
+
+
 async def _db_find_mcp_server_rows(
     prisma_client: PrismaClient,
     where: "prisma_db_types.LiteLLM_MCPServerTableWhereInput | None" = None,
 ) -> "list[prisma_db_models.LiteLLM_MCPServerTable]":
-    rows: list[prisma_db_models.LiteLLM_MCPServerTable] = await MCPServerRepository(prisma_client).table.find_many(
-        where=where
-    )
-    return rows
+    return await _mcp_server_table_actions(prisma_client).find_many(where=where)
 
 
 async def _db_find_mcp_server_row(
     prisma_client: PrismaClient, server_id: str
 ) -> "prisma_db_models.LiteLLM_MCPServerTable | None":
-    row: prisma_db_models.LiteLLM_MCPServerTable | None = await MCPServerRepository(prisma_client).table.find_unique(
-        where={"server_id": server_id}
-    )
-    return row
+    return await _mcp_server_table_actions(prisma_client).find_unique(where={"server_id": server_id})
 
 
 async def _db_update_mcp_server_row(
@@ -467,19 +535,17 @@ async def _db_update_mcp_server_row(
 
 def _user_credential_actions(
     prisma_client: PrismaClient,
-) -> "LiteLLM_MCPUserCredentialsActions[prisma_db_models.LiteLLM_MCPUserCredentials]":
-    table: Final[LiteLLM_MCPUserCredentialsActions[prisma_db_models.LiteLLM_MCPUserCredentials]] = (
-        MCPUserCredentialsRepository(prisma_client).table
-    )
+) -> "_TableActions[prisma_db_models.LiteLLM_MCPUserCredentials]":
+    table: Final[_TableActions[prisma_db_models.LiteLLM_MCPUserCredentials]] = MCPUserCredentialsRepository(
+        prisma_client
+    ).table
     return table
 
 
 def _user_env_var_actions(
     prisma_client: PrismaClient,
-) -> "LiteLLM_MCPUserEnvVarsActions[prisma_db_models.LiteLLM_MCPUserEnvVars]":
-    table: Final[LiteLLM_MCPUserEnvVarsActions[prisma_db_models.LiteLLM_MCPUserEnvVars]] = (
-        prisma_client.db.litellm_mcpuserenvvars
-    )
+) -> "_TableActions[prisma_db_models.LiteLLM_MCPUserEnvVars]":
+    table: Final[_TableActions[prisma_db_models.LiteLLM_MCPUserEnvVars]] = prisma_client.db.litellm_mcpuserenvvars
     return table
 
 
@@ -501,7 +567,7 @@ async def _db_find_user_credential_rows(
 async def _db_upsert_user_credential_row(
     prisma_client: PrismaClient, user_id: str, server_id: str, credential_b64: str
 ) -> None:
-    await MCPUserCredentialsRepository(prisma_client).table.upsert(
+    await _user_credential_actions(prisma_client).upsert(
         where={"user_id_server_id": {"user_id": user_id, "server_id": server_id}},
         data={
             "create": {
@@ -592,9 +658,9 @@ async def get_mcp_servers(prisma_client: PrismaClient, server_ids: Iterable[str]
     """
     Returns the matching mcp servers from the db with the server_ids
     """
-    _mcp_servers: Final[list[prisma_db_models.LiteLLM_MCPServerTable]] = await MCPServerRepository(
+    _mcp_servers: Final[list[prisma_db_models.LiteLLM_MCPServerTable]] = await _mcp_server_table_actions(
         prisma_client
-    ).table.find_many(
+    ).find_many(
         where={
             "server_id": {"in": server_ids},
         }
@@ -612,9 +678,9 @@ async def get_mcp_servers_by_verificationtoken(prisma_client: PrismaClient, toke
     """
     Returns the mcp servers from the db for the verification token
     """
-    verification_token_record: prisma_db_models.LiteLLM_VerificationToken | None = await VerificationTokenRepository(
-        prisma_client
-    ).table.find_unique(
+    verification_token_record: (
+        prisma_db_models.LiteLLM_VerificationToken | None
+    ) = await _verification_token_table_actions(prisma_client).find_unique(
         where={
             "token": token,
         },
@@ -633,7 +699,7 @@ async def get_mcp_servers_by_team(prisma_client: PrismaClient, team_id: str) ->
     """
     Returns the mcp servers from the db for the team id
     """
-    team_record: prisma_db_models.LiteLLM_TeamTable | None = await TeamRepository(prisma_client).table.find_unique(
+    team_record: prisma_db_models.LiteLLM_TeamTable | None = await _team_table_actions(prisma_client).find_unique(
         where={
             "team_id": team_id,
         },
@@ -760,9 +826,9 @@ async def delete_mcp_server(
     if deleted_server is not None:
         credential_user_ids: list[str] = []
         try:
-            credential_rows: Sequence[
-                prisma_db_models.LiteLLM_MCPUserCredentials
-            ] = await prisma_client.db.litellm_mcpusercredentials.find_many(where={"server_id": server_id})
+            credential_rows: Sequence[prisma_db_models.LiteLLM_MCPUserCredentials] = await _user_credential_actions(
+                prisma_client
+            ).find_many(where={"server_id": server_id})
             credential_user_ids = [row.user_id for row in credential_rows]
         except Exception as e:  # noqa: BLE001 - enumeration is best-effort; cached tokens expire by TTL
             verbose_proxy_logger.warning(
@@ -771,9 +837,9 @@ async def delete_mcp_server(
                 e,
             )
         for model, label in (
-            (prisma_client.db.litellm_mcpusercredentials, "credential"),
-            (prisma_client.db.litellm_mcpuserenvvars, "env var"),
-            (prisma_client.db.litellm_mcpserveroauthclient, "OAuth client"),
+            (_user_credential_actions(prisma_client), "credential"),
+            (_user_env_var_actions(prisma_client), "env var"),
+            (_oauth_client_table_actions(prisma_client), "OAuth client"),
         ):
             try:
                 await model.delete_many(where={"server_id": server_id})
@@ -1042,9 +1108,9 @@ async def get_mcp_server_oauth_client_credentials(prisma_client: PrismaClient, s
     LiteLLM_MCPServerTable row, so their dynamically registered client lives here keyed
     by server_id. The returned value is the raw credentials blob for
     ``_get_persisted_dcr_credentials`` to parse."""
-    row: Final[prisma_db_models.LiteLLM_MCPServerOAuthClient | None] = await MCPServerOAuthClientRepository(
+    row: Final[prisma_db_models.LiteLLM_MCPServerOAuthClient | None] = await _oauth_client_table_actions(
         prisma_client
-    ).table.find_unique(where={"server_id": server_id})
+    ).find_unique(where={"server_id": server_id})
     if row is None:
         return None
     return row.credentials
@@ -1062,7 +1128,7 @@ async def upsert_mcp_server_oauth_client_credentials(
 
     encrypted: Final = encrypt_credentials(credentials=MCPCredentials(**credentials), encryption_key=_get_salt_key())
     blob: Final = safe_dumps(encrypted)
-    await MCPServerOAuthClientRepository(prisma_client).table.upsert(
+    await _oauth_client_table_actions(prisma_client).upsert(
         where={"server_id": server_id},
         data={
             "create": {"server_id": server_id, "credentials": blob},
@@ -1109,21 +1175,21 @@ async def rotate_mcp_server_credentials_master_key(prisma_client: PrismaClient,
             continue
 
         update_data["updated_by"] = touched_by
-        await MCPServerRepository(prisma_client).table.update(
+        await _mcp_server_table_actions(prisma_client).update(
             where={"server_id": mcp_server.server_id},
             data=update_data,
         )
         updated += 1
 
-    oauth_clients: Final[list[prisma_db_models.LiteLLM_MCPServerOAuthClient]] = await MCPServerOAuthClientRepository(
+    oauth_clients: Final[list[prisma_db_models.LiteLLM_MCPServerOAuthClient]] = await _oauth_client_table_actions(
         prisma_client
-    ).table.find_many()
+    ).find_many()
     oauth_updated = 0
     for oauth_client in oauth_clients:
         rotated_credentials = _reencrypt_mcp_credentials_blob(oauth_client.credentials, new_master_key)
         if rotated_credentials is None:
             continue
-        await MCPServerOAuthClientRepository(prisma_client).table.update(
+        await _oauth_client_table_actions(prisma_client).update(
             where={"server_id": oauth_client.server_id},
             data={"credentials": rotated_credentials},
         )
@@ -1813,7 +1879,9 @@ async def get_mcp_submissions(
     along with a summary count breakdown by approval_status.
     Mirrors get_guardrail_submissions() from guardrail_endpoints.py.
     """
-    rows: list[prisma_db_models.LiteLLM_MCPServerTable] = await MCPServerRepository(prisma_client).table.find_many(
+    rows: Final[list[prisma_db_models.LiteLLM_MCPServerTable]] = await _mcp_server_table_actions(
+        prisma_client
+    ).find_many(
         where={"submitted_at": {"not": None}},
         order={"submitted_at": "desc"},
         take=500,  # safety cap; paginate if needed in a future iteration
@@ -1915,7 +1983,7 @@ async def merge_user_env_vars(
         "big",
         signed=True,
     )
-    async with prisma_client.db.tx() as tx:
+    async with _db_transaction_manager(prisma_client) as tx:
         await tx.execute_raw("SELECT pg_advisory_xact_lock($1::bigint)", lock_key)
         row: Final[prisma_db_models.LiteLLM_MCPUserEnvVars | None] = await tx.litellm_mcpuserenvvars.find_unique(
             where={"user_id_server_id": {"user_id": user_id, "server_id": server_id}}

litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py

@@ -1341,7 +1341,7 @@ async def _persist_dcr_client_registration(
     ``update_mcp_server`` merges credential blobs: a re-registered public client must not
     inherit the previous client's secret or auth method.
     """
-    if mcp_server.is_true_passthrough or mcp_server.is_oauth_delegate:
+    if mcp_server.is_client_forwarded_token:
         return "skipped"
 
     try:
@@ -1678,10 +1678,17 @@ async def authorize(
     lookup_name: Final[str | None] = mcp_server_name or client_id
     client_ip: Final = IPAddressUtils.get_mcp_client_ip(request)
     mcp_server = (
-        global_mcp_server_manager.get_mcp_server_by_name(lookup_name, client_ip=client_ip) if lookup_name else None
+        await global_mcp_server_manager.get_resolved_mcp_server_by_name(lookup_name, client_ip=client_ip)
+        if lookup_name
+        else None
     )
     if mcp_server is None and mcp_server_name is None:
-        mcp_server = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip)
+        unresolved_server: Final = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip)
+        mcp_server = (
+            await global_mcp_server_manager.ensure_oauth_metadata_discovered(unresolved_server)
+            if unresolved_server is not None
+            else None
+        )
     if mcp_server is None:
         raise HTTPException(status_code=404, detail="MCP server not found")
     _raise_if_not_oauth2(mcp_server)
@@ -1761,9 +1768,14 @@ async def token_endpoint(
 
     lookup_name: Final = mcp_server_name or client_id
     client_ip: Final = IPAddressUtils.get_mcp_client_ip(request)
-    mcp_server = global_mcp_server_manager.get_mcp_server_by_name(lookup_name, client_ip=client_ip)
+    mcp_server = await global_mcp_server_manager.get_resolved_mcp_server_by_name(lookup_name, client_ip=client_ip)
     if mcp_server is None and mcp_server_name is None:
-        mcp_server = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip)
+        unresolved_server: Final = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip)
+        mcp_server = (
+            await global_mcp_server_manager.ensure_oauth_metadata_discovered(unresolved_server)
+            if unresolved_server is not None
+            else None
+        )
     if mcp_server is None:
         raise HTTPException(status_code=404, detail="MCP server not found")
     return await exchange_token_with_server(
@@ -2175,7 +2187,7 @@ async def _build_oauth_protected_resource_response(
             )
 
         if upstream_metadata is not None:
-            if mcp_server.is_true_passthrough or mcp_server.is_oauth_delegate:
+            if mcp_server.is_client_forwarded_token:
                 return upstream_metadata
             return {**upstream_metadata, "resource": resource_url}
 
@@ -2397,6 +2409,7 @@ def _build_oauth_authorization_server_response(
 
     request_base_url: Final = get_request_base_url(request)
     client_ip: Final = IPAddressUtils.get_mcp_client_ip(request)
+    explicitly_named: Final = mcp_server_name is not None
 
     # When no server name provided, try to resolve the single OAuth2 server
     if mcp_server_name is None:
@@ -2415,8 +2428,10 @@ def _build_oauth_authorization_server_response(
 
     _raise_unless_oauth2_discovery_server(mcp_server, mcp_server_name, "not an OAuth authorization server")
 
+    issuer: Final = f"{request_base_url}/{mcp_server_name}" if explicitly_named else request_base_url
+
     return {
-        "issuer": request_base_url,  # point to your proxy
+        "issuer": issuer,
         "authorization_endpoint": authorization_endpoint,
         "token_endpoint": token_endpoint,
         "response_types_supported": ["code"],
@@ -2562,9 +2577,10 @@ async def register_client(request: Request, mcp_server_name: str | None = None):
             return await register_aggregate_client(request=request, request_body=data)
         resolved: Final = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip)
         if resolved:
+            resolved_server: Final = await global_mcp_server_manager.ensure_oauth_metadata_discovered(resolved)
             return await register_client_with_server(
                 request=request,
-                mcp_server=resolved,
+                mcp_server=resolved_server,
                 client_name=data.get("client_name", ""),
                 grant_types=data.get("grant_types", []),
                 response_types=data.get("response_types", []),
@@ -2574,7 +2590,10 @@ async def register_client(request: Request, mcp_server_name: str | None = None):
             )
         return dummy_return
 
-    mcp_server: Final = global_mcp_server_manager.get_mcp_server_by_name(mcp_server_name, client_ip=client_ip)
+    mcp_server: Final = await global_mcp_server_manager.get_resolved_mcp_server_by_name(
+        mcp_server_name,
+        client_ip=client_ip,
+    )
     if mcp_server is None:
         return dummy_return
     return await register_client_with_server(

litellm/proxy/_experimental/mcp_server/server.py

@@ -1704,7 +1704,7 @@ if MCP_AVAILABLE:
             )
 
         extra_headers: dict[str, str] | None = None
-        is_client_forwarded_mode: Final = server.is_true_passthrough or server.is_oauth_delegate
+        is_client_forwarded_mode: Final = server.is_client_forwarded_token
         # In a multi-server listing scope the request-wide Authorization can only carry one token,
         # so it is withheld from a client-forwarded server when another server in scope also consumes
         # it (RFC 9700 cross-resource replay); such scopes must bind per-server via
@@ -2013,6 +2013,9 @@ if MCP_AVAILABLE:
                         prefetched_creds=_prefetched_oauth_creds,
                     )
 
+                if server.is_byok and server.auth_type != MCPAuth.oauth2 and server_auth_header is None:
+                    server_auth_header = await _get_byok_credential(server, user_api_key_auth)
+
                 try:
                     tools: Final = await global_mcp_server_manager._get_tools_from_server(
                         server=server,
@@ -2824,6 +2827,7 @@ if MCP_AVAILABLE:
                 proxy_logging_obj=proxy_logging_obj,
                 server=mcp_server,
                 raw_headers=raw_headers,
+                litellm_logging_obj=litellm_logging_obj,
             )
             # `pre_call_tool_check` may return guardrail-modified
             # arguments; honor them on the local path too.
@@ -2962,6 +2966,7 @@ if MCP_AVAILABLE:
                     proxy_logging_obj=proxy_logging_obj,
                     server=prefix_server,
                     raw_headers=raw_headers,
+                    litellm_logging_obj=litellm_logging_obj,
                 )
                 if "arguments" in hook_result:
                     arguments = hook_result["arguments"]  # pyright: ignore[reportAny]  # hook returns untyped args
@@ -3149,6 +3154,20 @@ if MCP_AVAILABLE:
             traceback_str: Final = traceback.format_exc(limit=MAXIMUM_TRACEBACK_LINES_TO_LOG)
             from litellm.proxy.proxy_server import proxy_logging_obj
 
+            # Ordering is load-bearing. ``_ProxyDBLogger.async_post_call_failure_hook``,
+            # reached below, writes the failure spend-log row from this logger's
+            # ``standard_logging_object``, which only exists once the failure handlers
+            # have run. Flush them first or the row lands with
+            # ``guardrail_information=None`` and a guardrail block is never counted.
+            #
+            # Not double-logged: both handlers gate on ``should_run_logging`` and then
+            # mark it, so the ``@client`` wrapper's own post-raise logging no-ops on this
+            # logger, same as ``_fire_mcp_tool_call_logging`` does for ``isError=True``.
+            if litellm_logging_obj is not None:
+                end_time: Final = datetime.now()  # noqa: DTZ005  # naive to match `start_time`, which it is subtracted from
+                litellm_logging_obj.failure_handler(e, traceback_str, start_time, end_time)
+                await litellm_logging_obj.async_failure_handler(e, traceback_str, start_time, end_time)
+
             if proxy_logging_obj and user_api_key_auth:
                 await proxy_logging_obj.post_call_failure_hook(
                     request_data=kwargs,
@@ -3326,6 +3345,7 @@ if MCP_AVAILABLE:
             raw_headers=raw_headers,
             proxy_logging_obj=proxy_logging_obj,
             host_progress_callback=host_progress_callback,
+            litellm_logging_obj=litellm_logging_obj,
         )
         verbose_logger.debug("CALL TOOL RESULT: %s", call_tool_result)
         return call_tool_result
@@ -3737,6 +3757,14 @@ if MCP_AVAILABLE:
                 # preemptive challenge and let downstream authorization
                 # return 403.
                 continue
+            if server is not None and server.auth_type == MCPAuth.oauth2 and server.oauth2_flow == "client_credentials":
+                # Stamped M2M: the challenge decision below never reads discovered
+                # metadata, so deferred-discovery failures must not 503 this loop.
+                # Unstamped rows stay on the discover-first path because filling
+                # authorization_url/token_url can change their inferred flow.
+                continue
+            if server is not None:
+                server = await global_mcp_server_manager.ensure_oauth_metadata_discovered(server)
             if server and server.auth_type == MCPAuth.oauth2:
                 # The challenge decision is per oauth2 sub-mode, not per header:
                 # gateway-managed modes (M2M and interactive authorization_code)

litellm/proxy/_experimental/out/assets/logos/valkey.svg

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="64" height="73" viewBox="0 0 64 73" xmlns="http://www.w3.org/2000/svg">
+    <g id="Group-copy">
+        <path id="Path" fill="#123678" fill-rule="evenodd" stroke="none" d="M 13.482285 60.694962 L 0.998384 52.884399 L 0.998384 19.502914 L 31.527868 2.001205 L 61.317604 19.532024 L 61.317604 54.64489 L 31.054855 71.68927 L 20.548372 65.115807 L 20.548372 51.041328 L 20.548372 49.119896 L 14.851504 45.555508 L 14.851504 27.453159 L 31.346497 17.99712 L 47.464485 27.482262 L 47.464485 46.451157 L 34.703495 53.638138 L 34.703495 45.998573 C 38.52874 44.52552 41.274452 40.739189 41.274452 36.270489 C 41.274452 30.510658 36.712814 25.88438 31.158138 25.88438 C 25.603172 25.88438 21.041817 30.510658 21.041817 36.270489 C 21.041817 40.739189 23.787249 44.52552 27.612494 45.998573 L 27.612494 60.473576 L 31.261133 62.756348 L 53.635483 50.15464 L 53.635483 23.924595 L 31.477489 10.884869 L 8.680504 23.953705 L 8.680504 48.628967 L 13.482285 51.633297 L 13.482285 60.694962 Z M 31.158138 31.498383 C 33.671822 31.498383 35.660439 33.664162 35.660439 36.270489 C 35.660439 38.876804 33.671822 41.042587 31.158138 41.042587 C 28.644447 41.042587 26.655558 38.876804 26.655558 36.270489 C 26.655558 33.664162 28.644447 31.498383 31.158138 31.498383 Z" />
+    </g>
+</svg>

litellm/proxy/_lazy_openapi_snapshot.json

@@ -11349,6 +11349,40 @@
           "title": "UpdateGuardrailRequest",
           "type": "object"
         },
+        "UsageChartPoint": {
+          "properties": {
+            "blocked": {
+              "title": "Blocked",
+              "type": "integer"
+            },
+            "date": {
+              "title": "Date",
+              "type": "string"
+            },
+            "passed": {
+              "title": "Passed",
+              "type": "integer"
+            },
+            "score": {
+              "anyOf": [
+                {
+                  "type": "number"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Score"
+            }
+          },
+          "required": [
+            "date",
+            "passed",
+            "blocked"
+          ],
+          "title": "UsageChartPoint",
+          "type": "object"
+        },
         "UsageDetailResponse": {
           "properties": {
             "avgLatency": {
@@ -11410,8 +11444,7 @@
             },
             "time_series": {
               "items": {
-                "additionalProperties": true,
-                "type": "object"
+                "$ref": "#/components/schemas/UsageChartPoint"
               },
               "title": "Time Series",
               "type": "array"
@@ -11423,6 +11456,40 @@
             "type": {
               "title": "Type",
               "type": "string"
+            },
+            "usage_units": {
+              "additionalProperties": {
+                "type": "integer"
+              },
+              "title": "Usage Units",
+              "type": "object"
+            },
+            "usage_units_by_key": {
+              "additionalProperties": {
+                "additionalProperties": {
+                  "type": "integer"
+                },
+                "type": "object"
+              },
+              "title": "Usage Units By Key",
+              "type": "object"
+            },
+            "usage_units_by_team": {
+              "additionalProperties": {
+                "additionalProperties": {
+                  "type": "integer"
+                },
+                "type": "object"
+              },
+              "title": "Usage Units By Team",
+              "type": "object"
+            },
+            "usage_units_daily": {
+              "items": {
+                "$ref": "#/components/schemas/UsageUnitsDailyPoint"
+              },
+              "title": "Usage Units Daily",
+              "type": "array"
             }
           },
           "required": [
@@ -11437,7 +11504,11 @@
             "status",
             "trend",
             "description",
-            "time_series"
+            "time_series",
+            "usage_units",
+            "usage_units_daily",
+            "usage_units_by_team",
+            "usage_units_by_key"
           ],
           "title": "UsageDetailResponse",
           "type": "object"
@@ -11572,8 +11643,7 @@
           "properties": {
             "chart": {
               "items": {
-                "additionalProperties": true,
-                "type": "object"
+                "$ref": "#/components/schemas/UsageChartPoint"
               },
               "title": "Chart",
               "type": "array"
@@ -11596,6 +11666,13 @@
             "totalRequests": {
               "title": "Totalrequests",
               "type": "integer"
+            },
+            "totalUsageUnits": {
+              "additionalProperties": {
+                "type": "integer"
+              },
+              "title": "Totalusageunits",
+              "type": "object"
             }
           },
           "required": [
@@ -11603,7 +11680,8 @@
             "chart",
             "totalRequests",
             "totalBlocked",
-            "passRate"
+            "passRate",
+            "totalUsageUnits"
           ],
           "title": "UsageOverviewResponse",
           "type": "object"
@@ -11663,6 +11741,13 @@
             "type": {
               "title": "Type",
               "type": "string"
+            },
+            "usageUnits": {
+              "additionalProperties": {
+                "type": "integer"
+              },
+              "title": "Usageunits",
+              "type": "object"
             }
           },
           "required": [
@@ -11675,11 +11760,33 @@
             "avgScore",
             "avgLatency",
             "status",
-            "trend"
+            "trend",
+            "usageUnits"
           ],
           "title": "UsageOverviewRow",
           "type": "object"
         },
+        "UsageUnitsDailyPoint": {
+          "properties": {
+            "date": {
+              "title": "Date",
+              "type": "string"
+            },
+            "units": {
+              "additionalProperties": {
+                "type": "integer"
+              },
+              "title": "Units",
+              "type": "object"
+            }
+          },
+          "required": [
+            "date",
+            "units"
+          ],
+          "title": "UsageUnitsDailyPoint",
+          "type": "object"
+        },
         "ValidationError": {
           "properties": {
             "loc": {
@@ -21477,6 +21584,13 @@
             "totalRequests": {
               "title": "Totalrequests",
               "type": "integer"
+            },
+            "totalUsageUnits": {
+              "additionalProperties": {
+                "type": "integer"
+              },
+              "title": "Totalusageunits",
+              "type": "object"
             }
           },
           "required": [
@@ -21484,7 +21598,8 @@
             "chart",
             "totalRequests",
             "totalBlocked",
-            "passRate"
+            "passRate",
+            "totalUsageUnits"
           ],
           "title": "UsageOverviewResponse",
           "type": "object"
@@ -21544,6 +21659,13 @@
             "type": {
               "title": "Type",
               "type": "string"
+            },
+            "usageUnits": {
+              "additionalProperties": {
+                "type": "integer"
+              },
+              "title": "Usageunits",
+              "type": "object"
             }
           },
           "required": [
@@ -21556,7 +21678,8 @@
             "avgScore",
             "avgLatency",
             "status",
-            "trend"
+            "trend",
+            "usageUnits"
           ],
           "title": "UsageOverviewRow",
           "type": "object"

litellm/proxy/_types.py

@@ -287,6 +287,7 @@ class KeyManagementRoutes(str, enum.Enum):
 
     # team usage routes
     TEAM_DAILY_ACTIVITY = "/team/daily/activity"
+    TEAM_DAILY_ACTIVITY_AGGREGATED = "/team/daily/activity/aggregated"
 
     # team spend-log viewing
     SPEND_LOGS = "/spend/logs"
@@ -451,6 +452,7 @@ class LiteLLMRoutes(enum.Enum):
 
     mapped_pass_through_routes = [
         "/bedrock",
+        "/comprehendmedical",
         "/vertex-ai",
         "/vertex_ai",
         "/cohere",
@@ -611,6 +613,7 @@ class LiteLLMRoutes(enum.Enum):
         KeyManagementRoutes.KEY_BULK_UPDATE.value,
         KeyManagementRoutes.TEAM_KEY_BULK_UPDATE.value,
         KeyManagementRoutes.TEAM_DAILY_ACTIVITY.value,
+        KeyManagementRoutes.TEAM_DAILY_ACTIVITY_AGGREGATED.value,
         KeyManagementRoutes.SPEND_LOGS.value,
         KeyManagementRoutes.SPEND_LOGS_V2.value,
         KeyManagementRoutes.KEY_RESET_SPEND.value,
@@ -645,6 +648,7 @@ class LiteLLMRoutes(enum.Enum):
             "/team/permissions_update",
             "/team/permissions_bulk_update",
             "/team/daily/activity",
+            "/team/daily/activity/aggregated",
             # gateway request counts (SGR); deployment-wide, admin-only
             "/gateway/daily/activity",
             # model
@@ -680,6 +684,7 @@ class LiteLLMRoutes(enum.Enum):
         # permitted teams exactly like /spend/logs/ui — it belongs to the same
         # access tier, not to customer management.
         "/management/v1/spend_logs/end_users",
+        "/management/v1/spend_logs/users",
         "/cost/estimate",
     ]
 
@@ -799,12 +804,16 @@ class LiteLLMRoutes(enum.Enum):
         "/team/permissions_list",
         "/team/permissions_update",
         "/team/daily/activity",
+        "/team/daily/activity/aggregated",
         "/team/{team_id}/members/me",
         "/model/new",
         "/model/update",
         "/model/delete",
         "/user/daily/activity",
         "/user/daily/activity/aggregated",
+        # Endpoint restricts results to organizations the caller is ORG_ADMIN
+        # of; a caller who administers none gets an empty result set.
+        "/organization/daily/activity",
         "/user/available_roles",  # read-only role metadata; any authenticated user may read
         "/user/list",  # org admins checked in endpoint; non-admins get 403
         "/model/{model_id}/update",
@@ -861,6 +870,7 @@ class LiteLLMRoutes(enum.Enum):
             "/user/available_roles",
             "/user/daily/activity",
             "/team/daily/activity",
+            "/team/daily/activity/aggregated",
             "/tag/daily/activity",
             "/tag/list",
             "/audit",
@@ -872,12 +882,13 @@ class LiteLLMRoutes(enum.Enum):
             # PROXY_ADMIN_VIEW_ONLY — the route gate must match).
             "/customer/list",
             "/customer/info",
-            # UI Logs page detail drawer (single + session) and the end-user filter
-            # facet. The list endpoint `/spend/logs/ui` is covered via
+            # UI Logs page detail drawer (single + session) and the filter facets.
+            # The list endpoint `/spend/logs/ui` is covered via
             # spend_tracking_routes below.
             "/spend/logs/ui/{logId}",
             "/spend/logs/session/ui",
             "/management/v1/spend_logs/end_users",
+            "/management/v1/spend_logs/users",
             # Settings / observability read endpoints exposed in admin-only
             # sidebar groups (Logging & Alerts, Admin Settings, Budgets,
             # Invitations).
@@ -1987,6 +1998,18 @@ class AddTeamCallback(LiteLLMPydanticObjectBase):
         return values
 
 
+class TeamCallbackDeleteResponseData(LiteLLMPydanticObjectBase):
+    team_id: str
+    success_callbacks: tuple[str, ...]
+    failure_callbacks: tuple[str, ...]
+
+
+class TeamCallbackDeleteResponse(LiteLLMPydanticObjectBase):
+    status: Literal["success"]
+    message: str
+    data: TeamCallbackDeleteResponseData
+
+
 class TeamCallbackMetadata(LiteLLMPydanticObjectBase):
     success_callback: list[str] | None = []
     failure_callback: list[str] | None = []
@@ -3036,6 +3059,8 @@ class NewProjectRequest(LiteLLM_BudgetTable):
     models: list[str] = []
     model_rpm_limit: dict | None = None
     model_tpm_limit: dict | None = None
+    model_itpm_limit: Mapping[str, int] | None = None
+    model_otpm_limit: Mapping[str, int] | None = None
     blocked: bool = False
     object_permission: LiteLLM_ObjectPermissionBase | None = None
 
@@ -3068,6 +3093,8 @@ class UpdateProjectRequest(LiteLLM_BudgetTable):
     models: list[str] | None = None
     model_rpm_limit: dict | None = None
     model_tpm_limit: dict | None = None
+    model_itpm_limit: Mapping[str, int] | None = None
+    model_otpm_limit: Mapping[str, int] | None = None
     blocked: bool | None = None
     budget_id: str | None = None
     object_permission: LiteLLM_ObjectPermissionBase | None = None
@@ -4219,6 +4246,8 @@ class PassThroughEndpointLoggingTypedDict(TypedDict):
 LiteLLM_ManagementEndpoint_MetadataFields: Final = [
     "model_rpm_limit",
     "model_tpm_limit",
+    "model_itpm_limit",
+    "model_otpm_limit",
     "default_estimated_output_tokens",
     "default_estimated_output_tokens_per_model",
     "mcp_rpm_limit",

litellm/proxy/agent_endpoints/a2a_endpoints.py

@@ -13,6 +13,7 @@ The A2A SDK can point to LiteLLM's URL and invoke agents registered with LiteLLM
 import json
 from collections.abc import AsyncGenerator, Mapping
 from copy import deepcopy
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final
 from urllib.parse import urlparse
 
@@ -20,6 +21,7 @@ from fastapi import APIRouter, Depends, HTTPException, Request, Response
 from fastapi.responses import JSONResponse, StreamingResponse
 from pydantic import ValidationError
 
+import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.litellm_core_utils.url_utils import SSRFError, validate_url
 from litellm.proxy._types import UserAPIKeyAuth
@@ -36,6 +38,11 @@ from litellm.proxy.agent_endpoints.databricks_oauth import (
 )
 from litellm.proxy.agent_endpoints.utils import merge_agent_headers
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.proxy.common_utils.sse_keepalive import (
+    SSE_COMMENT_PING,
+    coerce_keepalive_interval,
+    wrap_sse_stream_with_keepalive_pings,
+)
 from litellm.proxy.utils import ProxyLogging, get_custom_url
 from litellm.types.utils import all_litellm_params
 
@@ -46,6 +53,15 @@ if TYPE_CHECKING:
 
 router: Final = APIRouter()
 
+# Mirrors the native seam's own headers: a reverse proxy that batches the whole
+# stream would swallow the keepalives this route sends to defeat idle timeouts.
+_SSE_KEEPALIVE_HEADERS: Final[Mapping[str, str]] = MappingProxyType(
+    {
+        "Cache-Control": "no-cache",
+        "X-Accel-Buffering": "no",
+    }
+)
+
 _PASCAL_TO_WIRE: Final[Mapping[str, str]] = {
     "SendMessage": "message/send",
     "SendStreamingMessage": "message/stream",
@@ -326,7 +342,19 @@ async def _forward_jsonrpc_sse(
 
         generator = _passthrough()
 
-    return StreamingResponse(generator, media_type="text/event-stream")
+    # The upstream agent is only contacted once this generator is first pulled, so
+    # a slow first event leaves the response body idle for its whole
+    # time-to-first-token and an intermediary with an idle read timeout drops a
+    # healthy connection. Off until an operator sets an interval, and the
+    # buffering hint only goes out when there are keepalives to protect.
+    keepalive_interval: Final = coerce_keepalive_interval(litellm.sse_keepalive_ping_interval_seconds)
+    if keepalive_interval is None:
+        return StreamingResponse(generator, media_type="text/event-stream")
+    return StreamingResponse(
+        wrap_sse_stream_with_keepalive_pings(generator, keepalive_interval, ping_chunk=SSE_COMMENT_PING),
+        media_type="text/event-stream",
+        headers=_SSE_KEEPALIVE_HEADERS,
+    )
 
 
 async def _handle_stream_message(

litellm/proxy/anthropic_endpoints/endpoints.py

@@ -193,6 +193,9 @@ async def anthropic_response(
         )
         verbose_proxy_logger.exception("litellm.proxy.proxy_server.anthropic_response(): Exception occured - %s", e)
 
+        if isinstance(e, ProxyException):
+            raise
+
         # Extract model_id from request metadata (same as success path)
         litellm_metadata: Final = data.get("litellm_metadata", {}) or {}
         model_info: Final = litellm_metadata.get("model_info", {}) or {}

litellm/proxy/auth/auth_checks.py

@@ -13,7 +13,7 @@ import asyncio
 import math
 import re
 import time
-from collections.abc import Iterator, Mapping, Sequence
+from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, cast
 
@@ -30,6 +30,9 @@ from litellm.constants import (
     DEFAULT_IN_MEMORY_TTL,
     DEFAULT_MAX_RECURSE_DEPTH,
     EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE,
+    END_USER_RESTRICTED_REGISTRY_MAX_SIZE,
+    REGISTRY_ERROR_NEGATIVE_CACHE_TTL,
+    TAG_REGISTRY_MAX_SIZE,
 )
 from litellm.litellm_core_utils.dd_tracing import tracer
 from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
@@ -74,9 +77,15 @@ from litellm.proxy.common_utils.http_parsing_utils import (
 )
 from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time
 from litellm.proxy.common_utils.user_api_key_cache import (
+    END_USER_RESTRICTED_REGISTRY_OVERFLOW_SENTINEL,
+    TAG_REGISTRY_OVERFLOW_SENTINEL,
     UserApiKeyCache,
+    end_user_cache_key,
+    end_user_restricted_registry_cache_key,
     get_management_object_ttl,
     object_permission_cache_key,
+    tag_cache_key,
+    tag_registry_cache_key,
 )
 from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
 from litellm.proxy.guardrails.tool_name_extraction import (
@@ -163,7 +172,7 @@ class _PrismaAuthTable(Protocol[RowT_co]):
     async def find_many(
         self,
         *,
-        where: Mapping[str, object],
+        where: Mapping[str, object] | None = None,
         include: Mapping[str, object] | None = None,
         take: int | None = None,
     ) -> Sequence[RowT_co]: ...
@@ -220,6 +229,16 @@ def _tag_table(repo: _PrismaTableHolder[_PrismaTagRow]) -> _PrismaAuthTable[_Pri
     return repo.table
 
 
+class _PrismaEndUserRow(Protocol):
+    user_id: str
+
+    def dict(self) -> Mapping[str, object]: ...
+
+
+def _end_user_table(repo: _PrismaTableHolder[_PrismaEndUserRow]) -> _PrismaAuthTable[_PrismaEndUserRow]:
+    return repo.table
+
+
 class _RawCacheRead(Protocol):
     async def async_get_cache(self, *, key: str) -> object: ...
 
@@ -1284,6 +1303,191 @@ async def _check_end_user_budget(
         )
 
 
+#: Columns whose non-null value makes an end-user row restrict something auth enforces. ``blocked``
+#: is separate: it restricts when true rather than when merely set.
+_RESTRICTED_COLUMNS: Final = ("budget_id", "allowed_model_region", "default_model", "object_permission_id")
+
+
+def _column_is_set(column: str) -> Mapping[str, object]:
+    """``column IS NOT NULL`` as a plain dict, which is the only shape prisma's builder accepts."""
+    return {column: {"not": None}}  # mutable-ok: prisma's query builder isinstance-checks for dict
+
+
+def _restricted_end_user_where() -> Mapping[str, object]:
+    """Prisma filter selecting every end-user row that carries a restriction auth enforces."""
+    return {"OR": [{"blocked": True}, *map(_column_is_set, _RESTRICTED_COLUMNS)]}  # mutable-ok: prisma needs dict/list
+
+
+class _RegistryNotCached:
+    """No cached registry answer, as distinct from the cached answer ``None`` (registry unusable)."""
+
+
+_REGISTRY_NOT_CACHED: Final = _RegistryNotCached()
+
+#: One lock per registry; module-level because the stampede to collapse is worker-wide.
+_TAG_REGISTRY_LOAD_LOCK: Final = asyncio.Lock()
+_END_USER_REGISTRY_LOAD_LOCK: Final = asyncio.Lock()
+
+
+async def _cached_registry(
+    cache_key: str,
+    overflow_sentinel: str,
+    user_api_key_cache: UserApiKeyCache,
+) -> frozenset[str] | None | _RegistryNotCached:
+    """The cached registry answer, or ``_REGISTRY_NOT_CACHED`` when the caller has to query."""
+    cached: Final = await _raw_cache(user_api_key_cache).async_get_cache(key=cache_key)
+    if cached == overflow_sentinel:
+        return None
+    # Memory hands back the tuple that was written; Redis round-trips it through JSON as a list.
+    if isinstance(cached, (list, tuple)):
+        return frozenset(entry for entry in cached if isinstance(entry, str))
+    return _REGISTRY_NOT_CACHED
+
+
+async def _cache_registry_answer(
+    cache_key: str,
+    value: tuple[str, ...] | str,
+    ttl: float,
+    user_api_key_cache: UserApiKeyCache,
+) -> None:
+    """Best-effort: a cache backend failure must not turn a registry load into a failed request."""
+    try:
+        await user_api_key_cache.async_set_cache(key=cache_key, value=value, ttl=ttl)
+    except Exception as e:  # noqa: BLE001  # best-effort cache write: auth must survive a cache backend error
+        verbose_proxy_logger.warning("Failed to cache registry %s: %s", cache_key, e)
+
+
+async def _fetch_and_cache_registry(
+    cache_key: str,
+    overflow_sentinel: str,
+    max_size: int,
+    fetch_ids: Callable[[], Awaitable[tuple[str, ...]]],
+    user_api_key_cache: UserApiKeyCache,
+) -> frozenset[str] | None:
+    """The registry as the database has it, cached whole, or ``None`` when it is unusable."""
+    try:
+        registry_ids: Final = await fetch_ids()
+    except Exception as e:  # noqa: BLE001  # fail-safe: any registry load error must degrade to per-id lookups, never break auth
+        verbose_proxy_logger.warning(
+            "Registry %s could not be loaded from the database, so per-id lookups will run and the "
+            "registry query is suppressed for %ss: %s",
+            cache_key,
+            REGISTRY_ERROR_NEGATIVE_CACHE_TTL,
+            e,
+        )
+        await _cache_registry_answer(
+            cache_key=cache_key,
+            value=overflow_sentinel,
+            ttl=REGISTRY_ERROR_NEGATIVE_CACHE_TTL,
+            user_api_key_cache=user_api_key_cache,
+        )
+        return None
+
+    if len(registry_ids) > max_size:
+        await _cache_registry_answer(
+            cache_key=cache_key,
+            value=overflow_sentinel,
+            ttl=get_management_object_ttl(user_api_key_cache),
+            user_api_key_cache=user_api_key_cache,
+        )
+        return None
+
+    await _cache_registry_answer(
+        cache_key=cache_key,
+        value=registry_ids,
+        ttl=get_management_object_ttl(user_api_key_cache),
+        user_api_key_cache=user_api_key_cache,
+    )
+    return frozenset(registry_ids)
+
+
+async def _load_bounded_registry(
+    cache_key: str,
+    overflow_sentinel: str,
+    max_size: int,
+    load_lock: asyncio.Lock,
+    fetch_ids: Callable[[], Awaitable[tuple[str, ...]]],
+    user_api_key_cache: UserApiKeyCache,
+) -> frozenset[str] | None:
+    """
+    A bounded id set under one cache key, so an id outside it costs no DB read.
+
+    ``None`` = unusable (overflow or recent DB error): fall back to per-id lookups. An empty
+    frozenset is a real, cacheable answer. Loads are single-flighted to stop TTL-expiry stampedes.
+    """
+    cached: Final = await _cached_registry(cache_key, overflow_sentinel, user_api_key_cache)
+    if not isinstance(cached, _RegistryNotCached):
+        return cached
+
+    async with load_lock:
+        # The request that held the lock has since cached an answer for everyone waiting on it.
+        cached_after_wait: Final = await _cached_registry(cache_key, overflow_sentinel, user_api_key_cache)
+        if not isinstance(cached_after_wait, _RegistryNotCached):
+            return cached_after_wait
+
+        return await _fetch_and_cache_registry(
+            cache_key=cache_key,
+            overflow_sentinel=overflow_sentinel,
+            max_size=max_size,
+            fetch_ids=fetch_ids,
+            user_api_key_cache=user_api_key_cache,
+        )
+
+
+async def _load_end_user_restricted_registry(
+    prisma_client: PrismaClient,
+    user_api_key_cache: UserApiKeyCache,
+) -> frozenset[str] | None:
+    """The set of end-user ids whose ``LiteLLM_EndUserTable`` row carries a restriction."""
+
+    async def fetch_ids() -> tuple[str, ...]:
+        restricted_rows: Final = await _end_user_table(EndUserRepository(prisma_client)).find_many(
+            where=_restricted_end_user_where(),
+            take=END_USER_RESTRICTED_REGISTRY_MAX_SIZE + 1,
+        )
+        return tuple(row.user_id for row in restricted_rows)
+
+    return await _load_bounded_registry(
+        cache_key=end_user_restricted_registry_cache_key(),
+        overflow_sentinel=END_USER_RESTRICTED_REGISTRY_OVERFLOW_SENTINEL,
+        max_size=END_USER_RESTRICTED_REGISTRY_MAX_SIZE,
+        load_lock=_END_USER_REGISTRY_LOAD_LOCK,
+        fetch_ids=fetch_ids,
+        user_api_key_cache=user_api_key_cache,
+    )
+
+
+async def _end_user_is_known_unrestricted(
+    end_user_id: str,
+    prisma_client: PrismaClient,
+    user_api_key_cache: UserApiKeyCache,
+    token_end_user_max_budget: float | None,
+) -> bool:
+    """
+    True when the cached registry proves the id restricts nothing, so its row need not be read.
+
+    Every field ``get_end_user_object`` callers consume (budget, spend under that budget, region,
+    default model, object permission, blocked) is part of the registry predicate, so an id outside
+    it is indistinguishable from one with no row at all. The skip is off whenever mere existence of
+    the row is meaningful: ``max_end_user_budget_id`` grafts a default budget onto any row that
+    exists, ``validate_end_user_id_in_db`` rejects ids that resolve to no row, and a token-supplied
+    ``end_user_max_budget`` (a ``user_custom_auth`` callable can set one against an otherwise
+    unrestricted row) is enforced against the row's recorded spend.
+    """
+    if (
+        litellm.max_end_user_budget_id is not None
+        or litellm.validate_end_user_id_in_db
+        or token_end_user_max_budget is not None
+    ):
+        return False
+
+    registry: Final = await _load_end_user_restricted_registry(
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+    )
+    return registry is not None and end_user_id not in registry
+
+
 @log_db_metrics
 async def get_end_user_object(
     end_user_id: str | None,
@@ -1292,6 +1496,7 @@ async def get_end_user_object(
     route: str | None = "",
     parent_otel_span: Span | None = None,
     proxy_logging_obj: ProxyLogging | None = None,
+    token_end_user_max_budget: float | None = None,
 ) -> LiteLLM_EndUserTable | None:
     """
     Returns end user object from database or cache.
@@ -1306,6 +1511,9 @@ async def get_end_user_object(
         route: The request route
         parent_otel_span: Optional OpenTelemetry span for tracing
         proxy_logging_obj: Optional proxy logging object
+        token_end_user_max_budget: ``valid_token.end_user_max_budget``, when the caller holds a
+            token. Budget enforcement reads the row's spend, so a row that restricts nothing on
+            its own must still be loaded when the token carries a budget for it.
 
     Returns:
         LiteLLM_EndUserTable if found, None otherwise
@@ -1316,7 +1524,7 @@ async def get_end_user_object(
     if end_user_id is None:
         return None
 
-    _key: Final = f"end_user_id:{end_user_id}"
+    _key: Final = end_user_cache_key(end_user_id)
 
     # Check cache first
     cached_user_obj: Final = await user_api_key_cache.async_get_cache(
@@ -1335,6 +1543,14 @@ async def get_end_user_object(
 
         return return_obj
 
+    if await _end_user_is_known_unrestricted(
+        end_user_id=end_user_id,
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+        token_end_user_max_budget=token_end_user_max_budget,
+    ):
+        return None
+
     # Fetch from database
     try:
         response: Final = await _dictable_table(EndUserRepository(prisma_client)).find_unique(
@@ -1358,9 +1574,10 @@ async def get_end_user_object(
 
         # Save to cache
         await user_api_key_cache.async_set_cache(
-            key=f"end_user_id:{end_user_id}",
+            key=_key,
             value=_response,
             model_type=LiteLLM_EndUserTable,
+            ttl=get_management_object_ttl(user_api_key_cache),
         )
 
         return _response
@@ -1480,6 +1697,67 @@ async def _end_user_id_exists_in_db(
     return False
 
 
+async def _load_tag_registry(
+    prisma_client: PrismaClient,
+    user_api_key_cache: UserApiKeyCache,
+) -> frozenset[str] | None:
+    """The set of tag names that have a row in ``LiteLLM_TagTable``."""
+
+    async def fetch_ids() -> tuple[str, ...]:
+        registry_rows: Final = await _tag_table(TagRepository(prisma_client)).find_many(
+            take=TAG_REGISTRY_MAX_SIZE + 1,
+        )
+        return tuple(row.tag_name for row in registry_rows)
+
+    return await _load_bounded_registry(
+        cache_key=tag_registry_cache_key(),
+        overflow_sentinel=TAG_REGISTRY_OVERFLOW_SENTINEL,
+        max_size=TAG_REGISTRY_MAX_SIZE,
+        load_lock=_TAG_REGISTRY_LOAD_LOCK,
+        fetch_ids=fetch_ids,
+        user_api_key_cache=user_api_key_cache,
+    )
+
+
+async def _fetch_uncached_tags(
+    uncached_tags: Sequence[str],
+    prisma_client: PrismaClient,
+    user_api_key_cache: UserApiKeyCache,
+) -> tuple[tuple[str, LiteLLM_TagTable], ...]:
+    """Rows for the tags a cache probe missed; names absent from the registry never reach the DB."""
+    if not uncached_tags:
+        return ()
+
+    registry: Final = await _load_tag_registry(
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+    )
+    tags_to_fetch: Final = (
+        tuple(uncached_tags) if registry is None else tuple(tag for tag in uncached_tags if tag in registry)
+    )
+    if not tags_to_fetch:
+        return ()
+
+    try:
+        db_tags: Final = await _tag_table(TagRepository(prisma_client)).find_many(
+            where={"tag_name": {"in": list(tags_to_fetch)}},
+            include={"litellm_budget_table": True},
+        )
+        fetched: Final = tuple((db_tag.tag_name, LiteLLM_TagTable.model_validate(db_tag.dict())) for db_tag in db_tags)
+        for fetched_name, fetched_obj in fetched:
+            await user_api_key_cache.async_set_cache(
+                key=tag_cache_key(fetched_name),
+                value=fetched_obj,
+                model_type=LiteLLM_TagTable,
+                ttl=get_management_object_ttl(user_api_key_cache),
+            )
+    except Exception as e:  # noqa: BLE001  # fail-safe: a tag fetch error must yield "no budget objects", never break auth
+        verbose_proxy_logger.debug("Error batch fetching tags from database: %s", e)
+        return ()
+    else:
+        return fetched
+
+
 @log_db_metrics
 async def get_tag_objects_batch(
     tag_names: list[str],
@@ -1492,8 +1770,9 @@ async def get_tag_objects_batch(
     Batch fetch multiple tag objects from cache and db.
 
     Optimizes for latency by:
-    1. Fetching all cached tags in parallel
-    2. Batch fetching uncached tags in one DB query
+    1. Serving already-cached tags without touching the DB
+    2. Skipping tags that no ``LiteLLM_TagTable`` row exists for, via the cached name registry
+    3. Batch fetching the remaining uncached tags in one DB query
 
     Args:
         tag_names: List of tag names to fetch
@@ -1505,50 +1784,22 @@ async def get_tag_objects_batch(
     Returns:
         Dictionary mapping tag_name to LiteLLM_TagTable object
     """
-    if prisma_client is None:
+    if prisma_client is None or not tag_names:
         return {}
 
-    if not tag_names:
-        return {}
-
-    tag_objects: Final = dict[str, LiteLLM_TagTable]()
-    uncached_tags: Final = list[str]()
-
-    # Try to get all tags from cache first
-    for tag_name in tag_names:
-        cache_key = f"tag:{tag_name}"
-        cached_tag = await user_api_key_cache.async_get_cache(
-            key=cache_key,
-            model_type=LiteLLM_TagTable,
+    probed: Final = [
+        (
+            tag_name,
+            await user_api_key_cache.async_get_cache(key=tag_cache_key(tag_name), model_type=LiteLLM_TagTable),
         )
-        if cached_tag is not None:
-            tag_objects[tag_name] = cached_tag
-        else:
-            uncached_tags.append(tag_name)
-
-    # Batch fetch uncached tags from DB in one query
-    if uncached_tags:
-        try:
-            db_tags: Final = await _tag_table(TagRepository(prisma_client)).find_many(
-                where={"tag_name": {"in": uncached_tags}},
-                include={"litellm_budget_table": True},
-            )
-
-            # Cache and add to tag_objects
-            for db_tag in db_tags:
-                tag_name = db_tag.tag_name
-                cache_key = f"tag:{tag_name}"
-                _tag_obj = LiteLLM_TagTable.model_validate(db_tag.dict())
-                await user_api_key_cache.async_set_cache(
-                    key=cache_key,
-                    value=_tag_obj,
-                    model_type=LiteLLM_TagTable,
-                )
-                tag_objects[tag_name] = _tag_obj
-        except Exception as e:
-            verbose_proxy_logger.debug("Error batch fetching tags from database: %s", e)
-
-    return tag_objects
+        for tag_name in tag_names
+    ]
+    fetched: Final = await _fetch_uncached_tags(
+        uncached_tags=tuple(tag_name for tag_name, tag_obj in probed if tag_obj is None),
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+    )
+    return {tag_name: tag_obj for tag_name, tag_obj in (*probed, *fetched) if tag_obj is not None}
 
 
 @log_db_metrics
@@ -4573,25 +4824,15 @@ async def delete_cached_project_object(
     user_api_key_cache: UserApiKeyCache,
 ) -> None:
     """
-    Every endpoint that mutates litellm_projecttable must call this: get_project_object
-    serves auth cache-first with no freshness check, so without invalidation a stale
-    project (e.g. a pre-update empty model allowlist) keeps being enforced until the
-    TTL expires (LIT-3803). Best-effort on both steps: the DB write has already
-    committed, so a cache backend error must not fail the endpoint; the stale entry
-    then expires via TTL.
+    Every endpoint that mutates litellm_projecttable must call this, or a stale project (e.g. a
+    pre-update empty model allowlist) keeps being enforced until the TTL expires (LIT-3803).
     """
-    from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import publish_auth_cache_invalidation
+    from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast
 
-    cache_key: Final = _project_cache_key(project_id)
-    try:
-        await user_api_key_cache.async_delete_cache(key=cache_key)
-    except Exception as e:  # noqa: BLE001  # best-effort eviction: any cache backend error must not fail the mutation
-        verbose_proxy_logger.warning(
-            "Failed to evict cached project entry %s; a stale project may be served until its TTL expires: %s",
-            cache_key,
-            e,
-        )
-    await publish_auth_cache_invalidation(cache_key=cache_key)
+    await evict_and_broadcast(
+        cache_keys=(_project_cache_key(project_id),),
+        user_api_key_cache=user_api_key_cache,
+    )
 
 
 async def _organization_max_budget_check(

litellm/proxy/auth/auth_utils.py

@@ -1172,7 +1172,7 @@ def enforce_output_token_estimates_are_admin_only(
 def get_model_rate_limit_from_metadata(
     user_api_key_dict: UserAPIKeyAuth,
     metadata_accessor_key: Literal["team_metadata", "organization_metadata", "project_metadata"],
-    rate_limit_key: Literal["model_rpm_limit", "model_tpm_limit"],
+    rate_limit_key: Literal["model_rpm_limit", "model_tpm_limit", "model_itpm_limit", "model_otpm_limit"],
 ) -> dict[str, int] | None:
     if getattr(user_api_key_dict, metadata_accessor_key):
         return getattr(user_api_key_dict, metadata_accessor_key).get(rate_limit_key)

litellm/proxy/auth/user_api_key_auth.py

@@ -2307,6 +2307,7 @@ async def _run_centralized_common_checks(
                     parent_otel_span=parent_otel_span,
                     proxy_logging_obj=proxy_logging_obj,
                     route=route,
+                    token_end_user_max_budget=user_api_key_auth_obj.end_user_max_budget,
                 ),
             )
         )
@@ -2841,6 +2842,7 @@ async def _lookup_end_user_and_apply_budget(
             parent_otel_span=parent_otel_span,
             proxy_logging_obj=proxy_logging_obj,
             route=route,
+            token_end_user_max_budget=valid_token.end_user_max_budget,
         )
         if end_user_object is not None:
             end_user_params = {

litellm/proxy/batches_endpoints/common_utils.py

@@ -0,0 +1,18 @@
+from litellm.proxy._types import ProxyException
+
+
+def validate_batch_list_limit(limit: int | None) -> None:
+    if limit is None or 0 <= limit <= 100:
+        return
+    bound, expected, openai_code = (
+        ("below minimum", ">= 0", "integer_below_min_value")
+        if limit < 0
+        else ("above maximum", "<= 100", "integer_above_max_value")
+    )
+    raise ProxyException(
+        message=f"Invalid 'limit': integer {bound} value. Expected a value {expected}, but got {limit} instead.",
+        type="invalid_request_error",
+        param="limit",
+        code=400,
+        openai_code=openai_code,
+    )

litellm/proxy/batches_endpoints/endpoints.py

@@ -5,6 +5,8 @@
 
 ######################################################################
 import asyncio
+import os
+from collections.abc import Mapping
 from typing import Any, Final, cast
 
 from fastapi import APIRouter, Depends, HTTPException, Path, Request, Response
@@ -14,6 +16,7 @@ from litellm._logging import verbose_proxy_logger
 from litellm.batches.main import CancelBatchRequest, RetrieveBatchRequest
 from litellm.proxy._types import *
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.proxy.batches_endpoints.common_utils import validate_batch_list_limit
 from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
 from litellm.proxy.common_utils.callback_utils import sanitize_openai_provider_metadata
 from litellm.proxy.common_utils.http_parsing_utils import _read_request_body
@@ -40,6 +43,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
     update_batch_in_database,
     validate_managed_id_requirement,
 )
+from litellm.proxy.route_llm_request import raise_if_required_body_param_missing
 from litellm.proxy.utils import handle_exception_on_proxy, is_known_model
 from litellm.repositories.table_repositories import ManagedFileRepository
 from litellm.types.llms.openai import LiteLLMBatchCreateRequest
@@ -47,6 +51,23 @@ from litellm.types.llms.openai import LiteLLMBatchCreateRequest
 router: Final = APIRouter()
 
 
+def _raise_not_found_when_openai_fallback_unservable(
+    requested_provider: "str | None",
+    data: Mapping[str, object],
+    not_found_message: str,
+) -> None:
+    if requested_provider is not None:
+        return
+    if data.get("api_key") or litellm.api_key or litellm.openai_key or os.getenv("OPENAI_API_KEY"):
+        return
+    raise ProxyException(
+        message=not_found_message,
+        type="invalid_request_error",
+        param=None,
+        code=404,
+    )
+
+
 async def _resolve_managed_input_file_storage_url(input_file_id: str) -> "str | None":
     """Resolve a managed (unified) input_file_id to its backend storage_url.
 
@@ -140,6 +161,8 @@ async def create_batch(
         )
         data["metadata"] = sanitize_openai_provider_metadata(data.get("metadata"))
 
+        raise_if_required_body_param_missing(route_type="acreate_batch", data=data)
+
         ## check if model is a loadbalanced model
         router_model: str | None = None
         is_router_model = False
@@ -147,12 +170,12 @@ async def create_batch(
             router_model = data.get("model", None)
             is_router_model = is_known_model(model=router_model, llm_router=llm_router)
 
-        custom_llm_provider: Final = (
+        requested_provider: Final = (
             provider
             or data.pop("custom_llm_provider", None)
             or get_custom_llm_provider_from_request_headers(request=request)
-            or "openai"
         )
+        custom_llm_provider: Final = requested_provider or "openai"
         _create_batch_data: Final = LiteLLMBatchCreateRequest(**data)
 
         # Apply team-level batch output expiry enforcement
@@ -314,6 +337,11 @@ async def create_batch(
                     user_api_key_dict=user_api_key_dict,
                     custom_llm_provider=custom_llm_provider,
                 )
+                _raise_not_found_when_openai_fallback_unservable(
+                    requested_provider=requested_provider,
+                    data=cast(dict, _create_batch_data),  # cast-ok: TypedDict is a dict at runtime
+                    not_found_message=f"No such File object: {input_file_id}",
+                )
                 response = await litellm.acreate_batch(
                     custom_llm_provider=custom_llm_provider,
                     **_create_batch_data,
@@ -563,18 +591,23 @@ async def retrieve_batch(
 
         # SCENARIO 3: Fallback to custom_llm_provider (uses env variables)
         else:
-            custom_llm_provider: Final = (
+            requested_provider: Final = (
                 provider
                 or get_custom_llm_provider_from_request_headers(request=request)
                 or get_custom_llm_provider_from_request_query(request=request)
-                or "openai"
             )
+            custom_llm_provider: Final = requested_provider or "openai"
             apply_team_provider_credentials(
                 data=data,
                 llm_router=llm_router,
                 user_api_key_dict=user_api_key_dict,
                 custom_llm_provider=custom_llm_provider,
             )
+            _raise_not_found_when_openai_fallback_unservable(
+                requested_provider=requested_provider,
+                data=data,
+                not_found_message=f"No batch found with id '{batch_id}'.",
+            )
             response = await litellm.aretrieve_batch(
                 custom_llm_provider=custom_llm_provider,
                 **data,
@@ -679,6 +712,7 @@ async def list_batches(
 
     ```
     """
+    validate_batch_list_limit(limit)
     from litellm.proxy.proxy_server import (
         general_settings,
         llm_router,
@@ -967,13 +1001,13 @@ async def cancel_batch(
         # SCENARIO 3: Fallback to custom_llm_provider (uses env variables)
         else:
             body_custom_llm_provider = data.pop("custom_llm_provider", None)
-            custom_llm_provider: Final = (
+            requested_provider: Final = (
                 provider
                 or body_custom_llm_provider
                 or get_custom_llm_provider_from_request_headers(request=request)
                 or get_custom_llm_provider_from_request_query(request=request)
-                or "openai"
             )
+            custom_llm_provider: Final = requested_provider or "openai"
             # Extract batch_id from data to avoid "multiple values for keyword argument" error
             # data was cast from CancelBatchRequest which already contains batch_id
             data.pop("batch_id", None)
@@ -983,6 +1017,11 @@ async def cancel_batch(
                 user_api_key_dict=user_api_key_dict,
                 custom_llm_provider=custom_llm_provider,
             )
+            _raise_not_found_when_openai_fallback_unservable(
+                requested_provider=requested_provider,
+                data=data,
+                not_found_message=f"No batch found with id '{batch_id}'.",
+            )
             _cancel_batch_data: Final = CancelBatchRequest(batch_id=batch_id, **data)
             response = await litellm.acancel_batch(
                 custom_llm_provider=custom_llm_provider,

litellm/proxy/common_request_processing.py

@@ -1,14 +1,15 @@
 import asyncio
+import contextlib
 import json
 import logging
 import math
 import time
 import traceback
-from collections.abc import AsyncGenerator, Callable, Mapping
+from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping
 from datetime import datetime
 from functools import lru_cache
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, Protocol, overload
+from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, Protocol, TypeAlias, TypeVar, overload
 
 import anyio
 import httpx
@@ -31,11 +32,13 @@ from litellm.constants import (
     UNSAFE_PROXY_RESPONSE_HEADERS,
 )
 from litellm.integrations.custom_guardrail import CustomGuardrail
+from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket
 from litellm.litellm_core_utils.dd_tracing import NullTracer, tracer
 from litellm.litellm_core_utils.get_supported_openai_params import (
     get_supported_openai_params,
 )
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost
 from litellm.litellm_core_utils.llm_response_utils.get_headers import (
     get_response_headers,
 )
@@ -47,7 +50,12 @@ from litellm.proxy.common_utils.callback_utils import (
     get_logging_caching_headers,
     get_remaining_tokens_and_requests_from_request_data,
 )
-from litellm.proxy.common_utils.sse_keepalive import wrap_sse_stream_with_keepalive_pings
+from litellm.proxy.common_utils.sse_keepalive import (
+    SSE_COMMENT_PING_BYTES,
+    coerce_keepalive_interval,
+    resolve_ttft_keepalive_interval,
+    wrap_sse_stream_with_keepalive_pings,
+)
 from litellm.proxy.dd_span_tagger import DDSpanTagger
 from litellm.proxy.route_llm_request import route_request
 from litellm.proxy.utils import ProxyLogging, _check_and_merge_model_level_guardrails
@@ -56,6 +64,100 @@ from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_di
 from litellm.router_utils.common_utils import resolve_model_group_alias
 from litellm.types.guardrails import GuardrailEventHooks
 from litellm.types.router import RouterRateLimitError
+
+_LateResponseT = TypeVar("_LateResponseT", bound=Response)
+_LlmCallT = TypeVar("_LlmCallT")
+
+ProxyRouteType: TypeAlias = Literal[
+    "acompletion",
+    "aembedding",
+    "aresponses",
+    "_arealtime",
+    "_aresponses_websocket",
+    "acreate_realtime_client_secret",
+    "arealtime_calls",
+    "aget_responses",
+    "adelete_responses",
+    "acancel_responses",
+    "acompact_responses",
+    "acreate_batch",
+    "aretrieve_batch",
+    "alist_batches",
+    "acancel_batch",
+    "afile_content",
+    "afile_retrieve",
+    "afile_delete",
+    "atext_completion",
+    "acreate_fine_tuning_job",
+    "acancel_fine_tuning_job",
+    "alist_fine_tuning_jobs",
+    "aretrieve_fine_tuning_job",
+    "alist_input_items",
+    "aimage_edit",
+    "agenerate_content",
+    "agenerate_content_stream",
+    "allm_passthrough_route",
+    "avector_store_search",
+    "avector_store_create",
+    "avector_store_retrieve",
+    "avector_store_list",
+    "avector_store_update",
+    "avector_store_delete",
+    "avector_store_file_create",
+    "avector_store_file_list",
+    "avector_store_file_retrieve",
+    "avector_store_file_content",
+    "avector_store_file_update",
+    "avector_store_file_delete",
+    "aocr",
+    "asearch",
+    "avideo_generation",
+    "avideo_list",
+    "avideo_status",
+    "avideo_content",
+    "avideo_remix",
+    "avideo_create_character",
+    "avideo_get_character",
+    "avideo_edit",
+    "avideo_extension",
+    "acreate_container",
+    "alist_containers",
+    "aingest",
+    "aretrieve_container",
+    "adelete_container",
+    "aupload_container_file",
+    "alist_container_files",
+    "aretrieve_container_file",
+    "adelete_container_file",
+    "aretrieve_container_file_content",
+    "acreate_skill",
+    "alist_skills",
+    "aget_skill",
+    "adelete_skill",
+    "anthropic_messages",
+    "acreate_interaction",
+    "aget_interaction",
+    "adelete_interaction",
+    "acancel_interaction",
+    "acreate_agent",
+    "alist_agents",
+    "aget_agent",
+    "adelete_agent",
+    "alist_agent_versions",
+    "asend_message",
+    "call_mcp_tool",
+    "acreate_eval",
+    "alist_evals",
+    "aget_eval",
+    "aupdate_eval",
+    "adelete_eval",
+    "acancel_eval",
+    "acreate_run",
+    "alist_runs",
+    "aget_run",
+    "acancel_run",
+    "adelete_run",
+]
 from litellm.types.utils import ServerToolUse
 
 # Type alias for streaming chunk serializer (chunk after hooks + cost injection -> wire format)
@@ -559,6 +661,11 @@ class _UpstreamClosingStreamingResponse(StreamingResponse):
         super().__init__(content, status_code=status_code, headers=headers, media_type=media_type)
         self._upstream_generator = upstream_generator
 
+    @property
+    def upstream_generator(self) -> AsyncGenerator[str, None] | None:
+        """The upstream LLM stream, for a caller that has to run this response's cleanup itself."""
+        return self._upstream_generator
+
     async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
         try:
             await super().__call__(scope, receive, send)
@@ -649,6 +756,39 @@ async def _buffer_first_chunk_honoring_disconnect(
     raise _ClientDisconnectedBeforeFirstChunk()
 
 
+def _sse_error_payload(exc: BaseException) -> tuple[int, Mapping[str, object]]:
+    """Build the ProxyException-shaped ``{"error": ...}`` body used in SSE error frames.
+
+    Matches ``ProxyException.to_dict()`` so streaming and non-streaming error frames
+    are byte-identical.
+    """
+    # Preserve status code from HTTPException (e.g. guardrail blocks)
+    error_status: Final = getattr(exc, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR)
+    raw_detail: Final = _getattr_object(exc, "detail", "Error processing stream start")
+    message, structured_fields = _serialize_http_exception_detail(raw_detail)
+
+    existing_fields: Final = getattr(exc, "provider_specific_fields", None) or {}
+    merged_fields: Final = {**existing_fields, **structured_fields} if structured_fields else (existing_fields or None)
+
+    # Built in one statement then given its one optional key, rather than spread
+    # conditionally: the spread form costs two extra dict constructions, which
+    # type-discipline-budget.json's LIT002 ceiling has no room for.
+    error_obj: Final = {
+        "message": message,
+        "type": getattr(exc, "type", "None"),
+        "param": getattr(exc, "param", "None"),
+        "code": str(error_status),
+    }
+    if merged_fields:
+        error_obj["provider_specific_fields"] = merged_fields
+    return error_status, error_obj
+
+
+def _sse_error_frames(error_obj: Mapping[str, object]) -> tuple[str, str]:
+    """The two frames an SSE stream ends with once it can no longer raise."""
+    return f"data: {json.dumps({'error': error_obj})}\n\n", "data: [DONE]\n\n"
+
+
 async def create_response(
     generator: AsyncGenerator[str, None],
     media_type: str,
@@ -740,31 +880,11 @@ async def create_response(
         # Unexpected error consuming first chunk.
         verbose_proxy_logger.exception("Error consuming first chunk from generator: %s", e)
 
-        # Preserve status code from HTTPException (e.g., guardrail blocks)
-        error_status: Final = getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR)
-        raw_detail: Final = _getattr_object(e, "detail", "Error processing stream start")
-        message, structured_fields = _serialize_http_exception_detail(raw_detail)
-
-        existing_fields: Final = getattr(e, "provider_specific_fields", None) or {}
-        if structured_fields:
-            merged_fields: dict | None = {**existing_fields, **structured_fields}
-        else:
-            merged_fields = existing_fields or None
-
-        # Match ProxyException.to_dict() shape so streaming and non-streaming
-        # error frames are byte-identical.
-        error_obj: Final[dict[str, object]] = {
-            "message": message,
-            "type": getattr(e, "type", "None"),
-            "param": getattr(e, "param", "None"),
-            "code": str(error_status),
-        }
-        if merged_fields:
-            error_obj["provider_specific_fields"] = merged_fields
+        error_status, error_obj = _sse_error_payload(e)
 
         async def error_gen_message() -> AsyncGenerator[str, None]:
-            yield f"data: {json.dumps({'error': error_obj})}\n\n"
-            yield "data: [DONE]\n\n"
+            for frame in _sse_error_frames(error_obj):
+                yield frame
 
         return StreamingResponse(
             error_gen_message(),
@@ -797,6 +917,176 @@ async def create_response(
     )
 
 
+_TTFT_KEEPALIVE_HEADERS: Final[Mapping[str, str]] = MappingProxyType(
+    {
+        "Cache-Control": "no-cache",
+        "X-Accel-Buffering": "no",
+    }
+)
+
+
+def ttft_keepalive_interval(request_data: Mapping[str, object], llm_router: Router | None = None) -> float | None:
+    """The operator's keepalive interval, but only for a request that asked to stream.
+
+    Resolved through the deployments the request could land on, so a deployment's
+    `keepalive_seconds: 0` stays the hard disable it is documented to be rather
+    than being switched back on by the global default.
+    """
+    if request_data.get("stream") is not True:
+        return None
+    requested_model: Final = request_data.get("model")
+    deployments: Final = (
+        llm_router.get_model_list(model_name=requested_model) or ()
+        if llm_router is not None and isinstance(requested_model, str)
+        else ()
+    )
+    return resolve_ttft_keepalive_interval(deployments, litellm.sse_keepalive_ping_interval_seconds)
+
+
+async def _aclose_late_response(produced: Response) -> None:
+    """Run the cleanup Starlette would have run, for a response it never called.
+
+    Closing an already-closed async generator is a no-op, so this is safe to call
+    from both the relay's own teardown and the outer one.
+    """
+    if not isinstance(produced, StreamingResponse):
+        return
+    targets: Final = (
+        (produced.body_iterator, produced.upstream_generator)
+        if isinstance(produced, _UpstreamClosingStreamingResponse)
+        else (produced.body_iterator,)
+    )
+    for target in targets:
+        aclose = getattr(target, "aclose", None)
+        if aclose is None:
+            continue
+        try:
+            await aclose()
+        except BaseException as exc:  # noqa: BLE001  # teardown must not mask why the stream ended
+            verbose_proxy_logger.debug("error closing relayed streaming generator: %s", exc)
+
+
+async def _relay_late_response(produced: Response) -> AsyncGenerator[bytes, None]:
+    """Replay a Response that was built after a keepalive had already opened the wire."""
+    if not isinstance(produced, StreamingResponse):
+        # The status line is already on the wire, so a non-streaming body, an error
+        # body included, can only reach the client as an SSE frame.
+        yield b"data: " + (bytes(produced.body) or b"{}") + b"\n\n"
+        yield b"data: [DONE]\n\n"
+        return
+
+    try:
+        async for chunk in produced.body_iterator:
+            yield chunk.encode("utf-8") if isinstance(chunk, str) else bytes(chunk)
+    finally:
+        # Starlette never called this response, so the cleanup its __call__ would
+        # have run has to happen here or the upstream LLM connection leaks.
+        with anyio.CancelScope(shield=True):
+            await _aclose_late_response(produced)
+
+
+async def _sanitized_late_failure(
+    exc: Exception,
+    on_late_failure: "Callable[[Exception], Awaitable[HTTPException | None]] | None",
+) -> Exception:
+    """Report a late failure and return whatever should reach the client.
+
+    ``post_call_failure_hook`` lets a callback replace the client-facing error, by
+    returning a replacement or by raising one, and both are used elsewhere in this
+    module. Serializing the original would leak provider detail a deployment had
+    configured away, so the hook's answer wins. A callback that fails some other
+    way is a bug in the callback, not a reason to lose the real error.
+    """
+    if on_late_failure is None:
+        return exc
+    try:
+        replacement: Final = await on_late_failure(exc)
+    except HTTPException as raised_replacement:
+        return raised_replacement
+    except Exception as hook_failure:  # noqa: BLE001  # a broken callback must not replace the real error
+        verbose_proxy_logger.exception("post_call_failure_hook raised while reporting a late failure: %s", hook_failure)
+        return exc
+    return replacement if replacement is not None else exc
+
+
+async def open_sse_before_first_byte(
+    produce_response: Awaitable[_LateResponseT],
+    ping_interval_seconds: float | str | None,
+    media_type: str = "text/event-stream",
+    on_late_failure: Callable[[Exception], Awaitable[HTTPException | None]] | None = None,
+) -> _LateResponseT | StreamingResponse:
+    """Write SSE keepalive comments while the upstream LLM call is still in flight.
+
+    The whole time-to-first-token is spent inside `produce_response`: the upstream
+    withholds its response headers until it emits its first token, so nothing has
+    entered the ASGI response phase yet and the proxy writes zero bytes. An
+    intermediary with an idle read timeout (AWS ALB and nginx both default to 60s)
+    then drops a connection that is perfectly healthy.
+
+    When `produce_response` does not finish within one interval, the response is
+    opened immediately and `: ping` comments, which every conformant SSE client
+    ignores, fill the wire until the real response is ready to be replayed onto it.
+    Committing the status line that early is the cost: a failure discovered after
+    the first ping reaches the client as an SSE error frame under a 200 rather than
+    as an HTTP error status, and LiteLLM's own `x-litellm-*` response headers are
+    not yet known. Both are why this stays off until an operator sets an interval.
+    """
+    interval: Final = coerce_keepalive_interval(ping_interval_seconds)
+    if interval is None:
+        return await produce_response
+
+    produce_task: Final = asyncio.ensure_future(produce_response)
+    await asyncio.wait((produce_task,), timeout=interval)
+    if produce_task.done():
+        # Fast path: the upstream answered inside one interval, so nothing was
+        # written early and this is byte-identical to not being wrapped at all.
+        return produce_task.result()
+
+    async def keepalive_then_relay() -> AsyncGenerator[bytes, None]:
+        try:
+            while not produce_task.done():
+                yield SSE_COMMENT_PING_BYTES
+                await asyncio.wait((produce_task,), timeout=interval)
+            try:
+                produced: Final = produce_task.result()
+            except Exception as exc:  # noqa: BLE001  # the status line is already sent; surface it as a frame
+                verbose_proxy_logger.exception(
+                    "request failed after its SSE keepalive had opened the response: %s", exc
+                )
+                # The caller's own `except` never sees this, so its failure hook
+                # would never fire and the failure would go unaudited. The hook
+                # also gets to sanitize what reaches the client, by returning or
+                # raising a replacement, so its answer decides the frame.
+                _, error_obj = _sse_error_payload(await _sanitized_late_failure(exc, on_late_failure))
+                for frame in _sse_error_frames(error_obj):
+                    yield frame.encode()
+                return
+            async for chunk in _relay_late_response(produced):
+                yield chunk
+        finally:
+            if not produce_task.done():
+                produce_task.cancel()
+                with anyio.CancelScope(shield=True):
+                    with contextlib.suppress(BaseException):
+                        await produce_task
+            elif not produce_task.cancelled():
+                # The upstream may have answered while nobody was draining this
+                # relay, e.g. the client vanished first. Nothing else holds that
+                # response, so its stream only gets closed here.
+                with anyio.CancelScope(shield=True):
+                    with contextlib.suppress(BaseException):
+                        await _aclose_late_response(produce_task.result())
+
+    verbose_proxy_logger.info(
+        "no upstream response after %ss, opening the SSE response early and sending keepalives", interval
+    )
+    return StreamingResponse(
+        keepalive_then_relay(),
+        media_type=media_type,
+        headers=_TTFT_KEEPALIVE_HEADERS,
+    )
+
+
 def _is_azure_model_router_request(model: str) -> bool:
     """
     Check if the requested model is an Azure Model Router.
@@ -1043,7 +1333,7 @@ def _log_llm_api_exception(e: Exception) -> None:
 
 async def _cancel_llm_call_on_client_disconnect(
     request: Request,
-    llm_api_call: "asyncio.Future[object]",
+    llm_api_call: "asyncio.Future[_LlmCallT]",
     disconnect_event: asyncio.Event,
 ) -> None:
     try:
@@ -1062,8 +1352,8 @@ async def _cancel_llm_call_on_client_disconnect(
 
 async def _await_llm_call_cancelling_on_disconnect(
     request: Request,
-    llm_api_call: "asyncio.Future[Any]",
-) -> Any:
+    llm_api_call: "asyncio.Future[_LlmCallT]",
+) -> _LlmCallT:
     disconnect_event: Final = asyncio.Event()
     monitor: Final = asyncio.create_task(_cancel_llm_call_on_client_disconnect(request, llm_api_call, disconnect_event))
     try:
@@ -1714,100 +2004,11 @@ class ProxyBaseLLMRequestProcessing:
         request: Request,
         fastapi_response: Response,
         user_api_key_dict: UserAPIKeyAuth,
-        route_type: Literal[
-            "acompletion",
-            "aembedding",
-            "aresponses",
-            "_arealtime",
-            "_aresponses_websocket",
-            "acreate_realtime_client_secret",
-            "arealtime_calls",
-            "aget_responses",
-            "adelete_responses",
-            "acancel_responses",
-            "acompact_responses",
-            "acreate_batch",
-            "aretrieve_batch",
-            "alist_batches",
-            "acancel_batch",
-            "afile_content",
-            "afile_retrieve",
-            "afile_delete",
-            "atext_completion",
-            "acreate_fine_tuning_job",
-            "acancel_fine_tuning_job",
-            "alist_fine_tuning_jobs",
-            "aretrieve_fine_tuning_job",
-            "alist_input_items",
-            "aimage_edit",
-            "agenerate_content",
-            "agenerate_content_stream",
-            "allm_passthrough_route",
-            "avector_store_search",
-            "avector_store_create",
-            "avector_store_retrieve",
-            "avector_store_list",
-            "avector_store_update",
-            "avector_store_delete",
-            "avector_store_file_create",
-            "avector_store_file_list",
-            "avector_store_file_retrieve",
-            "avector_store_file_content",
-            "avector_store_file_update",
-            "avector_store_file_delete",
-            "aocr",
-            "asearch",
-            "avideo_generation",
-            "avideo_list",
-            "avideo_status",
-            "avideo_content",
-            "avideo_remix",
-            "avideo_create_character",
-            "avideo_get_character",
-            "avideo_edit",
-            "avideo_extension",
-            "acreate_container",
-            "alist_containers",
-            "aingest",
-            "aretrieve_container",
-            "adelete_container",
-            "aupload_container_file",
-            "alist_container_files",
-            "aretrieve_container_file",
-            "adelete_container_file",
-            "aretrieve_container_file_content",
-            "acreate_skill",
-            "alist_skills",
-            "aget_skill",
-            "adelete_skill",
-            "anthropic_messages",
-            "acreate_interaction",
-            "aget_interaction",
-            "adelete_interaction",
-            "acancel_interaction",
-            "acreate_agent",
-            "alist_agents",
-            "aget_agent",
-            "adelete_agent",
-            "alist_agent_versions",
-            "asend_message",
-            "call_mcp_tool",
-            "acreate_eval",
-            "alist_evals",
-            "aget_eval",
-            "aupdate_eval",
-            "adelete_eval",
-            "acancel_eval",
-            "acreate_run",
-            "alist_runs",
-            "aget_run",
-            "acancel_run",
-            "adelete_run",
-        ],
+        route_type: ProxyRouteType,
         proxy_logging_obj: ProxyLogging,
-        general_settings: dict,
+        general_settings: dict[str, object],
         proxy_config: ProxyConfig,
-        select_data_generator: Callable | None = None,
+        select_data_generator: Callable[..., object] | None = None,
         llm_router: Router | None = None,
         model: str | None = None,
         user_model: str | None = None,
@@ -1817,7 +2018,72 @@ class ProxyBaseLLMRequestProcessing:
         user_api_base: str | None = None,
         version: str | None = None,
         is_streaming_request: bool | None = False,
-        contents: list | None = None,  # Add contents parameter
+        contents: list[object] | None = None,
+        skip_pre_call_logic: bool = False,
+    ) -> Any:
+        """Run the request, sending SSE keepalives while the upstream is still silent.
+
+        Everything below this point, the upstream call included, happens before the
+        proxy can write a byte, so a slow time-to-first-token leaves the response
+        idle. See ``open_sse_before_first_byte``; unwrapped unless an operator sets
+        ``litellm_settings.sse_keepalive_ping_interval_seconds``.
+        """
+
+        async def _audit_late_failure(exc: Exception) -> HTTPException | None:
+            # Once a keepalive is on the wire this can no longer raise, so the
+            # caller's `except` never runs its own post_call_failure_hook.
+            return await proxy_logging_obj.post_call_failure_hook(
+                user_api_key_dict=user_api_key_dict,
+                original_exception=exc,
+                request_data=self.data,
+            )
+
+        return await open_sse_before_first_byte(
+            self._process_llm_request(
+                request=request,
+                fastapi_response=fastapi_response,
+                user_api_key_dict=user_api_key_dict,
+                route_type=route_type,
+                proxy_logging_obj=proxy_logging_obj,
+                general_settings=general_settings,
+                proxy_config=proxy_config,
+                select_data_generator=select_data_generator,
+                llm_router=llm_router,
+                model=model,
+                user_model=user_model,
+                user_temperature=user_temperature,
+                user_request_timeout=user_request_timeout,
+                user_max_tokens=user_max_tokens,
+                user_api_base=user_api_base,
+                version=version,
+                is_streaming_request=is_streaming_request,
+                contents=contents,
+                skip_pre_call_logic=skip_pre_call_logic,
+            ),
+            ping_interval_seconds=ttft_keepalive_interval(self.data, llm_router),
+            on_late_failure=_audit_late_failure,
+        )
+
+    async def _process_llm_request(
+        self,
+        request: Request,
+        fastapi_response: Response,
+        user_api_key_dict: UserAPIKeyAuth,
+        route_type: ProxyRouteType,
+        proxy_logging_obj: ProxyLogging,
+        general_settings: dict[str, object],
+        proxy_config: ProxyConfig,
+        select_data_generator: Callable[..., object] | None = None,
+        llm_router: Router | None = None,
+        model: str | None = None,
+        user_model: str | None = None,
+        user_temperature: float | None = None,
+        user_request_timeout: float | None = None,
+        user_max_tokens: int | None = None,
+        user_api_base: str | None = None,
+        version: str | None = None,
+        is_streaming_request: bool | None = False,
+        contents: list[object] | None = None,  # Add contents parameter
         skip_pre_call_logic: bool = False,
     ) -> Any:
         """
@@ -2039,6 +2305,7 @@ class ProxyBaseLLMRequestProcessing:
                         return StreamingResponse(
                             content=generator,
                             status_code=status.HTTP_200_OK,
+                            media_type=self._passthrough_event_stream_media_type(),
                             headers=custom_headers,
                         )
                     else:
@@ -2197,11 +2464,21 @@ class ProxyBaseLLMRequestProcessing:
         additional_headers = hidden_params.get("additional_headers", {}) or {}
 
         recover_response_cost: Final = not response_cost and hidden_params.get("response_cost") is None
-        response_cost_for_headers: Final = (
+        llm_cost_for_headers: Final = (
             self._response_cost_from_logging_obj(response=response, logging_obj=logging_obj) or ""
             if recover_response_cost
             else response_cost
         )
+        _, request_metadata_bucket = get_or_create_metadata_bucket(self.data)
+        guardrail_cost_for_headers: Final = guardrail_information_cost(
+            request_metadata_bucket.get("standard_logging_guardrail_information")
+        )
+        response_cost_for_headers: Final = (
+            (llm_cost_for_headers if isinstance(llm_cost_for_headers, (int, float)) else 0.0)
+            + guardrail_cost_for_headers
+            if guardrail_cost_for_headers > 0
+            else llm_cost_for_headers
+        )
 
         fastapi_response.headers.update(
             ProxyBaseLLMRequestProcessing.get_custom_headers(
@@ -2494,10 +2771,16 @@ class ProxyBaseLLMRequestProcessing:
 
     def _passthrough_event_stream_media_type(self) -> str | None:
         """
-        Content-type for a buffered passthrough event-stream response, resolved
-        from the provider handler so the proxy stays provider-agnostic. Mirrors
-        the upstream content-type the non-streaming path forwards, since the
-        buffered streaming generator carries no headers of its own.
+        Content-type for a passthrough event-stream response, resolved from the
+        provider handler so the proxy stays provider-agnostic. Mirrors the
+        upstream content-type the non-streaming path forwards, since the
+        streaming generator carries no headers of its own. Used for both the
+        buffered (guardrail-rewritten) and the unbuffered relay paths so
+        clients that enforce the event-stream content-type (e.g. Claude Code on
+        Bedrock invoke-with-response-stream) see the correct header instead of
+        no content-type at all, which they fall back to reading as
+        application/octet-stream. Returns None for providers with no
+        event-stream media type, leaving the response headers unchanged.
         """
         from litellm.llms.pass_through.guardrail_translation.handler import (
             LlmPassthroughRouteHandler,
@@ -2669,10 +2952,10 @@ class ProxyBaseLLMRequestProcessing:
         streaming pipeline (including unified_guardrail end-of-stream blocks)
         has completed.
 
-        Guardrails with apply_guardrail are skipped — they already ran via
-        unified_guardrail's streaming iterator.  Only guardrails that override
-        async_post_call_success_hook directly (without apply_guardrail) run
-        here.
+        Guardrails routed through unified_guardrail are skipped, since they already ran
+        via its streaming iterator.  Guardrails that override
+        async_post_call_success_hook directly run here, including those that implement
+        apply_guardrail but keep their native lifecycle hooks.
 
         This is audit-only — content has already been delivered to the client.
 
@@ -2695,8 +2978,8 @@ class ProxyBaseLLMRequestProcessing:
                     continue
                 try:
                     guardrail_result = None
-                    if "apply_guardrail" in type(cb).__dict__:
-                        # Skip — apply_guardrail guardrails already ran via
+                    if "apply_guardrail" in type(cb).__dict__ and not cb.use_native_lifecycle_hooks:
+                        # Skip — unified-routed guardrails already ran via
                         # unified_guardrail's end-of-stream block in the
                         # streaming iterator pipeline.  Running them again
                         # here would duplicate the guardrail API call

litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py

@@ -1,5 +1,6 @@
 import asyncio
 import json
+from collections.abc import Sequence
 from dataclasses import asdict, dataclass
 from typing import TYPE_CHECKING, Final
 
@@ -72,6 +73,27 @@ async def publish_auth_cache_invalidation(cache_key: str) -> None:
         verbose_proxy_logger.warning("auth cache invalidation publish for %s failed: %s", cache_key, e)
 
 
+async def evict_and_broadcast(cache_keys: Sequence[str], user_api_key_cache: "UserApiKeyCache") -> None:
+    """
+    Drop cached management objects here and on every other worker.
+
+    Every endpoint that mutates a cached object must call this: auth serves those objects
+    cache-first with no freshness check, so a mutation that leaves the entry in place keeps the
+    stale object enforced until its TTL expires (LIT-3803). Best-effort on both steps: the DB write
+    has already committed, so a cache backend error must not fail the endpoint.
+    """
+    for cache_key in cache_keys:
+        try:
+            await user_api_key_cache.async_delete_cache(key=cache_key)
+        except Exception as e:  # noqa: BLE001  # best-effort eviction: any cache backend error must not fail the mutation
+            verbose_proxy_logger.warning(
+                "Failed to evict cached entry %s; a stale object may be served until its TTL expires: %s",
+                cache_key,
+                e,
+            )
+        await publish_auth_cache_invalidation(cache_key=cache_key)
+
+
 class AuthCacheInvalidationSubscriber:
     __slots__ = ("_redis_cache", "_task", "_user_api_key_cache")