Merge remote-tracking branch 'berri/litellm_internal_staging' into litellm_managed_batches_observability

# Conflicts:
#	tests/test_litellm/batches/test_batch_utils.py

basedpyright-code-budget.json

@@ -57,7 +57,7 @@
     "limit": 5681
   },
   "reportMissingTypeArgument": {
-    "limit": 15608
+    "limit": 15605
   },
   "reportMissingTypeStubs": {
     "limit": 40
@@ -108,7 +108,7 @@
     "limit": 39154
   },
   "reportUnknownParameterType": {
-    "limit": 19947
+    "limit": 19944
   },
   "reportUnknownVariableType": {
     "limit": 30772
@@ -132,7 +132,7 @@
     "limit": 27
   },
   "reportUnusedClass": {
-    "limit": 23
+    "limit": 21
   },
   "reportUnusedFunction": {
     "limit": 139

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.56"
+version = "0.1.57"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.56"
+version = "0.1.57"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.86"
+version = "0.4.87"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.86"
+version = "0.4.87"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm/_logging.py

@@ -10,7 +10,7 @@ from typing import Any, Final
 import litellm
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
-from litellm.litellm_core_utils.secret_redaction import redact_string
+from litellm.litellm_core_utils.secret_redaction import redact_string, redact_structured_value
 
 set_verbose = False
 
@@ -59,6 +59,12 @@ def _redact_string(value: str) -> str:
     return redact_string(value)
 
 
+def _redact_structured_value(key: str | None, value: str) -> str:
+    if not _ENABLE_SECRET_REDACTION:
+        return value
+    return redact_structured_value(key, value)
+
+
 def redact_secrets(value: str) -> str:
     """Public API: redact known secret/credential patterns from an arbitrary string.
 
@@ -265,7 +271,7 @@ class JsonFormatter(Formatter):
         if record.exc_info:
             json_record["stacktrace"] = record.exc_text or self.formatException(record.exc_info)
 
-        return safe_dumps(json_record)
+        return safe_dumps(json_record, value_transform=_redact_structured_value)
 
 
 class CorrelationPlainFormatter(logging.Formatter):
@@ -276,7 +282,7 @@ class CorrelationPlainFormatter(logging.Formatter):
     """
 
     def format(self, record: logging.LogRecord) -> str:
-        formatted: Final = super().format(record)
+        formatted: Final = _redact_string(super().format(record))
         trace_id: Final = getattr(record, "trace_id", None)
         session_id: Final = getattr(record, "session_id", None)
         if not trace_id and not session_id:

litellm/integrations/opentelemetry.py

@@ -20,6 +20,7 @@ from litellm.integrations.opentelemetry_utils.gen_ai_semconv import (
     OTELSemconvCategory,
     parse_semconv_opt_in,
 )
+from litellm.integrations.otel.model.db_endpoint import db_span_attributes
 from litellm.integrations.otel.model.semconv import Metric
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.litellm_core_utils.secret_redaction import redact_string
@@ -718,6 +719,28 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
     async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
         self._handle_failure(kwargs, response_obj, start_time, end_time)
 
+    def _start_service_span(self, payload: ServiceLoggerPayload, parent_otel_span: Span, start_time_ns: int) -> Span:
+        """Open a service span, named and classified by what the service is.
+
+        A datastore call is an outbound CLIENT span carrying ``db.*`` semconv.
+        Without those a Postgres span says only ``service=postgres``, so the
+        backend falls back to the transport peer, which for Prisma is the local
+        query engine on loopback. Everything else stays an INTERNAL span.
+        """
+        from opentelemetry import trace
+        from opentelemetry.trace import SpanKind
+
+        attributes: Final = db_span_attributes(payload.service.value, payload.call_type)
+        span: Final = self.tracer.start_span(
+            name=payload.service,
+            context=trace.set_span_in_context(parent_otel_span),
+            start_time=start_time_ns,
+            kind=SpanKind.CLIENT if attributes else SpanKind.INTERNAL,
+        )
+        for key, value in attributes.items():
+            self.safe_set_attribute(span=span, key=key, value=value)
+        return span
+
     async def async_service_success_hook(
         self,
         payload: ServiceLoggerPayload,
@@ -726,7 +749,6 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
         end_time: datetime | float | None = None,
         event_metadata: dict | None = None,
     ):
-        from opentelemetry import trace
         from opentelemetry.trace import Status, StatusCode
 
         _start_time_ns = 0
@@ -743,12 +765,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
             _end_time_ns = self._to_ns(end_time)
 
         if parent_otel_span is not None:
-            _span_name: Final = payload.service
-            service_logging_span: Final = self.tracer.start_span(
-                name=_span_name,
-                context=trace.set_span_in_context(parent_otel_span),
-                start_time=_start_time_ns,
-            )
+            service_logging_span: Final = self._start_service_span(payload, parent_otel_span, _start_time_ns)
             self.safe_set_attribute(
                 span=service_logging_span,
                 key="call_type",
@@ -786,7 +803,6 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
         end_time: float | datetime | None = None,
         event_metadata: dict | None = None,
     ):
-        from opentelemetry import trace
         from opentelemetry.trace import Status, StatusCode
 
         _start_time_ns = 0
@@ -803,12 +819,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
             _end_time_ns = self._to_ns(end_time)
 
         if parent_otel_span is not None:
-            _span_name: Final = payload.service
-            service_logging_span: Final = self.tracer.start_span(
-                name=_span_name,
-                context=trace.set_span_in_context(parent_otel_span),
-                start_time=_start_time_ns,
-            )
+            service_logging_span: Final = self._start_service_span(payload, parent_otel_span, _start_time_ns)
             self.safe_set_attribute(
                 span=service_logging_span,
                 key="call_type",

litellm/integrations/otel/mappers/genai.py

@@ -18,6 +18,7 @@ from litellm.integrations.otel.mappers.utils import (
     serialize_messages,
     tool_definition_attrs,
 )
+from litellm.integrations.otel.model.db_endpoint import db_span_attributes
 from litellm.integrations.otel.model.payloads import (
     GuardrailSpanData,
     LLMCallSpanData,
@@ -27,7 +28,6 @@ from litellm.integrations.otel.model.payloads import (
     ToolDefinition,
 )
 from litellm.integrations.otel.model.semconv import (
-    DB,
     MCP,
     Error,
     GenAI,
@@ -36,7 +36,6 @@ from litellm.integrations.otel.model.semconv import (
     RpcSystem,
     Server,
 )
-from litellm.integrations.otel.model.spans import db_system
 
 
 class GenAIMapper:
@@ -182,12 +181,8 @@ class GenAIMapper:
     def _service(cls, data: ServiceSpanData) -> AttributeMap:
         attrs: Final = collect(cls._SERVICE_ATTRS, data)
         # An outbound datastore call (DB_CALL / CLIENT span) also carries db.*
-        # semconv. Internal services (router, budget jobs, …) have no db.system,
-        # so they get only the litellm.service.* keys above.
-        system: Final = db_system(data.service_name)
-        if system is not None:
-            attrs[DB.SYSTEM_NAME] = system
-            if data.call_type:
-                attrs[DB.OPERATION_NAME] = data.call_type
+        # semconv naming the server it reached. Internal services (router, budget
+        # jobs, …) have no db.system, so they get only the litellm.service.* keys.
+        attrs.update(db_span_attributes(data.service_name, data.call_type))
         attrs.update({f"{LiteLLM.METADATA_PREFIX}{key}": value for key, value in data.event_metadata.items()})
         return attrs

litellm/integrations/otel/model/db_endpoint.py

@@ -0,0 +1,164 @@
+"""OTel ``db.*`` / ``server.*`` attributes naming the database litellm talks to.
+
+Prisma reaches PostgreSQL through a query engine listening on loopback, so
+transport-level instrumentation attributes the work to ``localhost`` and an
+operator cannot tell it is a PostgreSQL call or correlate it with the database's
+own metrics. These attributes name the real server on litellm's DB spans.
+
+Only the host, port, database and schema of the DSN are read, so no credential
+can reach an exporter.
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import Final
+from urllib.parse import ParseResult, parse_qs, unquote, urlparse
+
+from litellm.integrations.otel.model.semconv import DB, Server
+from litellm.integrations.otel.model.spans import POSTGRESQL, db_system
+
+_DATABASE_URL_ENV: Final = "DATABASE_URL"
+_READ_REPLICA_ENV: Final = "DATABASE_URL_READ_REPLICA"
+_DEFAULT_POSTGRES_PORT: Final = 5432
+_DEFAULT_POSTGRES_SCHEMA: Final = "public"
+_POSTGRES_SCHEMES: Final = frozenset({"postgres", "postgresql"})
+_EMPTY_ATTRIBUTES: Final[Mapping[str, str | int]] = MappingProxyType({})
+
+
+@dataclass(frozen=True, slots=True)
+class DatabaseEndpoint:
+    """The non-sensitive identity of a PostgreSQL server, parsed from a DSN."""
+
+    address: str | None
+    port: int | None
+    namespace: str | None
+
+
+def parse_database_endpoint(url: str | None) -> DatabaseEndpoint | None:
+    """Parse a PostgreSQL DSN into its exportable endpoint identity.
+
+    Returns ``None`` for an absent, malformed or non-PostgreSQL URL rather than
+    raising: an unparseable DSN must degrade to a span without endpoint
+    attributes, never break the request that emitted it.
+    """
+    if not url:
+        return None
+    try:
+        parsed: Final = urlparse(url)
+        if parsed.scheme not in _POSTGRES_SCHEMES:
+            return None
+        query: Final = parse_qs(parsed.query)
+        raw_database: Final = (parsed.path or "").lstrip("/")
+        if _is_misparsed_authority(parsed, url, raw_database):
+            return None
+        # ``host=`` beats the netloc: it is how libpq names a Unix socket
+        # directory and how the Cloud SQL connector sits behind a localhost
+        # netloc, where the netloc is the very answer this module replaces.
+        address: Final = _first(query.get("host")) or parsed.hostname
+        # ``port=`` accompanies ``host=`` in a libpq URI, so honour it the same way.
+        port: Final = _port(_first(query.get("port")), parsed.port) if address else None
+        namespace: Final = _namespace(unquote(raw_database), _first(query.get("schema")))
+    except ValueError:
+        return None
+    if address is None and namespace is None:
+        return None
+    return DatabaseEndpoint(address=address, port=port, namespace=namespace)
+
+
+def _is_misparsed_authority(parsed: ParseResult, url: str, raw_database: str) -> bool:
+    """Whether the URL authority may have been truncated by an unencoded character.
+
+    ``/``, ``#`` or ``?`` in a password ends the netloc early, so urlparse hands
+    back the username as the host, the leading digits of the password as the
+    port, and the rest of the credential as the path, query or fragment. The
+    stranded userinfo ``@`` is the only surviving evidence.
+
+    A database name cannot hold an unencoded slash either, so a second path
+    segment is the same evidence.
+
+    A DSN that carries the at-sign in a query parameter instead, such as
+    ``?application_name=svc@prod``, is indistinguishable from a mis-split by any
+    property of the parse: both leave no userinfo, a host, a port and a path.
+    Since guessing wrong publishes a credential fragment to a tracing backend,
+    that ambiguity resolves to refusing the endpoint. Such a DSN loses
+    ``server.address`` and ``db.namespace`` and keeps the rest of the span,
+    which is the cheaper error of the two. Percent-encode the at-sign to keep
+    them.
+    """
+    if "/" in raw_database:
+        return True
+    return "@" in url and "@" not in parsed.netloc
+
+
+def _first(values: Sequence[str] | None) -> str:
+    return values[0] if values else ""
+
+
+def _port(from_query: str, from_netloc: int | None) -> int:
+    return int(from_query) if from_query.isdigit() else (from_netloc or _DEFAULT_POSTGRES_PORT)
+
+
+def _namespace(database: str, schema: str) -> str | None:
+    """``{database}|{schema}`` per the PostgreSQL semconv, dropping absent halves.
+
+    Only Prisma's literal default schema stays implicit. The match is
+    case-sensitive because Prisma quotes the name, so ``?schema=PUBLIC`` builds
+    a second schema alongside ``public`` and the two must not collapse to one
+    namespace.
+    """
+    qualifier: Final = "" if schema == _DEFAULT_POSTGRES_SCHEMA else schema
+    return "|".join(part for part in (database, qualifier) if part) or None
+
+
+def postgres_endpoint() -> DatabaseEndpoint | None:
+    """The PostgreSQL endpoint the process is currently connected to.
+
+    Read from ``os.environ`` on every span, deliberately, on both counts.
+
+    The environment is what Prisma itself connects with, so the span cannot
+    disagree with the connection; ``get_secret_str`` would consult a configured
+    secret manager first and could name a different server than the one serving
+    the query. And the value is not static: the RDS IAM refresh rebuilds the URL
+    from ``DATABASE_HOST``/``PORT``/``NAME``/``SCHEMA`` every rotation, the
+    reconnect path re-reads ``DATABASE_URL``, and the DB-backed
+    ``environment_variables`` config overlay can rewrite any of them after
+    startup, so a value cached for the process lifetime goes stale against a
+    connection that has genuinely moved. Nothing is memoized either: a cache
+    keyed on the URL would hold a rotated credential past its rotation, and the
+    parse is a single ``urlparse`` on a short string.
+
+    A configured read replica yields ``None``: ``RoutingPrismaWrapper`` picks
+    reader or writer per Prisma call, underneath the span, so naming the writer
+    would attribute replica reads to the primary.
+    """
+    if os.environ.get(_READ_REPLICA_ENV):
+        return None
+    return parse_database_endpoint(os.environ.get(_DATABASE_URL_ENV, ""))
+
+
+def db_span_attributes(service_name: str, call_type: str | None = None) -> Mapping[str, str | int]:
+    """The ``db.*``/``server.*`` attributes for a datastore service call.
+
+    Empty for services that are not outbound datastore calls. Endpoint
+    attributes are PostgreSQL-only: ``DATABASE_URL`` says nothing about where
+    the redis-backed services point. ``db.system`` rides alongside the current
+    ``db.system.name`` because Datadog's OTLP intake still types a database span
+    from the older key.
+    """
+    system: Final = db_system(service_name)
+    if system is None:
+        return _EMPTY_ATTRIBUTES
+    endpoint: Final = postgres_endpoint() if system == POSTGRESQL else None
+    pairs: Final[tuple[tuple[str, str | int | None], ...]] = (
+        (DB.SYSTEM_NAME, system),
+        (DB.SYSTEM_LEGACY, system),
+        (DB.OPERATION_NAME, call_type),
+        (Server.ADDRESS, endpoint.address if endpoint is not None else None),
+        (Server.PORT, endpoint.port if endpoint is not None else None),
+        (DB.NAMESPACE, endpoint.namespace if endpoint is not None else None),
+    )
+    return MappingProxyType({key: value for key, value in pairs if value})

litellm/integrations/otel/model/semconv.py

@@ -238,7 +238,11 @@ class DB:
     """
 
     SYSTEM_NAME: Final = "db.system.name"
+    # Superseded by SYSTEM_NAME, dual-emitted because Datadog's OTLP intake
+    # still infers a span's database type from this key.
+    SYSTEM_LEGACY: Final = "db.system"
     OPERATION_NAME: Final = "db.operation.name"
+    NAMESPACE: Final = "db.namespace"
 
 
 class HTTP:

litellm/integrations/otel/model/spans.py

@@ -115,10 +115,12 @@ SPAN_REGISTRY: Final[dict[SpanRole, SpanSpec]] = {
 # redis-backed spend queues. Any service not mapped here is litellm-internal work
 # and stays an INTERNAL ``SERVICE`` span. This table is the single source of
 # datastore knowledge — both the role classifier and the mapper read it.
+POSTGRESQL: Final = "postgresql"
+
 _DB_SYSTEM_BY_SERVICE: Final[dict[str, str]] = {
     "redis": "redis",
-    "postgres": "postgresql",
-    "batch_write_to_db": "postgresql",
+    "postgres": POSTGRESQL,
+    "batch_write_to_db": POSTGRESQL,
 }
 
 

litellm/litellm_core_utils/litellm_logging.py

@@ -1094,10 +1094,10 @@ class Logging(LiteLLMLoggingBaseClass):
                             data=additional_args.get("complete_input_dict", {}),
                         )
 
-                        _metadata["raw_request"] = str(curl_command)
+                        _metadata["raw_request"] = _redact_string(str(curl_command))
                         # split up, so it's easier to parse in the UI
                         self.model_call_details["raw_request_typed_dict"] = RawRequestTypedDict(
-                            raw_request_api_base=str(additional_args.get("api_base") or ""),
+                            raw_request_api_base=self._get_masked_api_base(str(additional_args.get("api_base") or "")),
                             raw_request_body=self._get_raw_request_body(additional_args.get("complete_input_dict", {})),
                             # NOTE: setting ignore_sensitive_headers to True will cause
                             # the Authorization header to be leaked when calls to the health
@@ -1111,8 +1111,10 @@ class Logging(LiteLLMLoggingBaseClass):
                     self.model_call_details["raw_request_typed_dict"] = RawRequestTypedDict(
                         error=str(e),
                     )
-                    _metadata["raw_request"] = f"Unable to Log \
+                    _metadata["raw_request"] = _redact_string(
+                        f"Unable to Log \
                         raw request: {e}"
+                    )
             if getattr(self, "logger_fn", None) and callable(self.logger_fn):
                 try:
                     self.logger_fn(
@@ -1206,15 +1208,16 @@ class Logging(LiteLLMLoggingBaseClass):
         if _is_debugging_on() or self.litellm_request_debug:
             if json_logs:
                 masked_headers: Final = self._get_masked_headers(headers)
+                masked_api_base: Final = self._get_masked_api_base(str(api_base or ""))
                 if self.litellm_request_debug:
                     verbose_logger.warning(  # .warning ensures this shows up in all environments
                         "POST Request Sent from LiteLLM",
-                        extra={"api_base": {api_base}, **masked_headers},
+                        extra={"api_base": {masked_api_base}, **masked_headers},
                     )
                 else:
                     verbose_logger.debug(
                         "POST Request Sent from LiteLLM",
-                        extra={"api_base": {api_base}, **masked_headers},
+                        extra={"api_base": {masked_api_base}, **masked_headers},
                     )
             else:
                 headers = additional_args.get("headers", {})
@@ -1254,8 +1257,6 @@ class Logging(LiteLLMLoggingBaseClass):
             curl_command = "\nRequest Sent from LiteLLM:\n"
             request_str: Final = additional_args.get("request_str", "")
             curl_command += request_str
-        elif api_base == "":
-            curl_command = str(self.model_call_details)
         return curl_command
 
     def _get_masked_headers(self, headers: dict, ignore_sensitive_headers: bool = False) -> dict:

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -1816,16 +1816,19 @@ def split_concatenated_json_objects(raw: str) -> list[dict[str, Any]]:
     This helper uses ``json.JSONDecoder.raw_decode()`` to walk the string
     and extract each JSON object individually.
 
+    The walk degrades gracefully: if the string is malformed or truncated
+    (e.g. a stream that ended mid-tool-call), whatever complete objects were
+    parsed before the bad tail are returned and the remainder is discarded
+    with a warning, rather than raising.  The sole caller
+    (``_convert_to_bedrock_tool_call_invoke``) treats an empty result as
+    ``input={}`` so the conversation can continue instead of hard-failing.
+
     Returns
     -------
     list[dict]
         A list of parsed dicts – one per JSON object found.  If *raw* is
-        empty or whitespace-only, an empty list is returned.
-
-    Raises
-    ------
-    json.JSONDecodeError
-        If the string contains text that cannot be parsed as JSON at all.
+        empty, whitespace-only, or wholly unparseable, an empty list is
+        returned.
     """
     import json
 
@@ -1845,7 +1848,17 @@ def split_concatenated_json_objects(raw: str) -> list[dict[str, Any]]:
         if idx >= length:
             break
 
-        obj, end_idx = decoder.raw_decode(raw, idx)
+        try:
+            obj, end_idx = decoder.raw_decode(raw, idx)
+        except json.JSONDecodeError as e:
+            verbose_logger.warning(
+                "split_concatenated_json_objects: discarding unparseable tool-call "
+                "arguments tail after %d complete object(s); decode_start=%d error=%s",
+                len(results),
+                idx,
+                e,
+            )
+            break
         if isinstance(obj, dict):
             results.append(obj)
         else:

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -3712,7 +3712,13 @@ def _convert_to_bedrock_tool_call_invoke(
                         _parts_list.append(cache_point_block)
         return _parts_list
     except Exception as e:
-        raise Exception(f"Unable to convert openai tool calls={tool_calls} to bedrock tool calls. Received error={e}")
+        tool_call_ids: Final = tuple(tool.get("id") for tool in tool_calls if isinstance(tool, dict))
+        raise litellm.BadRequestError(
+            message=f"Unable to convert openai tool calls with ids={tool_call_ids} to bedrock tool calls. "
+            f"Received error={e}",
+            model=model or "",
+            llm_provider="bedrock",
+        ) from e
 
 
 def _append_bedrock_tool_result_media_block(

litellm/litellm_core_utils/safe_json_dumps.py

@@ -1,4 +1,5 @@
 import json
+from collections.abc import Callable
 from typing import Any, Final
 
 from pydantic import BaseModel
@@ -11,20 +12,32 @@ def strip_null_bytes(value: str) -> str:
     return value.replace("\x00", "")
 
 
-def safe_dumps(data: Any, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH) -> str:
+def safe_dumps(
+    data: Any,
+    max_depth: int = DEFAULT_MAX_RECURSE_DEPTH,
+    value_transform: Callable[[str | None, str], str] | None = None,
+) -> str:
     """
     Recursively serialize data while detecting circular references.
     If a circular reference is detected then a marker string is returned.
     NUL bytes are stripped from strings to prevent PostgreSQL 22P05 errors.
+
+    value_transform, when given, is applied to every string leaf (and to the
+    str() fallback for non-serializable objects) with the mapping key the leaf
+    was reached under, so callers can rewrite values without touching structure.
     """
 
-    def _serialize(obj: Any, seen: set, depth: int) -> Any:
+    def _transform(key: str | None, value: str) -> str:
+        return value if value_transform is None else value_transform(key, value)
+
+    def _serialize(obj: Any, seen: set, depth: int, key: str | None = None) -> Any:
         # Check for maximum depth.
         if depth > max_depth:
             return "MaxDepthExceeded"
         # Base-case: if it is a primitive, simply return it.
         if isinstance(obj, str):
-            return obj.replace("\x00", "") if "\x00" in obj else obj
+            cleaned = obj.replace("\x00", "") if "\x00" in obj else obj
+            return _transform(key, cleaned)
         if isinstance(obj, (int, float, bool, type(None))):
             return obj
         # Check for circular reference.
@@ -37,30 +50,30 @@ def safe_dumps(data: Any, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH) -> str:
             for k, v in obj.items():
                 if isinstance(k, (str)):
                     clean_k = k.replace("\x00", "") if "\x00" in k else k
-                    result[clean_k] = _serialize(v, seen, depth + 1)
+                    result[clean_k] = _serialize(v, seen, depth + 1, clean_k)
             seen.remove(id(obj))
             return result
         elif isinstance(obj, list):
-            result = [_serialize(item, seen, depth + 1) for item in obj]
+            result = [_serialize(item, seen, depth + 1, key) for item in obj]
             seen.remove(id(obj))
             return result
         elif isinstance(obj, tuple):
-            result = tuple(_serialize(item, seen, depth + 1) for item in obj)
+            result = tuple(_serialize(item, seen, depth + 1, key) for item in obj)
             seen.remove(id(obj))
             return result
         elif isinstance(obj, set):
-            result = sorted([_serialize(item, seen, depth + 1) for item in obj])
+            result = sorted([_serialize(item, seen, depth + 1, key) for item in obj])
             seen.remove(id(obj))
             return result
         elif isinstance(obj, BaseModel):
             dumped: Final = obj.model_dump()
-            result = _serialize(dumped, seen, depth + 1)
+            result = _serialize(dumped, seen, depth + 1, key)
             seen.remove(id(obj))
             return result
         else:
             # Fall back to string conversion for non-serializable objects.
             try:
-                return strip_null_bytes(str(obj))
+                return _transform(key, strip_null_bytes(str(obj)))
             except Exception:
                 return "Unserializable Object"
 

litellm/litellm_core_utils/secret_redaction.py

@@ -24,9 +24,6 @@ def _build_secret_patterns() -> "re.Pattern[str]":
         r"(?:client_secret|azure_password|azure_username)\s+[^\s,'\"})\]{}>]+",
         # AWS access key IDs
         r"(?:AKIA|ASIA)[0-9A-Z]{16}",
-        # AWS secrets / session tokens / access key IDs (key=value)
-        r"(?:aws_secret_access_key|aws_session_token|aws_access_key_id)"
-        r"\s*[:=]\s*[A-Za-z0-9/+=]{20,}",
         # Bearer tokens (OAuth, JWT, etc.)
         r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*",
         # Basic auth headers
@@ -61,6 +58,7 @@ def _build_secret_patterns() -> "re.Pattern[str]":
         # private_key with PEM-aware value capture
         r"""private_key['\"]?\s*[:=]\s*['\"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'\"})\]{}>]+)""",
         r"(?:master_key|xai_key|database_url|db_url|connection_string|"
+        r"aws_secret_access_key|aws_session_token|aws_access_key_id|"
         r"signing_key|encryption_key|"
         r"auth_token|access_token|refresh_token|"
         r"slack_webhook_url|webhook_url|"
@@ -83,3 +81,19 @@ _SECRET_RE: Final = _build_secret_patterns()
 def redact_string(value: str) -> str:
     """Scrub known secret/credential patterns from *value* and return the result."""
     return _SECRET_RE.sub(_REDACTED, value)
+
+
+def redact_structured_value(key: str | None, value: str) -> str:
+    """Scrub *value* as it appeared under *key* inside a structured record.
+
+    redact_string() replaces a whole ``key: value`` span with REDACTED, which is
+    fine inside free text but destroys the surrounding syntax when the span is a
+    JSON member rather than message content. This renders the pair the way a dict
+    repr would, so the key-name patterns still fire, but collapses only the value
+    so the caller's structure survives.
+    """
+    scrubbed: Final = redact_string(value)
+    if scrubbed != value or key is None:
+        return scrubbed
+    rendered: Final = f"'{key}': '{value}'"
+    return _REDACTED if redact_string(rendered) != rendered else value

litellm/llms/base_llm/chat/transformation.py

@@ -5,7 +5,7 @@ Common base config for all LLM providers
 import types
 from abc import ABC, abstractmethod
 from collections.abc import AsyncIterator, Iterator
-from typing import TYPE_CHECKING, Any, Final, Union, cast
+from typing import TYPE_CHECKING, Any, Final, Union
 
 import httpx
 from pydantic import BaseModel
@@ -90,9 +90,9 @@ class BaseConfig(ABC):
         return type_to_response_format_param(response_format=response_format)
 
     def is_thinking_enabled(self, non_default_params: dict) -> bool:
-        return (non_default_params.get("thinking") or {}).get("type") == "enabled" or non_default_params.get(
-            "reasoning_effort"
-        ) is not None
+        thinking: Final = non_default_params.get("thinking")
+        thinking_type: Final = thinking.get("type") if isinstance(thinking, dict) else None
+        return thinking is True or thinking_type == "enabled" or non_default_params.get("reasoning_effort") is not None
 
     def is_max_tokens_in_request(self, non_default_params: dict) -> bool:
         """
@@ -112,7 +112,10 @@ class BaseConfig(ABC):
         if is_thinking_enabled and (
             "max_tokens" not in non_default_params and "max_completion_tokens" not in non_default_params
         ):
-            thinking_token_budget: Final = cast(dict, optional_params["thinking"]).get("budget_tokens", None)
+            thinking_value: Final = optional_params.get("thinking")
+            thinking_token_budget: Final = (
+                thinking_value.get("budget_tokens") if isinstance(thinking_value, dict) else None
+            )
             if thinking_token_budget is not None:
                 optional_params["max_tokens"] = thinking_token_budget + DEFAULT_MAX_TOKENS
 

litellm/llms/bedrock/chat/converse_transformation.py

@@ -1090,7 +1090,10 @@ class AmazonConverseConfig(BaseConfig):
         is_thinking_enabled: Final = self.is_thinking_enabled(optional_params)
         is_max_tokens_in_request: Final = self.is_max_tokens_in_request(non_default_params)
         if is_thinking_enabled and not is_max_tokens_in_request:
-            thinking_token_budget: Final = cast(dict, optional_params["thinking"]).get("budget_tokens", None)
+            thinking_value: Final = optional_params.get("thinking")
+            thinking_token_budget: Final = (
+                thinking_value.get("budget_tokens") if isinstance(thinking_value, dict) else None
+            )
             if thinking_token_budget is not None:
                 optional_params["maxTokens"] = thinking_token_budget + DEFAULT_MAX_TOKENS
 

litellm/llms/custom_httpx/llm_http_handler.py

@@ -2,7 +2,7 @@ import asyncio
 import json
 import os
 import ssl
-from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping
+from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
 from contextlib import asynccontextmanager
 from functools import lru_cache
 from types import ModuleType
@@ -73,6 +73,7 @@ from litellm.llms.custom_httpx.http_handler import (
 from litellm.responses.streaming_iterator import (
     BaseResponsesAPIStreamingIterator,
     MockResponsesAPIStreamingIterator,
+    ProjectQuotaCallback,
     ResponsesAPIStreamingIterator,
     ResponsesWebSocketStreaming,
     SyncResponsesAPIStreamingIterator,
@@ -256,6 +257,27 @@ def _has_pre_call_deployment_hook(logging_obj: LiteLLMLoggingObj) -> bool:
     return False
 
 
+def _collect_ws_project_quota_callbacks() -> tuple[ProjectQuotaCallback, ...]:
+    """Duck-type discover proxy hooks exposing per-frame project ITPM/OTPM
+    enforcement, so the Responses WebSocket loop can charge every
+    ``response.create`` frame, not just the connection's first one.
+
+    Uses duck-typing on ``litellm.callbacks`` (rather than importing the
+    proxy hook directly) to avoid a layering violation (SDK importing from
+    the proxy layer).
+    """
+    import litellm as _litellm
+
+    callbacks: Final = cast(  # cast-ok: callback registry is inspected before protocol use
+        Sequence[object], _litellm.callbacks
+    )
+    return tuple(
+        cast(ProjectQuotaCallback, callback)  # cast-ok: required callback method is callable
+        for callback in callbacks
+        if callable(getattr(callback, "enforce_project_io_token_quota_for_frame", None))
+    )
+
+
 class BaseLLMHTTPHandler:
     async def _make_common_async_call(
         self,
@@ -6179,6 +6201,8 @@ class BaseLLMHTTPHandler:
         - Uses ManagedResponsesWebSocketHandler which makes HTTP streaming calls
         - Forwards events over the websocket connection
         """
+        _ws_quota_callbacks: Final = _collect_ws_project_quota_callbacks()
+
         if responses_api_provider_config is None or not responses_api_provider_config.supports_native_websocket():
             from litellm.responses.streaming_iterator import (
                 ManagedResponsesWebSocketHandler,
@@ -6195,6 +6219,7 @@ class BaseLLMHTTPHandler:
                 timeout=timeout,
                 custom_llm_provider=custom_llm_provider,
                 first_message=first_message,
+                quota_callbacks=_ws_quota_callbacks,
                 **kwargs,
             )
             await handler.run()
@@ -6315,6 +6340,7 @@ class BaseLLMHTTPHandler:
                     first_message=first_message,
                     guardrail_callbacks=_ws_guardrail_callbacks,
                     output_guardrail_callbacks=_ws_output_guardrail_callbacks,
+                    quota_callbacks=_ws_quota_callbacks,
                     authorized_model=model,
                 )
                 await streaming.bidirectional_forward()
@@ -9407,6 +9433,27 @@ class BaseLLMHTTPHandler:
             )
 
     ###### VECTOR STORE HANDLER ######
+    @staticmethod
+    def _pre_call_direct_vector_store_search(
+        logging_obj: LiteLLMLoggingObj,
+        custom_llm_provider: str,
+        vector_store_id: str,
+        query: str | Sequence[str],
+    ) -> None:
+        """Direct providers have no HTTP request to echo, and an empty api_base makes the debug
+        logger fall back to dumping model_call_details, which holds stored provider credentials."""
+        endpoint: Final = f"{custom_llm_provider}://{vector_store_id}"
+        logging_obj.pre_call(
+            input="",
+            api_key="",
+            additional_args={  # mutable-ok: pre_call's additional_args contract is a dict
+                "query": query,
+                "vector_store_id": vector_store_id,
+                "api_base": endpoint,
+                "request_str": f"direct vector store search: {endpoint}",
+            },
+        )
+
     async def async_vector_store_search_handler(
         self,
         vector_store_id: str,
@@ -9423,13 +9470,11 @@ class BaseLLMHTTPHandler:
         _is_async: bool = False,
     ) -> VectorStoreSearchResponse:
         if isinstance(vector_store_provider_config, BaseDirectVectorStoreConfig):
-            logging_obj.pre_call(
-                input="",
-                api_key="",
-                additional_args={  # mutable-ok: pre_call's additional_args contract is a dict
-                    "query": query,
-                    "vector_store_id": vector_store_id,
-                },
+            self._pre_call_direct_vector_store_search(
+                logging_obj=logging_obj,
+                custom_llm_provider=custom_llm_provider,
+                vector_store_id=vector_store_id,
+                query=query,
             )
             return await vector_store_provider_config.aexecute_search_vector_store_request(
                 vector_store_id=vector_store_id,
@@ -9554,13 +9599,11 @@ class BaseLLMHTTPHandler:
             )
 
         if isinstance(vector_store_provider_config, BaseDirectVectorStoreConfig):
-            logging_obj.pre_call(
-                input="",
-                api_key="",
-                additional_args={  # mutable-ok: pre_call's additional_args contract is a dict
-                    "query": query,
-                    "vector_store_id": vector_store_id,
-                },
+            self._pre_call_direct_vector_store_search(
+                logging_obj=logging_obj,
+                custom_llm_provider=custom_llm_provider,
+                vector_store_id=vector_store_id,
+                query=query,
             )
             return vector_store_provider_config.execute_search_vector_store_request(
                 vector_store_id=vector_store_id,

litellm/llms/deepseek/chat/transformation.py

@@ -131,9 +131,11 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
           - model supports reasoning (capability check)
           - user explicitly passed thinking={"type": "enabled"} (opt-in check)
         """
+        thinking: Final = optional_params.get("thinking")
         return (
             supports_reasoning(model=model, custom_llm_provider="deepseek")
-            and (optional_params.get("thinking") or {}).get("type") == "enabled"
+            and isinstance(thinking, dict)
+            and thinking.get("type") == "enabled"
         )
 
     @staticmethod

litellm/llms/tinyfish/search/transformation.py

@@ -14,6 +14,7 @@ import httpx
 from pydantic import TypeAdapter, ValidationError
 
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.core_helpers import process_response_headers
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.base_llm.search.transformation import (
@@ -22,7 +23,7 @@ from litellm.llms.base_llm.search.transformation import (
 )
 from litellm.secret_managers.main import get_secret_str
 
-_UrlEncodableParams: Final = TypeAdapter(dict[str, str | int | bool])
+_UrlEncodableParams: Final = TypeAdapter(dict[str, str | int | float | bool])
 _StrList: Final = TypeAdapter(list[str])
 _StrFrozenSet: Final = TypeAdapter(frozenset[str])
 
@@ -94,16 +95,16 @@ class TinyfishSearchConfig(BaseSearchConfig):
         TinyFish equivalents:
         - ``query`` (str or list[str]) → ``query`` (list joined by spaces)
         - ``country`` → ``location``
-        - ``search_domain_filter`` (list[str]) → folded into the query as
-          ``(<query>) (site:a OR site:b ...)`` (TinyFish has no first-class
-          field today; see ML-2084 for the planned ``include_domains``)
+        - ``search_domain_filter`` (list[str]) → folded into the query using
+          search operators
         - ``max_results`` → not sent on the wire; stashed on
           ``self._caller_max_results`` for client-side response truncation
           (TinyFish doesn't honor it server-side)
         - ``max_tokens_per_page`` → silently dropped (no TinyFish equivalent)
 
         Any other ``optional_params`` keys are forwarded to TinyFish as-is.
-        dict/list values are JSON-encoded so they survive ``urlencode``.
+        dict and list values are JSON-encoded so structured payloads survive
+        ``urlencode``.
 
         Returns:
             ``{_TINYFISH_PARAMS_KEY: <dict of querystring entries>}``.
@@ -144,14 +145,12 @@ class TinyfishSearchConfig(BaseSearchConfig):
         supported_perplexity: Final = _StrFrozenSet.validate_python(raw_supported)
         for param, value in optional_params.items():
             if param not in supported_perplexity and param not in request_data:
-                # `fetch` expects a JSON-encoded object on the wire; accept the
-                # natural Python dict form and serialize here so callers don't
-                # have to pre-stringify.
-                if isinstance(value, dict):
+                # Serialize dicts/lists as JSON so structured params survive urlencode.
+                if isinstance(value, (dict, list)):
                     value = json.dumps(value, separators=(",", ":"))
                 # `urlencode` would render Python bool as "True"/"False"
-                # (capitalized). ux-labs validators require lowercase
-                # "true"/"false" (e.g. `include_thumbnail`); normalize here.
+                # (capitalized). TinyFish Search's bool params require lowercase
+                # "true"/"false" strings on the wire; normalize here.
                 elif isinstance(value, bool):
                     value = "true" if value else "false"
                 request_data[param] = value
@@ -167,17 +166,35 @@ class TinyfishSearchConfig(BaseSearchConfig):
         """
         Transform a TinyFish response to LiteLLM's unified ``SearchResponse``.
 
-        Mappings (per-result):
-        - ``title`` → ``SearchResult.title`` (defaults to ``""`` if missing/null)
-        - ``url`` → ``SearchResult.url`` (defaults to ``""``)
-        - ``snippet`` → ``SearchResult.snippet`` (defaults to ``""``)
-        - all other per-result fields (``position``, ``site_name``,
-          ``thumbnail_url``, ``fetch``, ``fetch_error``, ...) ride through as
-          extras on ``SearchResult`` via its ``extra="allow"`` config.
+        Per-result field handling:
+        - ``title``, ``url``, ``snippet`` are declared on ``SearchResult`` and
+          populated by ``SearchResponse.model_validate`` when present. Missing
+          or ``None`` values are defaulted to ``""`` beforehand by
+          ``_default_missing_result_fields`` so a degraded result flows through
+          instead of failing the whole call.
+        - All undeclared per-result fields (``position``, ``site_name``, and
+          any others TinyFish returns) ride through as extras via
+          ``SearchResult``'s ``extra="allow"`` config — accessible as
+          attributes on the result object or enumerable via
+          ``result.model_extra``.
 
-        Top-level ``parameter_warnings`` (see ML-2085) is read when present and
-        each entry is re-fired via ``verbose_logger.warning``. Absent or
-        malformed entries are silently skipped — never throws.
+        Top-level ``parameter_warnings`` is read when present and each entry
+        is re-fired via ``verbose_logger.warning``. Absent or malformed
+        entries are silently skipped — never throws.
+
+        Top-level extras (``query``, ``total_results``, ``page``, and any
+        future TinyFish additions) ride through via
+        ``SearchResponse.extra="allow"``. The validated response is returned
+        in place after truncating ``results`` to the caller's ``max_results``,
+        so every field pydantic populated survives regardless of which
+        storage bucket (declared attribute or ``__pydantic_extra__``) holds it.
+
+        TinyFish response headers (e.g. ``x-request-id``, ``retry-after``,
+        ``x-ratelimit-limit`` — httpx normalizes header names to lowercase)
+        are stashed on ``response._hidden_params["headers"]`` (raw) and
+        ``response._hidden_params["additional_headers"]`` (sanitized via
+        ``process_response_headers``) so callers can correlate a search with
+        server-side logs.
 
         Error paths routed through ``self._wrap_error`` for uniform
         ``"TinyFish Search: <msg>. See <docs> for details."`` wrapping:
@@ -223,7 +240,12 @@ class TinyfishSearchConfig(BaseSearchConfig):
         _emit_parameter_warnings(parsed)
 
         max_results: Final = self._caller_max_results or _TINYFISH_RESULT_CAP
-        return SearchResponse(results=list(parsed.results[:max_results]))
+        parsed.results = parsed.results[:max_results]
+        raw_headers: Final = dict(raw_response.headers)
+        hidden: Final = parsed._hidden_params  # pyright: ignore[reportPrivateUsage]  # sole hidden-params channel
+        hidden["headers"] = raw_headers
+        hidden["additional_headers"] = process_response_headers(raw_headers)
+        return parsed
 
     def _wrap_error(
         self,
@@ -243,9 +265,9 @@ class TinyfishSearchConfig(BaseSearchConfig):
         carry the ``TinyFish Search:`` prefix — the bare error already names
         the host in the URL, so attribution is implicit there.
         """
-        # ux-labs frontend wraps every error body as {"error": {"code", "message", "details"?}}.
+        # TinyFish Search wraps every error body as {"error": {"code", "message", "details"?}}.
         # Best-effort unwrap to surface the inner message; fall back to the raw body
-        # for non-ux-labs responses (CDN HTML pages, other JSON envelopes, plain text).
+        # for other envelope shapes (CDN HTML pages, other JSON envelopes, plain text).
         inner_message = error_message
         try:
             body: Final[object] = json.loads(error_message)  # any-ok: json.loads -> Any
@@ -290,7 +312,7 @@ def _default_missing_result_fields(raw_json: object) -> None:
 
 
 def _emit_parameter_warnings(parsed: SearchResponse) -> None:
-    """Re-fire TinyFish-side ``parameter_warnings`` (see ML-2085) as warnings.
+    """Re-fire TinyFish-side ``parameter_warnings`` as warnings.
 
     Defensive: skip silently on any shape we don't recognize so a malformed
     entry (or an early/partial rollout of the field) never throws.

litellm/main.py

@@ -5007,7 +5007,6 @@ def completion(
     tool_choice = validate_chat_completion_tool_choice(tool_choice=tool_choice)
     # validate optional params
     stop = validate_openai_optional_params(stop=stop)
-    # normalize camelCase thinking keys (e.g. budgetTokens -> budget_tokens)
     thinking = validate_and_fix_thinking_param(thinking=thinking)
 
     ######### unpacking kwargs #####################

litellm/model_prices_and_context_window_backup.json

@@ -14456,6 +14456,25 @@
         "supports_tool_choice": true,
         "supports_output_config": true
     },
+    "databricks/databricks-claude-opus-4-6": {
+        "input_cost_per_token": 5.00003e-06,
+        "input_dbu_cost_per_token": 7.1429e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 2.5000010000000002e-05,
+        "output_dbu_cost_per_token": 0.000357143,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+        "supports_assistant_prefill": true,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },
     "databricks/databricks-claude-sonnet-4": {
         "input_cost_per_token": 2.9999900000000002e-06,
         "input_dbu_cost_per_token": 4.2857e-05,
@@ -14513,6 +14532,25 @@
         "supports_reasoning": true,
         "supports_tool_choice": true
     },
+    "databricks/databricks-claude-sonnet-4-6": {
+        "input_cost_per_token": 2.9999900000000002e-06,
+        "input_dbu_cost_per_token": 4.2857e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 64000,
+        "max_tokens": 64000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.5000020000000002e-05,
+        "output_dbu_cost_per_token": 0.000214286,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+        "supports_assistant_prefill": true,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },
     "databricks/databricks-gemini-2-5-flash": {
         "input_cost_per_token": 3.0001999999999996e-07,
         "input_dbu_cost_per_token": 4.285999999999999e-06,
@@ -14547,6 +14585,74 @@
         "supports_function_calling": true,
         "supports_tool_choice": true
     },
+    "databricks/databricks-gemini-3-1-flash-lite": {
+        "input_cost_per_token": 3.1248e-07,
+        "input_dbu_cost_per_token": 4.464e-06,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.87502e-06,
+        "output_dbu_cost_per_token": 2.6786e-05,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+        "supports_function_calling": true,
+        "supports_tool_choice": true
+    },
+    "databricks/databricks-gemini-3-1-pro": {
+        "input_cost_per_token": 2.49998e-06,
+        "input_dbu_cost_per_token": 3.5714e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.5000020000000002e-05,
+        "output_dbu_cost_per_token": 0.000214286,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+        "supports_function_calling": true,
+        "supports_tool_choice": true
+    },
+    "databricks/databricks-gemini-3-flash": {
+        "input_cost_per_token": 6.2503e-07,
+        "input_dbu_cost_per_token": 8.929e-06,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 3.74997e-06,
+        "output_dbu_cost_per_token": 5.3571e-05,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+        "supports_function_calling": true,
+        "supports_tool_choice": true
+    },
+    "databricks/databricks-gemini-3-pro": {
+        "input_cost_per_token": 2.49998e-06,
+        "input_dbu_cost_per_token": 3.5714e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.5000020000000002e-05,
+        "output_dbu_cost_per_token": 0.000214286,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+        "supports_function_calling": true,
+        "supports_tool_choice": true
+    },
     "databricks/databricks-gemma-3-12b": {
         "input_cost_per_token": 1.5000999999999998e-07,
         "input_dbu_cost_per_token": 2.1429999999999996e-06,
@@ -14592,6 +14698,126 @@
         "output_dbu_cost_per_token": 0.000142857,
         "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
     },
+    "databricks/databricks-gpt-5-1-codex-max": {
+        "input_cost_per_token": 1.24999e-06,
+        "input_dbu_cost_per_token": 1.7857e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 9.999990000000002e-06,
+        "output_dbu_cost_per_token": 0.000142857,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
+    "databricks/databricks-gpt-5-1-codex-mini": {
+        "input_cost_per_token": 2.4997e-07,
+        "input_dbu_cost_per_token": 3.571e-06,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.99997e-06,
+        "output_dbu_cost_per_token": 2.8571e-05,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
+    "databricks/databricks-gpt-5-2": {
+        "input_cost_per_token": 1.75e-06,
+        "input_dbu_cost_per_token": 2.5e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.4e-05,
+        "output_dbu_cost_per_token": 0.0002,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
+    "databricks/databricks-gpt-5-2-codex": {
+        "input_cost_per_token": 1.75e-06,
+        "input_dbu_cost_per_token": 2.5e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.4e-05,
+        "output_dbu_cost_per_token": 0.0002,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
+    "databricks/databricks-gpt-5-3-codex": {
+        "input_cost_per_token": 1.75e-06,
+        "input_dbu_cost_per_token": 2.5e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.4e-05,
+        "output_dbu_cost_per_token": 0.0002,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
+    "databricks/databricks-gpt-5-4": {
+        "input_cost_per_token": 2.49998e-06,
+        "input_dbu_cost_per_token": 3.5714e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.5000020000000002e-05,
+        "output_dbu_cost_per_token": 0.000214286,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
+    "databricks/databricks-gpt-5-4-mini": {
+        "input_cost_per_token": 7.4998e-07,
+        "input_dbu_cost_per_token": 1.0714e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 4.50002e-06,
+        "output_dbu_cost_per_token": 6.4286e-05,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
+    "databricks/databricks-gpt-5-4-nano": {
+        "input_cost_per_token": 1.9999e-07,
+        "input_dbu_cost_per_token": 2.857e-06,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.24999e-06,
+        "output_dbu_cost_per_token": 1.7857e-05,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
     "databricks/databricks-gpt-5-mini": {
         "input_cost_per_token": 2.4997000000000006e-07,
         "input_dbu_cost_per_token": 3.571e-06,

litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py

@@ -1341,7 +1341,7 @@ async def _persist_dcr_client_registration(
     ``update_mcp_server`` merges credential blobs: a re-registered public client must not
     inherit the previous client's secret or auth method.
     """
-    if mcp_server.is_true_passthrough or mcp_server.is_oauth_delegate:
+    if mcp_server.is_client_forwarded_token:
         return "skipped"
 
     try:
@@ -2187,7 +2187,7 @@ async def _build_oauth_protected_resource_response(
             )
 
         if upstream_metadata is not None:
-            if mcp_server.is_true_passthrough or mcp_server.is_oauth_delegate:
+            if mcp_server.is_client_forwarded_token:
                 return upstream_metadata
             return {**upstream_metadata, "resource": resource_url}
 

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -123,6 +123,7 @@ from litellm.proxy._experimental.mcp_server.utils import (
     iter_known_server_prefixes,
     iter_known_tool_name_spellings,
     logging_safe_mcp_headers,
+    lookup_mcp_server_auth_in_headers,
     match_known_server_prefix,
     match_known_tool_name,
     merge_mcp_headers,
@@ -873,6 +874,53 @@ def _openapi_forwarded_extra_headers(
     return forwarded or None
 
 
+def _resolve_openapi_tool_auth(
+    mcp_server: MCPServer,
+    mcp_auth_header: str | None,
+    mcp_server_auth_headers: Mapping[str, str | dict[str, str]] | None,  # mutable-ok: sink shape
+    raw_headers: dict[str, str] | None,  # mutable-ok: sink takes a concrete dict
+    user_api_key_auth: UserAPIKeyAuth | None,
+) -> tuple[str | None, dict[str, str] | None, str | dict[str, str] | None]:  # mutable-ok: sink shapes
+    """The caller's upstream credential for one ``spec_path`` server, for both OpenAPI dispatch arms.
+
+    A per-server ``x-mcp-{alias}-authorization`` wins over the deprecated global / BYOK
+    ``mcp_auth_header``, the same precedence ``_call_regular_mcp_tool`` applies, so the OpenAPI and
+    managed paths cannot disagree about which credential is authoritative. The two kinds are not
+    interchangeable: a per-server value is already a complete header value and is forwarded verbatim,
+    while a BYOK credential is a raw secret that takes the server's auth-type prefix. Formatting the
+    former would ship ``Bearer Bearer <token>``.
+
+    Returns the ``Authorization`` value to inject, the extra headers to forward, and the credential to
+    hand ``resolve_openapi_upstream_auth``, whose passthrough arm reads it via
+    ``_passthrough_token_from_mcp_auth_header``. The per-server Authorization travels only in the
+    credential, never also in the forwarded headers, because the resolver pops Authorization out of
+    those and would otherwise have two sources to reconcile.
+    """
+    forwarded: Final = _openapi_forwarded_extra_headers(mcp_server, raw_headers, user_api_key_auth)
+    per_server: Final = (
+        lookup_mcp_server_auth_in_headers(
+            mcp_server_auth_headers,
+            alias=mcp_server.alias,
+            server_name=mcp_server.server_name,
+        )
+        if mcp_server_auth_headers
+        else None
+    )
+
+    if isinstance(per_server, dict):
+        authorization: Final = next((v for k, v in per_server.items() if k.lower() == "authorization"), None)
+        merged: Final = merge_mcp_headers(extra_headers=forwarded, static_headers=_without_authorization(per_server))
+        if authorization is None:
+            byok: Final = _format_byok_openapi_auth_header(mcp_server, mcp_auth_header) if mcp_auth_header else None
+            return byok, merged, mcp_auth_header
+        return authorization, merged, per_server
+    if isinstance(per_server, str) and per_server:
+        return per_server, forwarded, per_server
+    if mcp_auth_header:
+        return _format_byok_openapi_auth_header(mcp_server, mcp_auth_header), forwarded, mcp_auth_header
+    return None, forwarded, None
+
+
 async def _resolve_byok_mcp_auth_header(
     mcp_server: MCPServer,
     user_api_key_auth: UserAPIKeyAuth | None,
@@ -1740,12 +1788,14 @@ class MCPServerManager:
             server: The MCP server whose OAuth metadata must be resolved.
 
         Returns:
-            The resolved server, or the registered server when no discovery is
-            pending.
+            The resolved server; the registered server when no discovery is
+            pending, or when discovery failed for a client-forwarded-token
+            server, whose session consumes no discovered endpoint.
 
         Raises:
             HTTPException: Status 503 when discovery times out or returns
-                incomplete metadata.
+                incomplete metadata for a server whose OAuth flow the gateway
+                runs itself.
         """
         acquisition: Final = self._get_or_start_oauth_discovery_task(server)
         if acquisition is None:
@@ -1764,6 +1814,8 @@ class MCPServerManager:
                 return await self.ensure_oauth_metadata_discovered(server)
             case _OAuthDiscoveryFailed(timed_out=timed_out):
                 current: Final = self._registered_server(server)
+                if current.is_client_forwarded_token:
+                    return current
                 server_ref: Final = current.alias or current.server_name or current.name or current.server_id
                 reason: Final = "timed out" if timed_out else "returned incomplete metadata"
                 raise HTTPException(
@@ -2321,23 +2373,30 @@ class MCPServerManager:
             openapi_key_prefix: Final = prefix_root + MCP_TOOL_PREFIX_SEPARATOR
             global_mcp_tool_registry.unregister_tools_with_prefix(openapi_key_prefix)
 
-        owned_raw: Final[set[str]] = set()
-        for p in iter_known_server_prefixes(server):
-            if p:
-                owned_raw.add(p)
-        if server.name:
-            owned_raw.add(server.name)
+        owned_normalized: Final = self._owned_mapping_values(server)
 
-        owned_normalized: Final = {normalize_server_name(x) for x in owned_raw}
-
-        stale_mapping_keys: Final[list[str]] = []
-        for tool_name, mapped_server in list(self.tool_name_to_mcp_server_name_mapping.items()):
-            if mapped_server in owned_raw or normalize_server_name(str(mapped_server)) in owned_normalized:
-                stale_mapping_keys.append(tool_name)
+        stale_mapping_keys: Final = tuple(
+            tool_name
+            for tool_name, mapped_server in self.tool_name_to_mcp_server_name_mapping.items()
+            if normalize_server_name(str(mapped_server)) in owned_normalized
+        )
 
         for key in stale_mapping_keys:
             del self.tool_name_to_mcp_server_name_mapping[key]
 
+    def _owned_mapping_values(self, server: MCPServer) -> frozenset[str]:
+        return frozenset(
+            normalize_server_name(value) for value in (*iter_known_server_prefixes(server), server.name) if value
+        )
+
+    def _server_exposes_tool(self, server: MCPServer, tool_name: str) -> bool:
+        owned: Final = self._owned_mapping_values(server)
+        mapped_owners: Final = (
+            self.tool_name_to_mcp_server_name_mapping.get(spelling)
+            for spelling in iter_known_tool_name_spellings(tool_name, server)
+        )
+        return any(owner is not None and normalize_server_name(owner) in owned for owner in mapped_owners)
+
     def remove_server(self, mcp_server: LiteLLM_MCPServerTable):
         """
         Remove a server from the registry
@@ -3182,10 +3241,6 @@ class MCPServerManager:
             # Get server-specific auth header if available
             server_auth_header: str | dict[str, str] | None = None
             if mcp_server_auth_headers:
-                from litellm.proxy._experimental.mcp_server.utils import (
-                    lookup_mcp_server_auth_in_headers,
-                )
-
                 server_auth_header = lookup_mcp_server_auth_in_headers(
                     mcp_server_auth_headers,
                     alias=server.alias,
@@ -5210,11 +5265,6 @@ class MCPServerManager:
         # the exact case of server alias/name (e.g., '1litellmagcgateway' vs '1LiteLLMAGCGateway')
         server_auth_header: dict[str, str] | str | None = None
         if mcp_server_auth_headers:
-            # Normalize keys for case-insensitive lookup
-            from litellm.proxy._experimental.mcp_server.utils import (
-                lookup_mcp_server_auth_in_headers,
-            )
-
             server_auth_header = lookup_mcp_server_auth_in_headers(
                 mcp_server_auth_headers,
                 alias=mcp_server.alias,
@@ -5249,7 +5299,7 @@ class MCPServerManager:
                     user_api_key_auth=user_api_key_auth,
                 ):
                     extra_headers = _without_authorization(extra_headers)
-        elif mcp_server.is_true_passthrough or mcp_server.is_oauth_delegate:
+        elif mcp_server.is_client_forwarded_token:
             extra_headers = _client_forwarded_authorization_headers(
                 mcp_server=mcp_server,
                 oauth2_headers=oauth2_headers,
@@ -5356,7 +5406,7 @@ class MCPServerManager:
             # Scoped to the two client-forwarded token modes this stack introduced; legacy
             # oauth2 + delegate_auth_to_upstream (is_oauth_passthrough) is being removed, so it is not
             # added here even though the list path still relays for it.
-            relays_upstream_auth: Final = mcp_server.is_true_passthrough or mcp_server.is_oauth_delegate
+            relays_upstream_auth: Final = mcp_server.is_client_forwarded_token
             server_label: Final = mcp_server.name or mcp_server.server_name or mcp_server.alias or ""
 
             async def _call_tool_via_client(client, params):
@@ -5463,13 +5513,8 @@ class MCPServerManager:
         if mcp_server is None:
             raise ValueError(f"Tool {name} not found")
 
-        if resolved_by_server_name_only:
-            tool_known: Final = (
-                name in self.tool_name_to_mcp_server_name_mapping
-                or prefixed_tool_name in self.tool_name_to_mcp_server_name_mapping
-            )
-            if not tool_known:
-                raise ValueError(f"Tool {name} not found")
+        if resolved_by_server_name_only and not self._server_exposes_tool(mcp_server, name):
+            raise ValueError(f"Tool {name} not found")
 
         return mcp_server
 
@@ -5712,16 +5757,20 @@ class MCPServerManager:
                     server_name,
                 )
 
-            auth_header_value: Final = (
-                _format_byok_openapi_auth_header(mcp_server, mcp_auth_header) if mcp_auth_header else None
+            auth_header_value, openapi_forwarded_headers, upstream_credential = _resolve_openapi_tool_auth(
+                mcp_server=mcp_server,
+                mcp_auth_header=mcp_auth_header,
+                mcp_server_auth_headers=mcp_server_auth_headers,
+                raw_headers=raw_headers,
+                user_api_key_auth=user_api_key_auth,
             )
             resolved_auth_headers, forwarded_headers = await self.resolve_openapi_upstream_auth(
                 mcp_server=mcp_server,
                 oauth2_headers=caller_oauth2_headers,
                 raw_headers=raw_headers,
-                mcp_auth_header=mcp_auth_header,
+                mcp_auth_header=upstream_credential,
                 user_api_key_auth=user_api_key_auth,
-                forwarded_headers=_openapi_forwarded_extra_headers(mcp_server, raw_headers, user_api_key_auth),
+                forwarded_headers=openapi_forwarded_headers,
             )
 
             async def _call_openapi_via_handler():
@@ -5847,10 +5896,7 @@ class MCPServerManager:
         if matched is not None:
             matched_prefix, original_tool_name = matched
             matched_server: Final = prefix_to_server.get(matched_prefix)
-            if matched_server is not None and (
-                original_tool_name in self.tool_name_to_mcp_server_name_mapping
-                or tool_name in self.tool_name_to_mcp_server_name_mapping
-            ):
+            if matched_server is not None and self._server_exposes_tool(matched_server, original_tool_name):
                 return matched_server
 
         return None

litellm/proxy/_experimental/mcp_server/server.py

@@ -430,6 +430,7 @@ if MCP_AVAILABLE:
         MCPServerManager,
         _caller_authorization_fans_out,
         _client_forwarded_authorization_headers,
+        _resolve_openapi_tool_auth,
         _should_strip_caller_authorization,
         _without_authorization,
         global_mcp_server_manager,
@@ -1704,7 +1705,7 @@ if MCP_AVAILABLE:
             )
 
         extra_headers: dict[str, str] | None = None
-        is_client_forwarded_mode: Final = server.is_true_passthrough or server.is_oauth_delegate
+        is_client_forwarded_mode: Final = server.is_client_forwarded_token
         # In a multi-server listing scope the request-wide Authorization can only carry one token,
         # so it is withheld from a client-forwarded server when another server in scope also consumes
         # it (RFC 9700 cross-resource replay); such scopes must bind per-server via
@@ -2013,6 +2014,9 @@ if MCP_AVAILABLE:
                         prefetched_creds=_prefetched_oauth_creds,
                     )
 
+                if server.is_byok and server.auth_type != MCPAuth.oauth2 and server_auth_header is None:
+                    server_auth_header = await _get_byok_credential(server, user_api_key_auth)
+
                 try:
                     tools: Final = await global_mcp_server_manager._get_tools_from_server(
                         server=server,
@@ -2832,58 +2836,26 @@ if MCP_AVAILABLE:
                 arguments = hook_result["arguments"]
 
             verbose_logger.debug("Executing local registry tool: %s", name)
-            # For BYOK servers the credential must be injected via a ContextVar
-            # because the tool function has headers baked into its closure.
-            # Pre-format the full Authorization header value using the server's
-            # configured auth_type so the generator doesn't need to know the prefix.
-            auth_header_value: str | None = None
-            if mcp_auth_header:
-                server_auth_type: Final = getattr(mcp_server, "auth_type", None) if mcp_server else None
-                if server_auth_type == MCPAuth.api_key:
-                    auth_header_value = f"ApiKey {mcp_auth_header}"
-                elif server_auth_type == MCPAuth.basic:
-                    auth_header_value = f"Basic {mcp_auth_header}"
-                else:
-                    auth_header_value = f"Bearer {mcp_auth_header}"
-
-            # Forward named client headers to OpenAPI tool upstream requests.
-            # MCPServer.extra_headers lists header names to copy from raw_headers.
-            # The strip decision is centralized in _should_strip_caller_authorization so this
-            # OpenAPI/local path agrees with the managed paths: M2M and the resolver-owned modes
-            # (token_exchange's raw subject token, authorization_code's stored token) must never
-            # have the caller's Authorization forwarded verbatim upstream.
-            forwarded_headers: dict[str, str] | None = None
-            if mcp_server and mcp_server.extra_headers and raw_headers:
-                normalized_raw: Final = {str(k).lower(): v for k, v in raw_headers.items() if isinstance(k, str)}
-                skip_caller_authorization: Final = _should_strip_caller_authorization(
-                    mcp_server=mcp_server,
-                    raw_headers=raw_headers,
-                    user_api_key_auth=user_api_key_auth,
-                )
-                for header_name in mcp_server.extra_headers:
-                    if not isinstance(header_name, str):
-                        continue
-                    if skip_caller_authorization and header_name.lower() == "authorization":
-                        continue
-                    value = normalized_raw.get(header_name.lower())
-                    if value is not None:
-                        if forwarded_headers is None:
-                            forwarded_headers = {}
-                        forwarded_headers[header_name] = value
-
-            resolved_auth_headers: dict[str, str] | None = None
-            if mcp_server:
-                (
-                    resolved_auth_headers,
-                    forwarded_headers,
-                ) = await global_mcp_server_manager.resolve_openapi_upstream_auth(
-                    mcp_server=mcp_server,
-                    oauth2_headers=oauth2_headers,
-                    raw_headers=raw_headers,
-                    mcp_auth_header=mcp_auth_header,
-                    user_api_key_auth=user_api_key_auth,
-                    forwarded_headers=forwarded_headers,
-                )
+            # The credential rides ContextVars because the tool function has its
+            # headers baked into the closure at registration time.
+            auth_header_value, openapi_forwarded_headers, upstream_credential = _resolve_openapi_tool_auth(
+                mcp_server=mcp_server,
+                mcp_auth_header=mcp_auth_header,
+                mcp_server_auth_headers=mcp_server_auth_headers,
+                raw_headers=raw_headers,
+                user_api_key_auth=user_api_key_auth,
+            )
+            (
+                resolved_auth_headers,
+                forwarded_headers,
+            ) = await global_mcp_server_manager.resolve_openapi_upstream_auth(
+                mcp_server=mcp_server,
+                oauth2_headers=oauth2_headers,
+                raw_headers=raw_headers,
+                mcp_auth_header=upstream_credential,
+                user_api_key_auth=user_api_key_auth,
+                forwarded_headers=openapi_forwarded_headers,
+            )
 
             _auth_token: Final = _request_auth_header.set(auth_header_value)
             _extra_token: Final = _request_extra_headers.set(forwarded_headers)

litellm/proxy/_types.py

@@ -3059,6 +3059,8 @@ class NewProjectRequest(LiteLLM_BudgetTable):
     models: list[str] = []
     model_rpm_limit: dict | None = None
     model_tpm_limit: dict | None = None
+    model_itpm_limit: Mapping[str, int] | None = None
+    model_otpm_limit: Mapping[str, int] | None = None
     blocked: bool = False
     object_permission: LiteLLM_ObjectPermissionBase | None = None
 
@@ -3091,6 +3093,8 @@ class UpdateProjectRequest(LiteLLM_BudgetTable):
     models: list[str] | None = None
     model_rpm_limit: dict | None = None
     model_tpm_limit: dict | None = None
+    model_itpm_limit: Mapping[str, int] | None = None
+    model_otpm_limit: Mapping[str, int] | None = None
     blocked: bool | None = None
     budget_id: str | None = None
     object_permission: LiteLLM_ObjectPermissionBase | None = None
@@ -4244,6 +4248,8 @@ class PassThroughEndpointLoggingTypedDict(TypedDict):
 LiteLLM_ManagementEndpoint_MetadataFields: Final = [
     "model_rpm_limit",
     "model_tpm_limit",
+    "model_itpm_limit",
+    "model_otpm_limit",
     "default_estimated_output_tokens",
     "default_estimated_output_tokens_per_model",
     "mcp_rpm_limit",

litellm/proxy/agent_endpoints/a2a_endpoints.py

@@ -168,14 +168,13 @@ def _jsonrpc_error(
     )
 
 
-def _get_agent(agent_id: str):
+async def _get_agent(agent_id: str) -> "AgentResponse | None":
     """Look up an agent by ID or name. Returns None if not found."""
-    from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
+    from litellm.proxy.common_utils.registry_read_through import (
+        get_agent_with_read_through,
+    )
 
-    agent = global_agent_registry.get_agent_by_id(agent_id=agent_id)
-    if agent is None:
-        agent = global_agent_registry.get_agent_by_name(agent_name=agent_id)
-    return agent
+    return await get_agent_with_read_through(agent_id)
 
 
 def _enforce_inbound_trace_id(agent: "AgentResponse", request: Request) -> None:
@@ -559,7 +558,7 @@ async def get_agent_card(
     )
 
     try:
-        agent: Final = _get_agent(agent_id)
+        agent: Final = await _get_agent(agent_id)
         if agent is None:
             raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
 
@@ -673,7 +672,7 @@ async def invoke_agent_a2a(
                 params.pop(key)
 
         # Find the agent
-        agent: Final = _get_agent(agent_id)
+        agent: Final = await _get_agent(agent_id)
         if agent is None:
             return _jsonrpc_error(request_id, -32000, f"Agent '{agent_id}' not found", 404)
 

litellm/proxy/agent_endpoints/a2a_routing.py

@@ -25,10 +25,12 @@ async def route_a2a_agent_request(
     Returns None if not an A2A request (allows normal routing to continue).
     """
     # Import here to avoid circular imports
-    from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
     from litellm.proxy.agent_endpoints.auth.agent_permission_handler import (
         AgentRequestHandler,
     )
+    from litellm.proxy.common_utils.registry_read_through import (
+        get_agent_with_read_through,
+    )
     from litellm.proxy.route_llm_request import (
         ROUTE_ENDPOINT_MAPPING,
         ProxyModelNotFoundError,
@@ -44,11 +46,11 @@ async def route_a2a_agent_request(
     agent_name: Final = model_name[4:]
 
     # Look up agent in registry
-    agent: Final = global_agent_registry.get_agent_by_name(agent_name)
+    agent: Final = await get_agent_with_read_through(agent_name)
     if agent is None:
         verbose_proxy_logger.error("[A2A] Agent '%s' not found in registry", agent_name)
         route_name = ROUTE_ENDPOINT_MAPPING.get(route_type, route_type)
-        raise ProxyModelNotFoundError(route=route_name, model_name=model_name)
+        raise ProxyModelNotFoundError(route=route_name, model_name=model_name, retryable_with_model_read_through=False)
 
     # Verify the caller is permitted to use this agent (admins bypass the check)
     is_admin: Final = user_api_key_dict is not None and (
@@ -70,7 +72,7 @@ async def route_a2a_agent_request(
     if not agent.agent_card_params or "url" not in agent.agent_card_params:
         verbose_proxy_logger.error("[A2A] Agent '%s' has no URL configured", agent_name)
         route_name = ROUTE_ENDPOINT_MAPPING.get(route_type, route_type)
-        raise ProxyModelNotFoundError(route=route_name, model_name=model_name)
+        raise ProxyModelNotFoundError(route=route_name, model_name=model_name, retryable_with_model_read_through=False)
 
     # Inject API base and route to litellm
     data["api_base"] = agent.agent_card_params["url"]

litellm/proxy/agent_endpoints/agent_registry.py

@@ -600,3 +600,4 @@ class AgentRegistry:
 
 
 global_agent_registry: Final = AgentRegistry()
+AGENT_RECONCILE_LOCK: Final = asyncio.Lock()

litellm/proxy/anthropic_endpoints/endpoints.py

@@ -179,7 +179,7 @@ async def anthropic_response(
             await proxy_logging_obj.post_call_failure_hook(
                 user_api_key_dict=user_api_key_dict,
                 original_exception=e,
-                request_data=data,
+                request_data=base_llm_response_processor.data,
             )
         body: Final = AnthropicExceptionMapping.transform_to_anthropic_error(
             status_code=e.status_code,
@@ -189,7 +189,7 @@ async def anthropic_response(
         return JSONResponse(status_code=e.status_code, content=body)
     except Exception as e:
         await proxy_logging_obj.post_call_failure_hook(
-            user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
+            user_api_key_dict=user_api_key_dict, original_exception=e, request_data=base_llm_response_processor.data
         )
         verbose_proxy_logger.exception("litellm.proxy.proxy_server.anthropic_response(): Exception occured - %s", e)
 

litellm/proxy/auth/auth_utils.py

@@ -1172,7 +1172,7 @@ def enforce_output_token_estimates_are_admin_only(
 def get_model_rate_limit_from_metadata(
     user_api_key_dict: UserAPIKeyAuth,
     metadata_accessor_key: Literal["team_metadata", "organization_metadata", "project_metadata"],
-    rate_limit_key: Literal["model_rpm_limit", "model_tpm_limit"],
+    rate_limit_key: Literal["model_rpm_limit", "model_tpm_limit", "model_itpm_limit", "model_otpm_limit"],
 ) -> dict[str, int] | None:
     if getattr(user_api_key_dict, metadata_accessor_key):
         return getattr(user_api_key_dict, metadata_accessor_key).get(rate_limit_key)

litellm/proxy/common_utils/registry_read_through.py

@@ -0,0 +1,224 @@
+"""Read-through recovery for in-memory registries in multi-replica deployments.
+
+A management write (POST /model/new, /guardrails, /v1/agents) lands on one
+replica and reaches Postgres, but sibling replicas only refresh their in-memory
+registries on the periodic config reload, so a request using the new object
+immediately can land on a sibling that has never heard of it and fail 400/404.
+On a registry miss, callers here fetch the missing row from the DB and load it
+into the local registry before giving up. A short negative-result TTL per key
+plus a global resync budget per window bound the DB load from lookups of
+genuinely unknown names.
+"""
+
+import asyncio
+import time
+from collections.abc import Awaitable, Callable
+from typing import TYPE_CHECKING, Final
+
+from litellm._logging import verbose_proxy_logger
+from litellm.caching.in_memory_cache import InMemoryCache
+
+if TYPE_CHECKING:
+    from prisma.types import (
+        LiteLLM_AgentsTableInclude,
+        LiteLLM_AgentsTableWhereUniqueInput,
+        LiteLLM_GuardrailsTableWhereInput,
+        LiteLLM_ProxyModelTableWhereInput,
+    )
+
+    from litellm.integrations.custom_guardrail import CustomGuardrail
+    from litellm.types.agents import AgentResponse
+
+READ_THROUGH_MISS_TTL_SECONDS: Final = 2.0
+READ_THROUGH_RESYNC_WINDOW_SECONDS: Final = 5.0
+READ_THROUGH_MAX_RESYNCS_PER_WINDOW: Final = 20
+
+
+class RegistryReadThrough:
+    __slots__ = (
+        "_lock",
+        "_max_resyncs_per_window",
+        "_miss_ttl_seconds",
+        "_recent_misses",
+        "_resync",
+        "_resync_window_seconds",
+        "_window_resyncs",
+        "_window_started_at",
+    )
+
+    def __init__(
+        self,
+        resync: Callable[[str], Awaitable[bool]],
+        miss_ttl_seconds: float = READ_THROUGH_MISS_TTL_SECONDS,
+        max_resyncs_per_window: int = READ_THROUGH_MAX_RESYNCS_PER_WINDOW,
+        resync_window_seconds: float = READ_THROUGH_RESYNC_WINDOW_SECONDS,
+    ) -> None:
+        self._resync = resync
+        self._miss_ttl_seconds = miss_ttl_seconds
+        self._max_resyncs_per_window = max_resyncs_per_window
+        self._resync_window_seconds = resync_window_seconds
+        self._lock = asyncio.Lock()
+        self._recent_misses = InMemoryCache(max_size_in_memory=1000)
+        self._window_started_at = float("-inf")
+        self._window_resyncs = 0
+
+    def _consume_resync_budget(self) -> bool:
+        now: Final = time.monotonic()
+        if now - self._window_started_at >= self._resync_window_seconds:
+            self._window_started_at = now
+            self._window_resyncs = 0
+        if self._window_resyncs >= self._max_resyncs_per_window:
+            return False
+        self._window_resyncs += 1
+        return True
+
+    async def attempt(self, key: str) -> bool:
+        if self._recent_misses.get_cache(key) is not None:
+            return False
+        async with self._lock:
+            if self._recent_misses.get_cache(key) is not None:
+                return False
+            if not self._consume_resync_budget():
+                verbose_proxy_logger.warning(
+                    "registry read-through for %r skipped: resync budget of %s per %ss exhausted",
+                    key,
+                    self._max_resyncs_per_window,
+                    self._resync_window_seconds,
+                )
+                return False
+            try:
+                found: Final = await self._resync(key)
+            except Exception as e:  # noqa: BLE001  # a failed read-through must surface the original miss error, not a 500
+                verbose_proxy_logger.warning("registry read-through for %r failed: %s", key, e)
+                return False
+            if not found:
+                self._recent_misses.set_cache(key, True, ttl=self._miss_ttl_seconds)
+            return found
+
+
+def _db_backed_registries_enabled(object_type: str) -> bool:
+    from litellm.proxy import proxy_server
+
+    if proxy_server.prisma_client is None or proxy_server.store_model_in_db is not True:
+        return False
+    return proxy_server.should_load_db_object(object_type=object_type)
+
+
+async def _resync_model_deployments(model_name: str) -> bool:
+    from litellm.proxy import proxy_server
+    from litellm.repositories.model_repository import ModelRepository
+
+    if not _db_backed_registries_enabled("models"):
+        return False
+    prisma_client: Final = proxy_server.prisma_client
+    assert prisma_client is not None
+    table: Final = ModelRepository(prisma_client).table
+    name_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_name": model_name}
+    id_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_id": model_name}
+    rows: Final = await table.find_many(where=name_filter) or await table.find_many(where=id_filter)
+    if not rows:
+        return False
+    router: Final = proxy_server.llm_router
+    if router is None:
+        await proxy_server.proxy_config.add_deployment(
+            prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj
+        )
+        return proxy_server.llm_router is not None
+    async with proxy_server.MODEL_RECONCILE_LOCK:
+        proxy_server.proxy_config._add_deployment(db_models=rows)
+        proxy_server.llm_model_list = router.get_model_list()
+    return True
+
+
+async def _resync_guardrails(guardrail_name: str) -> bool:
+    from litellm.proxy import proxy_server
+    from litellm.proxy.guardrails.guardrail_registry import (
+        GUARDRAIL_RECONCILE_LOCK,
+        IN_MEMORY_GUARDRAIL_HANDLER,
+    )
+    from litellm.repositories.table_repositories import GuardrailsRepository
+    from litellm.types.guardrails import Guardrail
+
+    if not _db_backed_registries_enabled("guardrails"):
+        return False
+    prisma_client: Final = proxy_server.prisma_client
+    assert prisma_client is not None
+    active_row_filter: Final[LiteLLM_GuardrailsTableWhereInput] = {
+        "guardrail_name": guardrail_name,
+        "status": "active",
+    }
+    row: Final = await GuardrailsRepository(prisma_client).table.find_first(where=active_row_filter)
+    if row is None:
+        return False
+    async with GUARDRAIL_RECONCILE_LOCK:
+        IN_MEMORY_GUARDRAIL_HANDLER.sync_guardrail_from_db(guardrail=Guardrail(**dict(row)))
+    return _initialized_guardrail(guardrail_name) is not None
+
+
+async def _resync_agents(agent_id_or_name: str) -> bool:
+    from litellm.proxy import proxy_server
+    from litellm.proxy.agent_endpoints.agent_registry import (
+        AGENT_RECONCILE_LOCK,
+        agents_table,
+        global_agent_registry,
+    )
+    from litellm.types.agents import AgentResponse
+
+    if not _db_backed_registries_enabled("agents"):
+        return False
+    if _agent_from_registry(agent_id_or_name) is not None:
+        return True
+    prisma_client: Final = proxy_server.prisma_client
+    assert prisma_client is not None
+    table: Final = agents_table(prisma_client)
+    id_filter: Final[LiteLLM_AgentsTableWhereUniqueInput] = {"agent_id": agent_id_or_name}
+    name_filter: Final[LiteLLM_AgentsTableWhereUniqueInput] = {"agent_name": agent_id_or_name}
+    include_permission: Final[LiteLLM_AgentsTableInclude] = {"object_permission": True}
+    async with AGENT_RECONCILE_LOCK:
+        if _agent_from_registry(agent_id_or_name) is not None:
+            return True
+        row: Final = await table.find_unique(where=id_filter, include=include_permission) or await table.find_unique(
+            where=name_filter, include=include_permission
+        )
+        if row is None:
+            return False
+        global_agent_registry.register_agent(agent_config=AgentResponse.model_validate(row.model_dump()))
+        return True
+
+
+model_registry_read_through: Final = RegistryReadThrough(resync=_resync_model_deployments)
+guardrail_registry_read_through: Final = RegistryReadThrough(resync=_resync_guardrails)
+agent_registry_read_through: Final = RegistryReadThrough(resync=_resync_agents)
+
+
+def _agent_from_registry(agent_id_or_name: str) -> "AgentResponse | None":
+    from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
+
+    by_id: Final = global_agent_registry.get_agent_by_id(agent_id=agent_id_or_name)
+    if by_id is not None:
+        return by_id
+    return global_agent_registry.get_agent_by_name(agent_name=agent_id_or_name)
+
+
+async def get_agent_with_read_through(agent_id_or_name: str) -> "AgentResponse | None":
+    agent: Final = _agent_from_registry(agent_id_or_name)
+    if agent is not None:
+        return agent
+    if not await agent_registry_read_through.attempt(agent_id_or_name):
+        return None
+    return _agent_from_registry(agent_id_or_name)
+
+
+def _initialized_guardrail(guardrail_name: str) -> "CustomGuardrail | None":
+    from litellm.proxy.guardrails import guardrail_endpoints
+
+    return guardrail_endpoints.GUARDRAIL_REGISTRY.get_initialized_guardrail_callback(guardrail_name=guardrail_name)
+
+
+async def get_initialized_guardrail_with_read_through(guardrail_name: str) -> "CustomGuardrail | None":
+    active: Final = _initialized_guardrail(guardrail_name)
+    if active is not None:
+        return active
+    if not await guardrail_registry_read_through.attempt(guardrail_name):
+        return None
+    return _initialized_guardrail(guardrail_name)

litellm/proxy/guardrails/guardrail_endpoints.py

@@ -2305,8 +2305,12 @@ async def apply_guardrail(
     litellm_logging_obj = None
     start_time: Final = datetime.now(timezone.utc)
 
+    from litellm.proxy.common_utils.registry_read_through import (
+        get_initialized_guardrail_with_read_through,
+    )
+
     try:
-        active_guardrail: Final[CustomGuardrail | None] = GUARDRAIL_REGISTRY.get_initialized_guardrail_callback(
+        active_guardrail: Final[CustomGuardrail | None] = await get_initialized_guardrail_with_read_through(
             guardrail_name=request.guardrail_name
         )
         if active_guardrail is None:

litellm/proxy/guardrails/guardrail_registry.py

@@ -1,5 +1,6 @@
 # litellm/proxy/guardrails/guardrail_registry.py
 
+import asyncio
 import importlib
 import os
 from collections.abc import Callable, Iterator, Mapping
@@ -813,4 +814,6 @@ class InMemoryGuardrailHandler:
 # In Memory Guardrail Handler for LiteLLM Proxy
 ########################################################
 IN_MEMORY_GUARDRAIL_HANDLER: Final = InMemoryGuardrailHandler()
+
+GUARDRAIL_RECONCILE_LOCK: Final = asyncio.Lock()
 ########################################################

litellm/proxy/hooks/batch_rate_limiter.py

@@ -18,11 +18,12 @@ Quick summary:
 """
 
 import json
-from collections.abc import Iterable
-from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn
+from collections.abc import Iterable, Mapping, Sequence
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, TypeAlias
 
 from fastapi import HTTPException
-from pydantic import BaseModel
+from pydantic import BaseModel, Field, TypeAdapter
 
 import litellm
 from litellm._logging import verbose_proxy_logger
@@ -40,10 +41,15 @@ from litellm.proxy._types import (
     SpecialModelNames,
     UserAPIKeyAuth,
 )
+from litellm.proxy.auth.auth_utils import get_model_rate_limit_from_metadata
 from litellm.proxy.common_utils.proxy_rate_limit_error import (
     ProxyRateLimitError,
     map_v3_rate_limit_type,
 )
+from litellm.proxy.hooks.parallel_request_limiter_v3 import (
+    PROJECT_ITPM_DESCRIPTOR_KEY,
+    PROJECT_OTPM_DESCRIPTOR_KEY,
+)
 from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
 
 if TYPE_CHECKING:
@@ -76,6 +82,11 @@ else:
     RateLimitDescriptor = dict[str, Any]
 
 
+_BATCH_BODY_ADAPTER: Final = TypeAdapter(dict[str, object])
+
+IncrementAmounts: TypeAlias = dict[Literal["requests", "tokens"], int]
+
+
 class BatchFileUsage(BaseModel):
     """
     Internal model for batch file usage tracking, used for batch rate limiting
@@ -83,6 +94,16 @@ class BatchFileUsage(BaseModel):
 
     total_tokens: int
     request_count: int
+    output_tokens: int = 0
+    # Keyed by each row's own `body.model`, distinct from `total_tokens`/
+    # `output_tokens` (the whole-file totals charged to the file-bound/
+    # top-level routing model's key/team/model limits). A batch's rows can
+    # each target a different model, so the project's per-model ITPM/OTPM
+    # quota for a row's actual model must be charged with that row's own
+    # tokens -- see `_create_project_io_descriptors_for_models`.
+    per_model_usage: dict[str, dict[str, int]] = Field(
+        default_factory=dict
+    )  # mutable-ok: accumulated incrementally per row while parsing the batch file
 
 
 class _PROXY_BatchRateLimiter(CustomLogger):
@@ -198,6 +219,15 @@ class _PROXY_BatchRateLimiter(CustomLogger):
         user_api_key_dict: UserAPIKeyAuth,
         data: dict,
     ) -> list["RateLimitDescriptor"]:
+        """Build the standard key/user/team/model descriptor list a batch is charged against.
+
+        Deliberately excludes the project-scoped ITPM/OTPM descriptors: those
+        are charged per the JSONL row's own `body.model` once the file is
+        parsed (`_create_project_io_descriptors_for_models`), not the
+        file-bound/top-level routing model this function resolves. Charging
+        project quotas here would let a caller bind the file to a model
+        without a quota while rows execute against a quota-limited model.
+        """
         return self.parallel_request_limiter._create_rate_limit_descriptors(
             user_api_key_dict=user_api_key_dict,
             data=data,
@@ -206,6 +236,57 @@ class _PROXY_BatchRateLimiter(CustomLogger):
             model_has_failures=False,
         )
 
+    @staticmethod
+    def _project_has_any_io_token_limits(user_api_key_dict: UserAPIKeyAuth) -> bool:
+        """True when the project has any per-model ITPM/OTPM quota configured.
+
+        Used to stop the "skip batch input file processing" fast path from
+        bypassing a project quota configured for a model other than the
+        batch's file-bound/top-level routing model: the row models that
+        actually drive execution and billing aren't known until the JSONL
+        is parsed, so the file must be read whenever *any* model could be
+        quota-limited, not only when the routing model itself is.
+        """
+        if user_api_key_dict.project_id is None:
+            return False
+        return bool(
+            get_model_rate_limit_from_metadata(user_api_key_dict, "project_metadata", "model_itpm_limit")
+        ) or bool(get_model_rate_limit_from_metadata(user_api_key_dict, "project_metadata", "model_otpm_limit"))
+
+    def _create_project_io_descriptors_for_models(
+        self,
+        user_api_key_dict: UserAPIKeyAuth,
+        per_model_usage: Mapping[str, Mapping[str, int]],
+    ) -> tuple[list["RateLimitDescriptor"], list[IncrementAmounts]]:  # mutable-ok: see below
+        """Build project ITPM/OTPM descriptors charged against each row's own model.
+
+        One descriptor pair per distinct `body.model` found in the JSONL,
+        each incremented only by that model's own counted usage -- never the
+        whole-batch total -- so a quota-limited model can't hide behind an
+        unlimited routing model, and an unrelated model's rows can't inflate
+        a different model's counter.
+        """
+        extra_descriptors: Final[list[RateLimitDescriptor]] = []  # mutable-ok: see above
+        extra_increments: Final[list[IncrementAmounts]] = []  # mutable-ok: see above
+        for model, usage in per_model_usage.items():
+            model_descriptors: list[RateLimitDescriptor] = []  # mutable-ok: reset per loop iteration, not module state
+            self.parallel_request_limiter.add_project_io_token_rate_limit_descriptors_from_metadata(
+                user_api_key_dict=user_api_key_dict,
+                requested_model=model,
+                descriptors=model_descriptors,
+            )
+            for descriptor in model_descriptors:
+                extra_descriptors.append(descriptor)
+                extra_increments.append(
+                    {  # mutable-ok: atomic limiter API requires mutable increment records
+                        "requests": 0,
+                        "tokens": usage.get("output_tokens", 0)
+                        if descriptor["key"] == PROJECT_OTPM_DESCRIPTOR_KEY
+                        else usage.get("total_tokens", 0),
+                    }
+                )
+        return extra_descriptors, extra_increments
+
     def _should_skip_batch_input_file_processing(
         self,
         data: dict,
@@ -232,6 +313,11 @@ class _PROXY_BatchRateLimiter(CustomLogger):
         routing deployment's trusted credentials and the batch is constrained
         to run on that provider.
 
+        The no-limits check also treats any project-configured ITPM/OTPM
+        quota as an applicable limit, even when it isn't scoped to the
+        routing model: a row can target a different, quota-limited model,
+        and that isn't knowable without parsing the JSONL.
+
         Returns ``(should_skip, descriptors)`` where ``descriptors`` is the
         rate-limit descriptor list computed for the no-limits check, so the
         caller can reuse it for counter enforcement without recomputing.
@@ -257,7 +343,9 @@ class _PROXY_BatchRateLimiter(CustomLogger):
             user_api_key_dict=user_api_key_dict,
             data=data,
         )
-        if not self._has_applicable_batch_rate_limits(descriptors):
+        if not self._has_applicable_batch_rate_limits(descriptors) and not self._project_has_any_io_token_limits(
+            user_api_key_dict
+        ):
             verbose_proxy_logger.debug("Skipping batch input file processing: no rate limits configured")
             return True, None
 
@@ -297,6 +385,58 @@ class _PROXY_BatchRateLimiter(CustomLogger):
             return False
         return True
 
+    def _estimate_entry_output_tokens(
+        self,
+        entry: Mapping[str, object],
+        min_configured_otpm_limit: int | None,
+    ) -> int:
+        """Conservative per-row output-token estimate for the project OTPM reservation.
+
+        Batch completion never reconciles actual usage back into the rate
+        limiter, so this pre-call estimate is the only OTPM enforcement a
+        batch gets. Mirrors the real-time no-``max_tokens`` floor so a row
+        that omits an output cap can't be used to bypass OTPM the way an
+        unbounded streaming request could.
+
+        Embeddings rows are identified by the row's own ``url`` (the OpenAI
+        batch schema puts the target route there, e.g. ``/v1/embeddings``),
+        never by body shape: a `/v1/responses` row also carries `body.input`
+        with no `messages`/`prompt`, so guessing from body shape alone would
+        misclassify a token-generating Responses row as a zero-output
+        embeddings row and let it skip the OTPM reservation entirely.
+        """
+        url: Final = entry.get("url")
+        if isinstance(url, str) and "embeddings" in url:
+            return 0  # embeddings: no output tokens
+        raw_body: Final = entry.get("body")
+        body: Final[Mapping[str, object]] = (
+            MappingProxyType(_BATCH_BODY_ADAPTER.validate_python(raw_body))
+            if isinstance(raw_body, Mapping)
+            else MappingProxyType({})  # mutable-ok: immediately frozen empty fallback
+        )
+        # `max_tokens`/`max_completion_tokens` cap chat completions; `/v1/responses`
+        # rows cap output with `max_output_tokens` instead -- omitting it here
+        # would fall through to the floor estimate for every capped Responses row.
+        explicit_cap: Final = next(
+            (
+                v
+                for v in (
+                    body.get("max_tokens"),
+                    body.get("max_completion_tokens"),
+                    body.get("max_output_tokens"),
+                )
+                if v is not None
+            ),
+            None,
+        )
+        candidate_count: Final = self.parallel_request_limiter.get_output_candidate_count(body)
+        if explicit_cap is not None:
+            try:
+                return max(0, int(explicit_cap)) * candidate_count
+            except (TypeError, ValueError, OverflowError):
+                pass
+        return self.parallel_request_limiter.no_max_tokens_output_floor(min_configured_otpm_limit) * candidate_count
+
     @staticmethod
     def _has_applicable_batch_rate_limits(
         descriptors: list["RateLimitDescriptor"],
@@ -382,9 +522,22 @@ class _PROXY_BatchRateLimiter(CustomLogger):
         """Raise :class:`ProxyRateLimitError` (a 429) for batch rate limit exceeded."""
         from datetime import datetime
 
-        # Find the descriptor for this status
+        # Find the descriptor for this status. Matching on (key, value) is
+        # required, not key alone: a batch can carry several project ITPM/OTPM
+        # descriptors sharing one key (e.g. `model_per_project_otpm`) but
+        # scoped to different models via `value`
+        # ("{project_id}:{model}") -- key-only matching would always resolve
+        # to the first same-keyed descriptor regardless of which one was
+        # actually over its limit. Falls back to key-only matching for
+        # statuses that predate `descriptor_value` (e.g. from should_rate_limit).
+        status_descriptor_value: Final = status.get("descriptor_value")
         descriptor_index: Final = next(
-            (i for i, d in enumerate(descriptors) if d.get("key") == status.get("descriptor_key")),
+            (
+                i
+                for i, d in enumerate(descriptors)
+                if d.get("key") == status.get("descriptor_key")
+                and (status_descriptor_value is None or d.get("value") == status_descriptor_value)
+            ),
             0,
         )
         descriptor: Final[RateLimitDescriptor] = (
@@ -407,9 +560,27 @@ class _PROXY_BatchRateLimiter(CustomLogger):
                 f"Limit resets at: {reset_time_formatted}"
             )
         else:  # tokens
+            # Project ITPM/OTPM descriptors are keyed "{project_id}:{model}" and
+            # charged with that model's own rows (see
+            # `_create_project_io_descriptors_for_models`), not the whole
+            # batch's totals -- report the matching per-model figure when one
+            # is available so the error reflects what was actually charged.
+            descriptor_model: Final = (
+                descriptor.get("value", "").split(":", 1)[-1]
+                if descriptor.get("key") in (PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY)
+                else None
+            )
+            model_usage: Final = batch_usage.per_model_usage.get(descriptor_model) if descriptor_model else None
+            batch_token_count: Final = (
+                (model_usage or {}).get("output_tokens", batch_usage.output_tokens)
+                if descriptor.get("key") == PROJECT_OTPM_DESCRIPTOR_KEY
+                else (model_usage or {}).get("total_tokens", batch_usage.total_tokens)
+                if descriptor.get("key") == PROJECT_ITPM_DESCRIPTOR_KEY
+                else batch_usage.total_tokens
+            )
             detail = (
                 f"Batch rate limit exceeded for {descriptor.get('key', 'unknown')}: {descriptor.get('value', 'unknown')}. "
-                f"Batch contains {batch_usage.total_tokens} tokens but only {remaining_display} tokens remaining "
+                f"Batch contains {batch_token_count} tokens but only {remaining_display} tokens remaining "
                 f"out of {current_limit} TPM limit. "
                 f"Limit resets at: {reset_time_formatted}"
             )
@@ -444,7 +615,10 @@ class _PROXY_BatchRateLimiter(CustomLogger):
         falls back to a per-process asyncio.Lock + in-memory operation.
 
         ``descriptors`` may be passed in by the pre-call hook to reuse the list
-        already computed when deciding whether to skip file processing.
+        already computed when deciding whether to skip file processing. It
+        never contains project ITPM/OTPM descriptors (those are model-specific
+        and only knowable once ``batch_usage.per_model_usage`` is populated by
+        parsing the JSONL), so this always builds and appends them here.
         """
         if descriptors is None:
             descriptors = self._create_batch_rate_limit_descriptors(
@@ -452,11 +626,20 @@ class _PROXY_BatchRateLimiter(CustomLogger):
                 data=data,
             )
 
-        increment: Final[dict[Literal["requests", "tokens"], int]] = {
-            "requests": batch_usage.request_count,
-            "tokens": batch_usage.total_tokens,
-        }
-        increments: Final[list[dict[Literal["requests", "tokens"], int]]] = [increment for _ in descriptors]
+        increments: list[IncrementAmounts] = [  # mutable-ok: reassigned below to append project IO increments
+            {  # mutable-ok: atomic limiter API requires mutable increment records
+                "requests": batch_usage.request_count,
+                "tokens": batch_usage.total_tokens,
+            }
+            for _d in descriptors
+        ]
+
+        project_io_descriptors, project_io_increments = self._create_project_io_descriptors_for_models(
+            user_api_key_dict=user_api_key_dict,
+            per_model_usage=batch_usage.per_model_usage,
+        )
+        descriptors = [*descriptors, *project_io_descriptors]
+        increments = [*increments, *project_io_increments]
 
         rate_limit_response: Final = await self.parallel_request_limiter.atomic_check_and_increment_by_n(
             descriptors=descriptors,
@@ -482,6 +665,7 @@ class _PROXY_BatchRateLimiter(CustomLogger):
         custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
         user_api_key_dict: UserAPIKeyAuth | None = None,
         data: dict | None = None,
+        descriptors: Sequence["RateLimitDescriptor"] | None = None,
     ) -> BatchFileUsage:
         """
         Count number of requests and tokens in a batch input file.
@@ -490,10 +674,37 @@ class _PROXY_BatchRateLimiter(CustomLogger):
             file_id: The file ID to read
             custom_llm_provider: The custom LLM provider to use for token encoding
             user_api_key_dict: User authentication information for file access (required for managed files)
+            descriptors: Rate limit descriptors already computed for this batch, so the
+                configured project OTPM limit can scale the no-``max_tokens`` output floor
 
         Returns:
-            BatchFileUsage with total_tokens and request_count
+            BatchFileUsage with total_tokens, output_tokens, request_count, and
+            per_model_usage (each row's own totals, keyed by its `body.model`)
         """
+        descriptor_otpm_limits: Final = tuple(
+            int(v)
+            for d in (descriptors or ())
+            if d.get("key") == PROJECT_OTPM_DESCRIPTOR_KEY
+            for rate_limit in (d.get("rate_limit"),)
+            for v in (rate_limit.get("tokens_per_unit") if rate_limit is not None else None,)
+            if v is not None
+        )
+        # `descriptors` only ever carries the routing model's own OTPM limit
+        # (see `_create_batch_rate_limit_descriptors`), but a row can target
+        # any project-configured model. Folding in every configured model's
+        # OTPM limit keeps the no-`max_tokens` floor from drifting wide just
+        # because a row's specific model isn't known until parsed below.
+        project_otpm_limits: Final = (
+            tuple(int(v) for v in project_otpm_limit_map.values())
+            if user_api_key_dict is not None
+            and (
+                project_otpm_limit_map := get_model_rate_limit_from_metadata(
+                    user_api_key_dict, "project_metadata", "model_otpm_limit"
+                )
+            )
+            else ()
+        )
+        min_configured_otpm_limit: Final = min((*descriptor_otpm_limits, *project_otpm_limits), default=None)
         try:
             # Check if this is a managed file (base64 encoded unified file ID)
             from litellm.proxy.openai_files_endpoints.common_utils import (
@@ -545,23 +756,51 @@ class _PROXY_BatchRateLimiter(CustomLogger):
             # Counting stays best-effort, so a legitimate (e.g. multimodal) row
             # the counter can't measure is estimated, not hard-rejected.
             models: Final[set] = set()
+            # Keyed by each row's own `body.model`, so the project ITPM/OTPM
+            # quota for that model is charged with only its own rows' tokens,
+            # never the whole batch's -- see `_create_project_io_descriptors_for_models`.
+            per_model_usage: Final[dict[str, dict[str, int]]] = {}
             total_tokens = 0
+            output_tokens = 0  # rebind-ok: accumulated per JSONL row in the loop below
             request_count = 0
             for raw_line in _iter_batch_input_lines(file_content_bytes):
                 request_count += 1
                 try:
                     entry = json.loads(raw_line)
                 except Exception:
-                    total_tokens += _estimate_batch_entry_tokens(raw_line)
+                    entry_total_tokens = _estimate_batch_entry_tokens(raw_line)
+                    entry_output_tokens = self.parallel_request_limiter.no_max_tokens_output_floor(
+                        min_configured_otpm_limit
+                    )
+                    total_tokens += entry_total_tokens
+                    output_tokens += entry_output_tokens
                     continue
+
+                model: str | None = (entry.get("body") or {}).get("model") if isinstance(entry, dict) else None
+                if model:
+                    models.add(model)
+
                 if isinstance(entry, dict):
-                    model = (entry.get("body") or {}).get("model")
-                    if model:
-                        models.add(model)
+                    entry_output_tokens = self._estimate_entry_output_tokens(entry, min_configured_otpm_limit)
+                else:
+                    entry_output_tokens = self.parallel_request_limiter.no_max_tokens_output_floor(
+                        min_configured_otpm_limit
+                    )
+                output_tokens += entry_output_tokens
+
                 try:
-                    total_tokens += _count_entry_tokens(entry)
+                    entry_total_tokens = _count_entry_tokens(entry)
                 except Exception:
-                    total_tokens += _estimate_batch_entry_tokens(raw_line)
+                    entry_total_tokens = _estimate_batch_entry_tokens(raw_line)
+                total_tokens += entry_total_tokens
+
+                if model:
+                    model_usage = per_model_usage.setdefault(
+                        model, {"total_tokens": 0, "output_tokens": 0, "request_count": 0}
+                    )
+                    model_usage["total_tokens"] += entry_total_tokens
+                    model_usage["output_tokens"] += entry_output_tokens
+                    model_usage["request_count"] += 1
 
             # Validate every model named in the batch JSONL against the
             # caller's per-key model allowlist. Without this, a caller
@@ -578,6 +817,8 @@ class _PROXY_BatchRateLimiter(CustomLogger):
             return BatchFileUsage(
                 total_tokens=total_tokens,
                 request_count=request_count,
+                output_tokens=output_tokens,
+                per_model_usage=per_model_usage,
             )
 
         except HTTPException as e:
@@ -814,6 +1055,7 @@ class _PROXY_BatchRateLimiter(CustomLogger):
                 custom_llm_provider=custom_llm_provider,
                 user_api_key_dict=user_api_key_dict,
                 data=data,
+                descriptors=batch_rate_limit_descriptors,
             )
 
             verbose_proxy_logger.debug(

litellm/proxy/hooks/dynamic_rate_limiter_v3.py

@@ -454,7 +454,9 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger):
             parent_otel_span=user_api_key_dict.parent_otel_span,
         )
 
-        verbose_proxy_logger.debug("Atomic check+increment response: %s", json.dumps(atomic_response, indent=2))
+        verbose_proxy_logger.debug(
+            "Atomic check+increment response: %s", json.dumps(atomic_response, indent=2, default=list)
+        )
 
         if atomic_response["overall_code"] == "OVER_LIMIT":
             resolved_model, llm_provider = resolve_llm_provider_for_rate_limit(model)

litellm/proxy/management_endpoints/common_daily_activity.py

@@ -594,13 +594,16 @@ def _build_aggregated_where_clause(
     sql_params.append(adjusted_end)
     p += 1
 
-    # Optional entity filter
+    # Optional entity filter; an empty list must match nothing, not everything
     if entity_id is not None:
         if isinstance(entity_id, list):
-            placeholders = ", ".join(f"${p + i}" for i in range(len(entity_id)))
-            sql_conditions.append(f'"{entity_id_field}" IN ({placeholders})')
-            sql_params.extend(entity_id)
-            p += len(entity_id)
+            if entity_id:
+                placeholders = ", ".join(f"${p + i}" for i in range(len(entity_id)))
+                sql_conditions.append(f'"{entity_id_field}" IN ({placeholders})')
+                sql_params.extend(entity_id)
+                p += len(entity_id)
+            else:
+                sql_conditions.append("FALSE")
         else:
             sql_conditions.append(f'"{entity_id_field}" = ${p}')
             sql_params.append(entity_id)

litellm/proxy/management_endpoints/model_access_group_management_endpoints.py

@@ -57,20 +57,37 @@ def _model_table(prisma_client: PrismaClient) -> _ModelTableClient:
     return ModelRepository(prisma_client).table
 
 
-def validate_models_exist(model_names: list[str], llm_router: "Router | None") -> tuple[bool, list[str]]:
+def validate_models_exist(model_names: Sequence[str], llm_router: "Router | None") -> tuple[bool, Sequence[str]]:
     """
     Validate that all requested model names exist in the router.
     Checks only exact model name matches.
 
     Returns:
-        Tuple[bool, List[str]]: (all_valid, missing_models)
+        (all_valid, missing_models)
     """
     if llm_router is None:
         return False, model_names
 
-    router_model_names: Final = set(llm_router.get_model_names())
-    missing: Final = [m for m in model_names if m not in router_model_names]
-    return (len(missing) == 0, missing)
+    router_model_names: Final = frozenset(llm_router.get_model_names())
+    missing: Final = tuple(m for m in model_names if m not in router_model_names)
+    return (not missing, missing)
+
+
+async def _missing_models_after_read_through(
+    model_names: Sequence[str], llm_router: "Router | None"
+) -> tuple[str, ...]:
+    from litellm.proxy import proxy_server
+    from litellm.proxy.common_utils.registry_read_through import (
+        model_registry_read_through,
+    )
+
+    _, missing = validate_models_exist(model_names=model_names, llm_router=llm_router)
+    if not missing:
+        return ()
+    for name in missing:
+        await model_registry_read_through.attempt(name)
+    _, still_missing = validate_models_exist(model_names=model_names, llm_router=proxy_server.llm_router)
+    return tuple(still_missing)
 
 
 def add_access_group_to_deployment(model_info: dict[str, Any], access_group: str) -> tuple[dict[str, Any], bool]:
@@ -101,13 +118,21 @@ def _raise_http_if_reload_degraded_serving(
     before: frozenset[str],
     written_models: Sequence[tuple[str, object]],
     access_group: str,
+    still_desired: frozenset[str] | None,
+    live_after: frozenset[str] | None,
 ) -> None:
     """Same verdict as the model-write endpoints, expressed through this file's
     HTTPException error convention, with the metadata-only obligation: these writes
     change group membership, not the models themselves, so a row that was already not
     serving before the reload is never blamed here; only a model this reload stopped
     serving is reported."""
-    missing, collateral = reload_serving_verdict(before=before, written_models=written_models, written_must_serve=False)
+    missing, collateral = reload_serving_verdict(
+        before=before,
+        written_models=written_models,
+        written_must_serve=False,
+        still_desired=still_desired,
+        live_after=live_after,
+    )
     gone: Final = tuple(dict.fromkeys((*missing, *collateral)))
     if not gone:
         return
@@ -390,12 +415,12 @@ async def create_model_group(
     # Validate model_names exist in router (only if using model_names path)
     if not use_model_ids and has_model_names:
         assert data.model_names is not None
-        all_valid, missing_models = validate_models_exist(
+        missing_models: Final = await _missing_models_after_read_through(
             model_names=data.model_names,
             llm_router=llm_router,
         )
 
-        if not all_valid:
+        if missing_models:
             raise HTTPException(
                 status_code=400,
                 detail={"error": f"Model(s) not found: {', '.join(missing_models)}"},
@@ -439,11 +464,13 @@ async def create_model_group(
 
         live_before_reload: Final = live_model_ids_snapshot()
 
-        await clear_cache()
+        reload_outcome: Final = await clear_cache()
         _raise_http_if_reload_degraded_serving(
             before=live_before_reload,
             written_models=updated_pairs,
             access_group=data.access_group,
+            still_desired=reload_outcome.still_desired,
+            live_after=reload_outcome.live_after,
         )
 
         verbose_proxy_logger.info(
@@ -654,12 +681,12 @@ async def update_access_group(
     # Validation: Check if all new models exist (only if using model_names path)
     if not use_model_ids and has_model_names:
         assert data.model_names is not None
-        all_valid, missing_models = validate_models_exist(
+        missing_models: Final = await _missing_models_after_read_through(
             model_names=data.model_names,
             llm_router=llm_router,
         )
 
-        if not all_valid:
+        if missing_models:
             raise HTTPException(
                 status_code=400,
                 detail={"error": f"Model(s) not found: {', '.join(missing_models)}"},
@@ -699,11 +726,13 @@ async def update_access_group(
 
         # Clear cache and reload models to pick up the access group changes
         live_before_reload: Final = live_model_ids_snapshot()
-        await clear_cache()
+        reload_outcome: Final = await clear_cache()
         _raise_http_if_reload_degraded_serving(
             before=live_before_reload,
             written_models=list({**dict(stripped_pairs), **dict(updated_pairs)}.items()),
             access_group=access_group,
+            still_desired=reload_outcome.still_desired,
+            live_after=reload_outcome.live_after,
         )
 
         verbose_proxy_logger.info(
@@ -801,11 +830,13 @@ async def delete_access_group(
 
         # Clear cache and reload models to pick up the access group changes
         live_before_reload: Final = live_model_ids_snapshot()
-        await clear_cache()
+        reload_outcome: Final = await clear_cache()
         _raise_http_if_reload_degraded_serving(
             before=live_before_reload,
             written_models=removed_pairs,
             access_group=access_group,
+            still_desired=reload_outcome.still_desired,
+            live_after=reload_outcome.live_after,
         )
 
         verbose_proxy_logger.info(

litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py

@@ -50,7 +50,7 @@ from litellm.proxy.vector_store_endpoints.utils import (
     get_litellm_managed_vector_store,
     is_allowed_to_call_vector_store_endpoint,
 )
-from litellm.secret_managers.main import get_secret_str
+from litellm.secret_managers.main import get_secret_str, str_to_bool
 from litellm.types.passthrough_endpoints.pass_through_endpoints import (
     LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY,
     LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
@@ -1016,15 +1016,21 @@ async def bedrock_proxy_route(
         raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
 
     aws_region_name: Final = litellm.utils.get_secret(secret_name="AWS_REGION_NAME")
-    if _is_bedrock_agent_runtime_route(endpoint=endpoint):  # handle bedrock agents
-        base_target_url: Final = f"https://bedrock-agent-runtime.{aws_region_name}.amazonaws.com"
-    else:
+    if not _is_bedrock_agent_runtime_route(endpoint=endpoint):
         return await bedrock_llm_proxy_route(
             endpoint=endpoint,
             request=request,
             fastapi_response=fastapi_response,
             user_api_key_dict=user_api_key_dict,
         )
+
+    if _is_bedrock_agent_runtime_passthrough_disabled():
+        raise HTTPException(
+            status_code=403,
+            detail="bedrock-agent-runtime pass-through is disabled on this proxy.",
+        )
+
+    base_target_url: Final = f"https://bedrock-agent-runtime.{aws_region_name}.amazonaws.com"
     encoded_endpoint = httpx.URL(endpoint).path
 
     # Ensure endpoint starts with '/' for proper URL construction
@@ -1292,6 +1298,15 @@ def _is_bedrock_agent_runtime_route(endpoint: str) -> bool:
     return False
 
 
+def _is_bedrock_agent_runtime_passthrough_disabled() -> bool:
+    from litellm.proxy.proxy_server import general_settings
+
+    setting: Final = general_settings.get("disable_bedrock_agent_runtime_passthrough")
+    if isinstance(setting, str):
+        return str_to_bool(setting) is True
+    return setting is True
+
+
 @router.api_route(
     "/assemblyai/{endpoint:path}",
     methods=["GET", "POST", "PUT", "DELETE", "PATCH"],

litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py

@@ -229,6 +229,25 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
             verbose_proxy_logger.warning("Error calculating image editing cost: %s", e)
             return 0.0
 
+    @staticmethod
+    def _calculate_embeddings_cost(
+        litellm_model_response: EmbeddingResponse,
+        model: str,
+        custom_llm_provider: str,
+    ) -> float:
+        try:
+            return litellm.completion_cost(
+                completion_response=litellm_model_response,
+                model=model,
+                custom_llm_provider=custom_llm_provider,
+                call_type="aembedding",
+            )
+        except Exception as e:  # noqa: BLE001  # completion_cost raises bare Exception for unmapped models; cost failure must never drop the spend log
+            verbose_proxy_logger.warning(
+                "Error calculating embeddings cost for model %s, logging spend with cost 0: %s", model, e
+            )
+            return 0.0
+
     @staticmethod
     def _build_responses_api_response_and_cost(
         model: str,
@@ -351,11 +370,10 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
                     model_response_object=EmbeddingResponse(),
                     response_type="embedding",
                 )
-                response_cost = litellm.completion_cost(
-                    completion_response=litellm_model_response,
+                response_cost = OpenAIPassthroughLoggingHandler._calculate_embeddings_cost(
+                    litellm_model_response=litellm_model_response,
                     model=model,
                     custom_llm_provider=custom_llm_provider,
-                    call_type="aembedding",
                 )
                 litellm_model_response._hidden_params["response_cost"] = response_cost
             elif is_image_generation:
@@ -471,6 +489,12 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
 
         except Exception as e:
             verbose_proxy_logger.error("Error in OpenAI passthrough cost tracking: %s", e)
+            if not is_chat_completions:
+                unbilled_result: Final[PassThroughEndpointLoggingTypedDict] = {
+                    "result": None,
+                    "kwargs": kwargs,
+                }
+                return unbilled_result
             # Fall back to base handler without cost tracking
             base_handler = OpenAIPassthroughLoggingHandler()
             return base_handler.passthrough_chat_handler(

litellm/proxy/proxy_server.py

@@ -4120,6 +4120,31 @@ def _swap_in_model_cost_map(new_model_cost_map: dict) -> int:
     return fetched_model_count
 
 
+def should_load_db_object(object_type: str | SupportedDBObjectType) -> bool:
+    """
+    Check if an object type should be loaded from the database based on general_settings.supported_db_objects.
+
+    Args:
+        object_type: Type of object to check (e.g., SupportedDBObjectType.MODELS, "models", etc.)
+
+    Returns:
+        True if the object should be loaded, False otherwise
+    """
+    supported_db_objects: Final = general_settings.get("supported_db_objects", None)
+
+    if supported_db_objects is None:
+        return True
+
+    if not isinstance(supported_db_objects, list):
+        verbose_proxy_logger.warning(
+            "supported_db_objects is not a list, got %s. Loading all objects.", type(supported_db_objects)
+        )
+        return True
+
+    object_type_str: Final = str(object_type)
+    return any(str(obj) == object_type_str for obj in supported_db_objects)
+
+
 class ProxyConfig:
     """
     Abstraction class on top of config loading/updating logic. Gives us one place to control all config updating logic.
@@ -6522,36 +6547,7 @@ class ProxyConfig:
         return config
 
     def _should_load_db_object(self, object_type: str | SupportedDBObjectType) -> bool:
-        """
-        Check if an object type should be loaded from the database based on general_settings.supported_db_objects.
-
-        Args:
-            object_type: Type of object to check (e.g., SupportedDBObjectType.MODELS, "models", etc.)
-
-        Returns:
-            True if the object should be loaded, False otherwise
-        """
-        global general_settings
-
-        # Get the supported_db_objects configuration
-        supported_db_objects: Final = general_settings.get("supported_db_objects", None)
-
-        # If supported_db_objects is not set, load all objects (default behavior)
-        if supported_db_objects is None:
-            return True
-
-        # If supported_db_objects is set, only load specified objects
-        if not isinstance(supported_db_objects, list):
-            verbose_proxy_logger.warning(
-                "supported_db_objects is not a list, got %s. Loading all objects.", type(supported_db_objects)
-            )
-            return True
-
-        # Convert object_type to string for comparison (handles both str and enum)
-        object_type_str: Final = str(object_type)
-
-        # Check if the object type is in the list (supports both str and enum values)
-        return any(str(obj) == object_type_str for obj in supported_db_objects)
+        return should_load_db_object(object_type=object_type)
 
     async def _get_models_from_db(self, prisma_client: PrismaClient) -> list | None:
         """
@@ -7094,38 +7090,40 @@ class ProxyConfig:
 
     async def _init_guardrails_in_db(self, prisma_client: PrismaClient):
         from litellm.proxy.guardrails.guardrail_registry import (
+            GUARDRAIL_RECONCILE_LOCK,
             IN_MEMORY_GUARDRAIL_HANDLER,
             Guardrail,
             GuardrailRegistry,
         )
 
         try:
-            guardrails_in_db: Final[list[Guardrail]] = await GuardrailRegistry.get_all_guardrails_from_db(
-                prisma_client=prisma_client
-            )
-            verbose_proxy_logger.debug("guardrails from the DB %s", str(guardrails_in_db))
-            db_guardrail_ids: Final[set] = set()
-            for guardrail in guardrails_in_db:
-                guardrail_id = guardrail.get("guardrail_id")
-                if guardrail_id:
-                    db_guardrail_ids.add(guardrail_id)
-                try:
-                    IN_MEMORY_GUARDRAIL_HANDLER.sync_guardrail_from_db(
-                        guardrail=cast(Guardrail, guardrail),
-                    )
-                except Exception as e:  # noqa: BLE001  # one unloadable row must not stop the remaining guardrails
-                    verbose_proxy_logger.error(
-                        "litellm.proxy.proxy_server.py::ProxyConfig:_init_guardrails_in_db - "
-                        "skipping guardrail '%s' (ID: %s): %s: %s",
-                        guardrail.get("guardrail_name"),
-                        guardrail_id,
-                        type(e).__name__,
-                        e,
-                    )
+            async with GUARDRAIL_RECONCILE_LOCK:
+                guardrails_in_db: Final[list[Guardrail]] = await GuardrailRegistry.get_all_guardrails_from_db(
+                    prisma_client=prisma_client
+                )
+                verbose_proxy_logger.debug("guardrails from the DB %s", str(guardrails_in_db))
+                db_guardrail_ids: Final[set] = set()
+                for guardrail in guardrails_in_db:
+                    guardrail_id = guardrail.get("guardrail_id")
+                    if guardrail_id:
+                        db_guardrail_ids.add(guardrail_id)
+                    try:
+                        IN_MEMORY_GUARDRAIL_HANDLER.sync_guardrail_from_db(
+                            guardrail=cast(Guardrail, guardrail),
+                        )
+                    except Exception as e:  # noqa: BLE001  # one unloadable row must not stop the remaining guardrails
+                        verbose_proxy_logger.error(
+                            "litellm.proxy.proxy_server.py::ProxyConfig:_init_guardrails_in_db - "
+                            "skipping guardrail '%s' (ID: %s): %s: %s",
+                            guardrail.get("guardrail_name"),
+                            guardrail_id,
+                            type(e).__name__,
+                            e,
+                        )
 
-            # Drop in-memory DB-backed entries whose row was deleted on another
-            # pod. Config-loaded entries are never touched.
-            IN_MEMORY_GUARDRAIL_HANDLER.reconcile_db_guardrails(db_guardrail_ids=db_guardrail_ids)
+                # Drop in-memory DB-backed entries whose row was deleted on another
+                # pod. Config-loaded entries are never touched.
+                IN_MEMORY_GUARDRAIL_HANDLER.reconcile_db_guardrails(db_guardrail_ids=db_guardrail_ids)
         except Exception as e:
             verbose_proxy_logger.exception("litellm.proxy.proxy_server.py::ProxyConfig:_init_guardrails_in_db - %s", e)
 
@@ -7278,13 +7276,17 @@ class ProxyConfig:
             )
 
     async def _init_agents_in_db(self, prisma_client: PrismaClient):
+        from litellm.proxy.agent_endpoints.agent_registry import (
+            AGENT_RECONCILE_LOCK,
+        )
         from litellm.proxy.agent_endpoints.agent_registry import (
             global_agent_registry as AGENT_REGISTRY,
         )
 
         try:
-            db_agents: Final = await AGENT_REGISTRY.get_all_agents_from_db(prisma_client=prisma_client)
-            AGENT_REGISTRY.load_agents_from_db_and_config(db_agents=db_agents)
+            async with AGENT_RECONCILE_LOCK:
+                db_agents: Final = await AGENT_REGISTRY.get_all_agents_from_db(prisma_client=prisma_client)
+                AGENT_REGISTRY.load_agents_from_db_and_config(db_agents=db_agents)
         except Exception as e:
             verbose_proxy_logger.exception("litellm.proxy.proxy_server.py::ProxyConfig:_init_agents_in_db - %s", e)
 
@@ -10254,11 +10256,9 @@ async def embeddings(
 
 """
     global proxy_logging_obj
-    data: Any = {}
+    data: Final = await _read_request_body(request=request)
+    base_llm_response_processor: Final = ProxyBaseLLMRequestProcessing(data=data)
     try:
-        # Use shared request body reading helper (same as chat/completions)
-        data = await _read_request_body(request=request)
-
         ### HANDLE TOKEN ARRAY INPUT DECODING ###
         # This must happen BEFORE base_process_llm_request() since it modifies the input
         router_model_names: Final = llm_router.model_names if llm_router is not None else []
@@ -10302,10 +10302,6 @@ async def embeddings(
             if hasattr(user_api_key_dict, "agent_id") and user_api_key_dict.agent_id is not None:
                 data["metadata"]["agent_id"] = user_api_key_dict.agent_id
 
-        # Use unified request processor (same as chat/completions and responses)
-        base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data)
-
-        # Process the request with all optimizations (shared sessions, network tuning, etc.)
         response: Final = await base_llm_response_processor.base_process_llm_request(
             request=request,
             fastapi_response=fastapi_response,
@@ -10327,8 +10323,6 @@ async def embeddings(
 
         return response
     except Exception as e:
-        # Use unified error handler
-        base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data)
         raise await base_llm_response_processor._handle_llm_api_exception(
             e=e,
             user_api_key_dict=user_api_key_dict,

litellm/proxy/route_llm_request.py

@@ -146,7 +146,8 @@ ROUTE_ENDPOINT_MAPPING: Final = {
 
 
 class ProxyModelNotFoundError(HTTPException):
-    def __init__(self, route: str, model_name: str):
+    def __init__(self, route: str, model_name: str, retryable_with_model_read_through: bool = True):
+        self.retryable_with_model_read_through: Final = retryable_with_model_read_through
         detail: Final = {
             "error": f"{route}: Invalid model name passed in model={model_name}. Call `/v1/models` to view available models for your key."
         }
@@ -320,112 +321,150 @@ async def add_shared_session_to_data(data: dict) -> None:
             pass
 
 
+RouteType = Literal[
+    "acompletion",
+    "atext_completion",
+    "aembedding",
+    "aimage_generation",
+    "aspeech",
+    "atranscription",
+    "amoderation",
+    "arerank",
+    "aresponses",
+    "aget_responses",
+    "adelete_responses",
+    "acancel_responses",
+    "acompact_responses",
+    "acreate_response_reply",
+    "alist_input_items",
+    "_arealtime",  # private function for realtime API
+    "acreate_realtime_client_secret",
+    "arealtime_calls",
+    "acreate_realtime_transcription_session",
+    "_aresponses_websocket",  # private function for responses WebSocket mode
+    "aimage_edit",
+    "agenerate_content",
+    "agenerate_content_stream",
+    "allm_passthrough_route",
+    "acreate_batch",
+    "aretrieve_batch",
+    "alist_batches",
+    "afile_content",
+    "afile_retrieve",
+    "acreate_fine_tuning_job",
+    "acancel_fine_tuning_job",
+    "alist_fine_tuning_jobs",
+    "aretrieve_fine_tuning_job",
+    "avector_store_search",
+    "avector_store_create",
+    "avector_store_retrieve",
+    "avector_store_list",
+    "avector_store_update",
+    "avector_store_delete",
+    "avector_store_file_create",
+    "avector_store_file_list",
+    "avector_store_file_retrieve",
+    "avector_store_file_content",
+    "avector_store_file_update",
+    "avector_store_file_delete",
+    "aocr",
+    "asearch",
+    "avideo_generation",
+    "avideo_list",
+    "avideo_status",
+    "avideo_content",
+    "avideo_remix",
+    "avideo_create_character",
+    "avideo_get_character",
+    "avideo_edit",
+    "avideo_extension",
+    "acreate_container",
+    "alist_containers",
+    "aretrieve_container",
+    "adelete_container",
+    "aupload_container_file",
+    "alist_container_files",
+    "aretrieve_container_file",
+    "adelete_container_file",
+    "aretrieve_container_file_content",
+    "acreate_skill",
+    "alist_skills",
+    "aget_skill",
+    "adelete_skill",
+    "aingest",
+    "anthropic_messages",
+    "acreate_interaction",
+    "aget_interaction",
+    "adelete_interaction",
+    "acancel_interaction",
+    "acreate_agent",
+    "alist_agents",
+    "aget_agent",
+    "adelete_agent",
+    "alist_agent_versions",
+    "asend_message",
+    "call_mcp_tool",
+    "acancel_batch",
+    "afile_delete",
+    "acreate_eval",
+    "alist_evals",
+    "aget_eval",
+    "aupdate_eval",
+    "adelete_eval",
+    "acancel_eval",
+    "acreate_run",
+    "alist_runs",
+    "aget_run",
+    "acancel_run",
+    "adelete_run",
+]
+
+
 async def route_request(
     data: dict,
     llm_router: LitellmRouter | None,
     user_model: str | None,
-    route_type: Literal[
-        "acompletion",
-        "atext_completion",
-        "aembedding",
-        "aimage_generation",
-        "aspeech",
-        "atranscription",
-        "amoderation",
-        "arerank",
-        "aresponses",
-        "aget_responses",
-        "adelete_responses",
-        "acancel_responses",
-        "acompact_responses",
-        "acreate_response_reply",
-        "alist_input_items",
-        "_arealtime",  # private function for realtime API
-        "acreate_realtime_client_secret",
-        "arealtime_calls",
-        "acreate_realtime_transcription_session",
-        "_aresponses_websocket",  # private function for responses WebSocket mode
-        "aimage_edit",
-        "agenerate_content",
-        "agenerate_content_stream",
-        "allm_passthrough_route",
-        "acreate_batch",
-        "aretrieve_batch",
-        "alist_batches",
-        "afile_content",
-        "afile_retrieve",
-        "acreate_fine_tuning_job",
-        "acancel_fine_tuning_job",
-        "alist_fine_tuning_jobs",
-        "aretrieve_fine_tuning_job",
-        "avector_store_search",
-        "avector_store_create",
-        "avector_store_retrieve",
-        "avector_store_list",
-        "avector_store_update",
-        "avector_store_delete",
-        "avector_store_file_create",
-        "avector_store_file_list",
-        "avector_store_file_retrieve",
-        "avector_store_file_content",
-        "avector_store_file_update",
-        "avector_store_file_delete",
-        "aocr",
-        "asearch",
-        "avideo_generation",
-        "avideo_list",
-        "avideo_status",
-        "avideo_content",
-        "avideo_remix",
-        "avideo_create_character",
-        "avideo_get_character",
-        "avideo_edit",
-        "avideo_extension",
-        "acreate_container",
-        "alist_containers",
-        "aretrieve_container",
-        "adelete_container",
-        "aupload_container_file",
-        "alist_container_files",
-        "aretrieve_container_file",
-        "adelete_container_file",
-        "aretrieve_container_file_content",
-        "acreate_skill",
-        "alist_skills",
-        "aget_skill",
-        "adelete_skill",
-        "aingest",
-        "anthropic_messages",
-        "acreate_interaction",
-        "aget_interaction",
-        "adelete_interaction",
-        "acancel_interaction",
-        "acreate_agent",
-        "alist_agents",
-        "aget_agent",
-        "adelete_agent",
-        "alist_agent_versions",
-        "asend_message",
-        "call_mcp_tool",
-        "acancel_batch",
-        "afile_delete",
-        "acreate_eval",
-        "alist_evals",
-        "aget_eval",
-        "aupdate_eval",
-        "adelete_eval",
-        "acancel_eval",
-        "acreate_run",
-        "alist_runs",
-        "aget_run",
-        "acancel_run",
-        "adelete_run",
-    ],
+    route_type: RouteType,
     user_api_key_dict: UserAPIKeyAuth | None = None,
 ):
     """
     Common helper to route the request
     """
+    try:
+        return await _route_request_single_attempt(
+            data=data,
+            llm_router=llm_router,
+            user_model=user_model,
+            route_type=route_type,
+            user_api_key_dict=user_api_key_dict,
+        )
+    except ProxyModelNotFoundError as e:
+        requested_model: Final = data.get("model", "")
+        if not e.retryable_with_model_read_through or not isinstance(requested_model, str) or not requested_model:
+            raise
+        from litellm.proxy import proxy_server
+        from litellm.proxy.common_utils.registry_read_through import (
+            model_registry_read_through,
+        )
+
+        if not await model_registry_read_through.attempt(requested_model):
+            raise
+        return await _route_request_single_attempt(
+            data=data,
+            llm_router=proxy_server.llm_router,
+            user_model=user_model,
+            route_type=route_type,
+            user_api_key_dict=user_api_key_dict,
+        )
+
+
+async def _route_request_single_attempt(  # noqa: ANN202  # returns unawaited provider coroutines; the inferred union keeps route_request's callers typed
+    data: dict,  # mutable-ok: request body is the proxy-wide mutable dict contract shared with route_request
+    llm_router: LitellmRouter | None,
+    user_model: str | None,
+    route_type: RouteType,
+    user_api_key_dict: UserAPIKeyAuth | None = None,
+):
     raise_if_required_body_param_missing(route_type=route_type, data=data)
 
     await add_shared_session_to_data(data)

litellm/proxy/utils.py

@@ -40,7 +40,7 @@ from litellm.proxy._types import (
 from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
 from litellm.types.guardrails import GuardrailEventHooks
 from litellm.types.proxy.model_listing import ModelInfoResponse
-from litellm.types.utils import CallTypes, CallTypesLiteral, ModelInfo
+from litellm.types.utils import CallTypes, CallTypesLiteral, ModelInfo, Usage
 
 try:
     from litellm_enterprise.enterprise_callbacks.send_emails.base_email import (
@@ -403,6 +403,120 @@ def _exception_changes_request_flow(exc: BaseException) -> bool:
     return isinstance(exc, (SensitiveDataRouteException, ModifyResponseException))
 
 
+def _prompt_block_text(block: object) -> str:
+    if isinstance(block, str):
+        return block
+    if not isinstance(block, dict):
+        return ""
+    block_text: Final = block.get("text")
+    return block_text if isinstance(block_text, str) else ""
+
+
+def _system_prompt_text(system_input: object) -> str:
+    if isinstance(system_input, str):
+        return system_input
+    if not isinstance(system_input, list):
+        return ""
+    return "".join(_prompt_block_text(block) for block in system_input)
+
+
+def _count_request_input_tokens(model: str, request_input: object, system_input: object) -> int:
+    system_text: Final = _system_prompt_text(system_input)
+    system_tokens: Final = litellm.token_counter(model=model, text=system_text) if system_text else 0
+    if isinstance(request_input, str):
+        return system_tokens + litellm.token_counter(model=model, text=request_input)
+    if not isinstance(request_input, list) or not request_input:
+        return system_tokens
+    text_entries: Final = tuple(entry for entry in request_input if isinstance(entry, str))
+    if len(text_entries) == len(request_input):
+        return system_tokens + litellm.token_counter(model=model, text="".join(text_entries))
+    return system_tokens + litellm.token_counter(
+        model=model, messages=request_input, use_default_image_token_count=True
+    )
+
+
+def _estimate_dispatched_failure_usage(model: str, request_input: object, system_input: object) -> Usage | None:
+    """A request that failed after dispatch consumed provider-billed input
+    tokens, but no provider usage ever came back. Estimate the input side with
+    the same tokenizer fallback interrupted streams use, so the spend log's
+    failure row records what was sent instead of zero."""
+    try:
+        input_tokens: Final = _count_request_input_tokens(
+            model=model, request_input=request_input, system_input=system_input
+        )
+    except Exception:
+        return None
+    if input_tokens <= 0:
+        return None
+    return Usage(prompt_tokens=input_tokens, completion_tokens=0, total_tokens=input_tokens)
+
+
+_INPUT_ESTIMABLE_CALL_TYPES: Final = frozenset(
+    call_type.value
+    for call_type in (
+        CallTypes.completion,
+        CallTypes.acompletion,
+        CallTypes.text_completion,
+        CallTypes.atext_completion,
+        CallTypes.anthropic_messages,
+        CallTypes.aanthropic_messages,
+        CallTypes.responses,
+        CallTypes.aresponses,
+        CallTypes.embedding,
+        CallTypes.aembedding,
+        CallTypes.moderation,
+        CallTypes.amoderation,
+        CallTypes.image_generation,
+        CallTypes.aimage_generation,
+        CallTypes.speech,
+        CallTypes.aspeech,
+        CallTypes.rerank,
+        CallTypes.arerank,
+        CallTypes.generate_content,
+        CallTypes.agenerate_content,
+        CallTypes.generate_content_stream,
+        CallTypes.agenerate_content_stream,
+    )
+)
+
+
+def _failure_usage_to_lift(
+    model_call_details: Mapping[str, object],
+    request_body: Mapping[str, object],
+    dispatched: bool,
+) -> tuple[object, object] | None:
+    """A stream that broke mid-flight still billed the provider for the chunks
+    already delivered; the streaming handler stashes that recovered usage and
+    cost in model_call_details, so prefer it. Otherwise a request that was
+    dispatched to a provider and failed without upstream usage gets an
+    estimated input-side Usage with zero cost. The raw request body backfills
+    the system prompt when the SDK bridges an endpoint (e.g. /v1/messages on a
+    chat-completions provider) without filling optional_params. Returns the
+    (combined_usage_object, response_cost) pair to lift, or None."""
+    recovered_usage: Final = model_call_details.get("combined_usage_object")
+    if recovered_usage is not None:
+        return recovered_usage, model_call_details.get("response_cost")
+    if not dispatched or model_call_details.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL):
+        return None
+    if str(model_call_details.get("call_type")) not in _INPUT_ESTIMABLE_CALL_TYPES:
+        return None
+    optional_params: Final = model_call_details.get("optional_params")
+    dispatched_system: Final = (
+        (optional_params.get("system") or optional_params.get("instructions"))
+        if isinstance(optional_params, dict)
+        else None
+    )
+    system_input: Final = dispatched_system or request_body.get("system") or request_body.get("instructions")
+    estimated_usage: Final = _estimate_dispatched_failure_usage(
+        model=str(model_call_details.get("model") or ""),
+        request_input=model_call_details.get("messages"),
+        system_input=system_input,
+    )
+    if estimated_usage is None:
+        return None
+    return estimated_usage, 0.0
+
+
 @dataclass(frozen=True)
 class _CallbackCapabilities:
     """Cached per-hook capability flags derived from ``litellm.callbacks``.
@@ -2190,15 +2304,19 @@ class ProxyLogging:
             if _first_handoff is not None:
                 request_data["first_api_call_start_time"] = _first_handoff
 
-            # A stream that broke mid-flight still billed the provider for the
-            # chunks already delivered; the streaming handler stashes that
-            # recovered usage and cost here. Lift them onto request_data so the
+            # Lift recovered partial-stream usage, or an estimated input-side
+            # usage for a dispatched failure, onto request_data so the
             # failure-path spend callbacks (which run after the logging object
-            # is popped) record the real partial spend instead of zero.
-            _recovered_usage: Final = _model_call_details.get("combined_usage_object")
-            if _recovered_usage is not None:
-                request_data["combined_usage_object"] = _recovered_usage
-                request_data["response_cost"] = _model_call_details.get("response_cost")
+            # is popped) record real token counts instead of zero.
+            _usage_to_lift: Final = _failure_usage_to_lift(
+                model_call_details=_model_call_details,
+                request_body=request_data,
+                dispatched=_first_handoff is not None,
+            )
+            if _usage_to_lift is not None:
+                _lifted_usage, _lifted_cost = _usage_to_lift
+                request_data["combined_usage_object"] = _lifted_usage
+                request_data["response_cost"] = _lifted_cost
 
         # Remove before callbacks iterate — not serialisable
         request_data.pop("litellm_logging_obj", None)

litellm/responses/streaming_iterator.py

@@ -20,7 +20,7 @@ from litellm.constants import (
     LITELLM_MAX_STREAMING_DURATION_SECONDS,
     STREAM_SSE_DONE_STRING,
 )
-from litellm.exceptions import MidStreamFallbackError
+from litellm.exceptions import MidStreamFallbackError, RateLimitError
 from litellm.litellm_core_utils.asyncify import run_async_function
 from litellm.litellm_core_utils.core_helpers import process_response_headers
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@@ -50,6 +50,16 @@ if TYPE_CHECKING:
     )
 
 
+class ProjectQuotaCallback(Protocol):
+    async def enforce_project_io_token_quota_for_frame(
+        self,
+        user_api_key_dict: UserAPIKeyAuth | None,
+        requested_model: str | None,
+        estimated_input_tokens: int,
+        estimated_output_tokens: int,
+    ) -> None: ...
+
+
 @lru_cache(maxsize=1)
 def _get_openai_response_types():
     from litellm.types.llms import openai as openai_types
@@ -1326,6 +1336,84 @@ def _build_synthetic_response_events(
 
 from litellm._logging import verbose_logger
 
+# Conservative per-frame output-token floor used when a response.create
+# frame omits max_output_tokens, so a project OTPM quota can't be bypassed
+# by simply never declaring an output cap.
+_FRAME_NO_MAX_OUTPUT_TOKENS_FLOOR: Final = 1024
+
+# Rough chars-per-token ratio for estimating a frame's input tokens without
+# resolving a real per-model tokenizer, matching the conservative estimate
+# the proxy's own rate limiter uses for the same purpose.
+_FRAME_CHARS_PER_TOKEN_ESTIMATE: Final = 4
+
+
+def _extract_frame_quota_estimate_inputs(msg_obj: Mapping[str, object]) -> tuple[int, int | None]:
+    """Extract a rough input-token count and any explicit max_output_tokens
+    from a ``response.create`` frame, handling both wire shapes:
+      flat:   {"type": "response.create", "input": ..., "max_output_tokens": ...}
+      nested: {"type": "response.create", "response": {"input": ..., "max_output_tokens": ...}}
+    """
+    nested: Final = msg_obj.get("response")
+    params: Final[Mapping[str, object]] = (
+        nested
+        if _is_json_object(nested) and nested
+        else MappingProxyType(  # mutable-ok: immediately frozen filtered frame
+            {k: v for k, v in msg_obj.items() if k != "type"}
+        )
+    )
+    text_parts: Final[list[str]] = []  # mutable-ok: local accumulator built in one pass, not shared
+    pending: Final[list[object]] = [  # mutable-ok: explicit worklist avoids recursion
+        params.get("input"),
+        params.get("instructions"),
+    ]
+    while pending:
+        value = pending.pop()
+        if isinstance(value, str):
+            text_parts.append(value)
+        elif _is_json_array(value):
+            for item in value:
+                if isinstance(item, str):
+                    text_parts.append(item)
+                elif _is_json_object(item):
+                    pending.append(item.get("content"))
+                    pending.append(item.get("text"))
+    total_chars: Final = sum(len(part) for part in text_parts)
+    estimated_input_tokens: Final = max(1, total_chars // _FRAME_CHARS_PER_TOKEN_ESTIMATE) if total_chars else 0
+
+    max_output_tokens: Final = params.get("max_output_tokens")
+    return estimated_input_tokens, max_output_tokens if isinstance(max_output_tokens, int) else None
+
+
+async def _enforce_frame_project_quota(
+    quota_callbacks: Sequence[ProjectQuotaCallback],
+    user_api_key_dict: UserAPIKeyAuth | None,
+    model: str | None,
+    raw_message: str,
+) -> None:
+    """Charge one response.create frame's estimated tokens against every
+    registered project ITPM/OTPM quota callback, in isolation from PII
+    masking / logging so a malformed frame still reaches those callbacks."""
+    if not quota_callbacks:
+        return
+    try:
+        msg_obj = json.loads(raw_message)
+    except (json.JSONDecodeError, TypeError):
+        return
+    if not _is_json_object(msg_obj) or msg_obj.get("type") != "response.create":
+        return
+    estimated_input_tokens, explicit_max_output_tokens = _extract_frame_quota_estimate_inputs(msg_obj)
+    estimated_output_tokens: Final = (
+        explicit_max_output_tokens if explicit_max_output_tokens is not None else _FRAME_NO_MAX_OUTPUT_TOKENS_FLOOR
+    )
+    for callback in quota_callbacks:
+        await callback.enforce_project_io_token_quota_for_frame(
+            user_api_key_dict=user_api_key_dict,
+            requested_model=model,
+            estimated_input_tokens=estimated_input_tokens,
+            estimated_output_tokens=estimated_output_tokens,
+        )
+
+
 RESPONSES_WS_LOGGED_EVENT_TYPES: Final = [
     "response.created",
     "response.completed",
@@ -1360,6 +1448,7 @@ class ResponsesWebSocketStreaming:
         first_message: str | None = None,
         guardrail_callbacks: list[Any] | None = None,
         output_guardrail_callbacks: list[PresidioGuardrailCallback] | None = None,
+        quota_callbacks: Sequence[ProjectQuotaCallback] | None = None,
         authorized_model: str | None = None,
     ):
         self.websocket = websocket
@@ -1372,6 +1461,7 @@ class ResponsesWebSocketStreaming:
         self.first_message = first_message
         self.guardrail_callbacks: list[Any] = guardrail_callbacks or []
         self.output_guardrail_callbacks: list[PresidioGuardrailCallback] = output_guardrail_callbacks or []
+        self.quota_callbacks: tuple[ProjectQuotaCallback, ...] = tuple(quota_callbacks) if quota_callbacks else ()
         # Model name authorized at connection time; enforced on every
         # response.create frame to prevent deployment-substitution attacks.
         self.authorized_model: str | None = authorized_model
@@ -1781,10 +1871,39 @@ class ResponsesWebSocketStreaming:
 
         return json.dumps(evt_obj) if modified else response_str
 
+    async def _enforce_or_reject_frame(self, message: str) -> bool:
+        """Run the per-frame project quota check.
+
+        On rejection, sends an ``error`` event to the client and reports that
+        the frame must be dropped instead of forwarded, so the connection
+        stays open for the client to retry once the window resets.
+        """
+        try:
+            await _enforce_frame_project_quota(
+                self.quota_callbacks, self.user_api_key_dict, self.authorized_model, message
+            )
+        except RateLimitError as e:
+            try:
+                await self.websocket.send_text(
+                    json.dumps(  # mutable-ok: WebSocket wire payload requires JSON objects
+                        {  # mutable-ok: WebSocket wire payload requires JSON objects
+                            "type": "error",
+                            "error": {  # mutable-ok: nested WebSocket error object
+                                "type": "rate_limit_exceeded",
+                                "message": str(e),
+                            },
+                        }
+                    )
+                )
+            except Exception:  # noqa: BLE001, S110  # client may already be gone
+                pass
+            return False
+        return True
+
     async def client_to_backend(self) -> None:
         """Forward response.create events from client to backend."""
         try:
-            if self.first_message is not None:
+            if self.first_message is not None and await self._enforce_or_reject_frame(self.first_message):
                 masked_first: Final = await self._mask_response_create(self.first_message)
                 self._store_input(masked_first)
                 self._store_event(masked_first)
@@ -1792,6 +1911,8 @@ class ResponsesWebSocketStreaming:
 
             while True:
                 message = await self.websocket.receive_text()
+                if not await self._enforce_or_reject_frame(message):
+                    continue
                 masked = await self._mask_response_create(message)
                 self._store_input(masked)
                 self._store_event(masked)
@@ -1871,6 +1992,7 @@ class ManagedResponsesWebSocketHandler:
         timeout: float | None = None,
         custom_llm_provider: str | None = None,
         first_message: str | None = None,
+        quota_callbacks: Sequence[ProjectQuotaCallback] | None = None,
         **kwargs: object,
     ) -> None:
         self.websocket = websocket
@@ -1887,6 +2009,7 @@ class ManagedResponsesWebSocketHandler:
         self.custom_llm_provider = custom_llm_provider
         self._connection_provider = self._resolve_provider(model) or custom_llm_provider
         self.first_message = first_message
+        self.quota_callbacks: tuple[ProjectQuotaCallback, ...] = tuple(quota_callbacks) if quota_callbacks else ()
         # Carry through safe pass-through kwargs (e.g. extra_headers)
         self.extra_kwargs: dict[str, object] = {k: v for k, v in kwargs.items() if k not in _MANAGED_WS_SKIP_KWARGS}
         # In-memory session history: response_id → full accumulated message list.
@@ -2292,6 +2415,14 @@ class ManagedResponsesWebSocketHandler:
                 verbose_logger.debug("ManagedResponsesWS: error sending warmup ack: %s", exc)
             return
 
+        try:
+            await _enforce_frame_project_quota(
+                self.quota_callbacks, self.user_api_key_dict, self.model_group or self.model, raw_message
+            )
+        except RateLimitError as e:
+            await self._send_error(str(e), error_type="rate_limit_exceeded")
+            return
+
         call_kwargs: Final = self._build_base_call_kwargs(msg_obj)
         call_kwargs["stream"] = True
 

litellm/router.py

@@ -8573,11 +8573,9 @@ class Router:
         Returns:
         - The added/updated deployment
         """
+        _deployment_model_id: Final = deployment.model_info.id or ""
+        _deployment_on_router: Final[Deployment | None] = self.get_deployment(model_id=_deployment_model_id)
         try:
-            # check if deployment already exists
-            _deployment_model_id: Final = deployment.model_info.id or ""
-
-            _deployment_on_router: Final[Deployment | None] = self.get_deployment(model_id=_deployment_model_id)
             if _deployment_on_router is not None:
                 # deployment with this model_id exists on the router
                 if (
@@ -8628,10 +8626,31 @@ class Router:
                     deployment.model_info.id,
                     e,
                 )
+                self._restore_deployment_after_failed_upsert(
+                    previous_deployment=_deployment_on_router, model_id=_deployment_model_id
+                )
                 return None
             else:
                 raise e
 
+    def _restore_deployment_after_failed_upsert(self, previous_deployment: Deployment | None, model_id: str) -> None:
+        if previous_deployment is None or self.has_model_id(model_id):
+            return
+        try:
+            self.add_deployment(deployment=previous_deployment)
+            verbose_router_logger.info(
+                "Restored deployment %s (id=%s); it keeps serving its previous configuration.",
+                previous_deployment.model_name,
+                model_id,
+            )
+        except Exception as restore_error:  # noqa: BLE001  # best-effort restore: a second failure must not abort the reload
+            verbose_router_logger.warning(
+                "Could not restore previously served deployment %s (id=%s) after the failed upsert: %s",
+                previous_deployment.model_name,
+                model_id,
+                restore_error,
+            )
+
     @staticmethod
     def _backend_cost_map_keys(model: str, custom_llm_provider: str | None) -> tuple[str, ...]:
         """The ``litellm.model_cost`` keys a deployment's shared backend info is registered under."""
@@ -11463,8 +11482,10 @@ class Router:
         deployment the strategy was registered from via its (model_name, tags)
         pair.
 
-        With tag filtering enabled, strategies that all carry real tags matching
-        none of the request's do not capture it when the name also has plain
+        With tag filtering enabled, router-wide or by the request's
+        enable_tag_filtering (which the proxy sets from key/team
+        router_settings), strategies that all carry real tags matching none of
+        the request's do not capture it when the name also has plain
         deployments: returning None hands the request to ordinary tag-aware
         deployment selection.
         """
@@ -11487,8 +11508,9 @@ class Router:
         for tagged in candidates:
             if "default" in tagged.tags:
                 return tagged
+        request_scoped_filtering: Final = request_kwargs.get("enable_tag_filtering") is True
         if (
-            self.enable_tag_filtering
+            (self.enable_tag_filtering or request_scoped_filtering)
             and all(tagged.tags for tagged in candidates)
             and self._model_name_has_plain_deployments(model)
         ):

litellm/router_utils/fallback_event_handlers.py

@@ -253,6 +253,7 @@ def get_fallback_model_group(fallbacks: list[Any], model_group: str) -> tuple[li
 
 
 PROVIDER_SCOPED_RESOURCE_KEYS: Final = ("input_file_id", "training_file")
+PROVIDER_SCOPED_CREATION_FUNCTION_NAMES: Final = frozenset({"_acreate_file"})
 
 
 def _get_fallback_target_model_group(fallback_entry: str | Mapping[str, object]) -> str | None:
@@ -274,6 +275,18 @@ def references_provider_scoped_resource(kwargs: Mapping[str, object]) -> bool:
     return any(kwargs.get(key) for key in PROVIDER_SCOPED_RESOURCE_KEYS)
 
 
+def creates_provider_scoped_resource(kwargs: Mapping[str, object]) -> bool:
+    """
+    True when the request creates a resource that will live under one provider's credentials.
+
+    A file uploaded for batches or fine-tuning is stored in the account of the deployment
+    that handled it, and its id is only usable against the model group the caller named.
+    Letting the upload fall back to a different model group silently stores the file with
+    the wrong provider, and every later use of the returned id fails.
+    """
+    return getattr(kwargs.get("original_function"), "__name__", None) in PROVIDER_SCOPED_CREATION_FUNCTION_NAMES
+
+
 async def run_async_fallback(
     *args: tuple[Any],
     litellm_router: LitellmRouter,
@@ -322,7 +335,9 @@ async def run_async_fallback(
     metadata_variable_name: Final = _get_router_metadata_variable_name(
         function_name=getattr(kwargs.get("original_function"), "__name__", None)
     )
-    same_model_group_only: Final = references_provider_scoped_resource(kwargs)
+    same_model_group_only: Final = references_provider_scoped_resource(kwargs) or creates_provider_scoped_resource(
+        kwargs
+    )
     # Read out of kwargs and narrowed here rather than declared as a parameter: every caller
     # reaches this function by spreading a loosely-typed kwargs dict, so a declared parameter
     # would carry an annotation that no call site can actually be checked against.

litellm/types/mcp_server/mcp_server_manager.py

@@ -224,6 +224,14 @@ class MCPServer(BaseModel):
         JWT) but forwards the caller's separate upstream ``Authorization`` unchanged, minting nothing."""
         return self.auth_type == MCPAuth.oauth_delegate
 
+    @property
+    def is_client_forwarded_token(self) -> bool:
+        """True for the two modes whose upstream credential is the caller's own bearer, forwarded
+        unchanged: the gateway mints nothing for them and holds no OAuth client identity, so a
+        discovered ``authorization_url`` / ``token_url`` enriches only the gateway's own OAuth front
+        door and is never a precondition for opening a session."""
+        return self.is_true_passthrough or self.is_oauth_delegate
+
     @property
     def is_dcr_bridge(self) -> bool:
         """True when this client-forwarded-token server serves the gateway-hosted DCR front door
@@ -231,7 +239,7 @@ class MCPServer(BaseModel):
         authorize, and token relays) instead of relaying the upstream's own OAuth discovery
         verbatim. ``dcr_bridge`` is rejected on every other auth type at create, update, and
         config load, so the mode gate here only defends rows edited outside those paths."""
-        return bool(self.dcr_bridge) and (self.is_true_passthrough or self.is_oauth_delegate)
+        return bool(self.dcr_bridge) and self.is_client_forwarded_token
 
     @property
     def requires_per_user_auth(self) -> bool:
@@ -248,7 +256,7 @@ class MCPServer(BaseModel):
         if self.needs_user_oauth_token:
             return True
 
-        if self.is_true_passthrough or self.is_oauth_delegate:
+        if self.is_client_forwarded_token:
             return True
 
         # PAT passthrough: auth_type is none but extra_headers includes auth headers

litellm/types/utils.py

@@ -1631,8 +1631,20 @@ class PromptTokensDetailsWrapper(
 
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
+        extra_fields: Final = self.model_extra
+        nested_cache_creation_input_tokens: Final = (
+            extra_fields.get("cache_creation_input_tokens") if extra_fields is not None else None
+        )
         self.cache_write_tokens = (
-            self.cache_write_tokens if self.cache_write_tokens is not None else self.cache_creation_tokens
+            self.cache_write_tokens
+            if self.cache_write_tokens is not None
+            else (
+                self.cache_creation_tokens
+                if self.cache_creation_tokens is not None
+                else (
+                    nested_cache_creation_input_tokens if isinstance(nested_cache_creation_input_tokens, int) else None
+                )
+            )
         )
         if self.character_count is None:
             del self.character_count

litellm/utils.py

@@ -65,6 +65,7 @@ from litellm.constants import (
     DEFAULT_EMBEDDING_PARAM_VALUES,
     DEFAULT_MAX_LRU_CACHE_SIZE,
     DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT,
+    DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
     DEFAULT_TRIM_RATIO,
     FUNCTION_DEFINITION_TOKEN_COUNT,
     INITIAL_RETRY_DELAY,
@@ -7638,12 +7639,20 @@ def validate_and_fix_openai_tools(tools: list | None) -> list[dict] | None:
 
 
 def validate_and_fix_thinking_param(
-    thinking: AnthropicThinkingParam | None,
+    thinking: AnthropicThinkingParam | bool | None,
 ) -> AnthropicThinkingParam | None:
     """
-    Normalizes camelCase keys in the thinking param to snake_case.
+    Coerces bool thinking values (True becomes enabled with the default medium budget, False becomes None)
+    and normalizes camelCase keys in the thinking param to snake_case.
     Handles clients that send budgetTokens instead of budget_tokens.
     """
+    if thinking is True:
+        return cast(
+            "AnthropicThinkingParam",
+            {"type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET},
+        )
+    if thinking is False:
+        return None
     if thinking is None or not isinstance(thinking, dict):
         return thinking
     normalized: Final = dict(thinking)

model_prices_and_context_window.json

@@ -14456,6 +14456,25 @@
         "supports_tool_choice": true,
         "supports_output_config": true
     },
+    "databricks/databricks-claude-opus-4-6": {
+        "input_cost_per_token": 5.00003e-06,
+        "input_dbu_cost_per_token": 7.1429e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 2.5000010000000002e-05,
+        "output_dbu_cost_per_token": 0.000357143,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+        "supports_assistant_prefill": true,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },
     "databricks/databricks-claude-sonnet-4": {
         "input_cost_per_token": 2.9999900000000002e-06,
         "input_dbu_cost_per_token": 4.2857e-05,
@@ -14513,6 +14532,25 @@
         "supports_reasoning": true,
         "supports_tool_choice": true
     },
+    "databricks/databricks-claude-sonnet-4-6": {
+        "input_cost_per_token": 2.9999900000000002e-06,
+        "input_dbu_cost_per_token": 4.2857e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 64000,
+        "max_tokens": 64000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.5000020000000002e-05,
+        "output_dbu_cost_per_token": 0.000214286,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+        "supports_assistant_prefill": true,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },
     "databricks/databricks-gemini-2-5-flash": {
         "input_cost_per_token": 3.0001999999999996e-07,
         "input_dbu_cost_per_token": 4.285999999999999e-06,
@@ -14547,6 +14585,74 @@
         "supports_function_calling": true,
         "supports_tool_choice": true
     },
+    "databricks/databricks-gemini-3-1-flash-lite": {
+        "input_cost_per_token": 3.1248e-07,
+        "input_dbu_cost_per_token": 4.464e-06,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.87502e-06,
+        "output_dbu_cost_per_token": 2.6786e-05,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+        "supports_function_calling": true,
+        "supports_tool_choice": true
+    },
+    "databricks/databricks-gemini-3-1-pro": {
+        "input_cost_per_token": 2.49998e-06,
+        "input_dbu_cost_per_token": 3.5714e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.5000020000000002e-05,
+        "output_dbu_cost_per_token": 0.000214286,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+        "supports_function_calling": true,
+        "supports_tool_choice": true
+    },
+    "databricks/databricks-gemini-3-flash": {
+        "input_cost_per_token": 6.2503e-07,
+        "input_dbu_cost_per_token": 8.929e-06,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 3.74997e-06,
+        "output_dbu_cost_per_token": 5.3571e-05,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+        "supports_function_calling": true,
+        "supports_tool_choice": true
+    },
+    "databricks/databricks-gemini-3-pro": {
+        "input_cost_per_token": 2.49998e-06,
+        "input_dbu_cost_per_token": 3.5714e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.5000020000000002e-05,
+        "output_dbu_cost_per_token": 0.000214286,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+        "supports_function_calling": true,
+        "supports_tool_choice": true
+    },
     "databricks/databricks-gemma-3-12b": {
         "input_cost_per_token": 1.5000999999999998e-07,
         "input_dbu_cost_per_token": 2.1429999999999996e-06,
@@ -14592,6 +14698,126 @@
         "output_dbu_cost_per_token": 0.000142857,
         "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
     },
+    "databricks/databricks-gpt-5-1-codex-max": {
+        "input_cost_per_token": 1.24999e-06,
+        "input_dbu_cost_per_token": 1.7857e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 9.999990000000002e-06,
+        "output_dbu_cost_per_token": 0.000142857,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
+    "databricks/databricks-gpt-5-1-codex-mini": {
+        "input_cost_per_token": 2.4997e-07,
+        "input_dbu_cost_per_token": 3.571e-06,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.99997e-06,
+        "output_dbu_cost_per_token": 2.8571e-05,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
+    "databricks/databricks-gpt-5-2": {
+        "input_cost_per_token": 1.75e-06,
+        "input_dbu_cost_per_token": 2.5e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.4e-05,
+        "output_dbu_cost_per_token": 0.0002,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
+    "databricks/databricks-gpt-5-2-codex": {
+        "input_cost_per_token": 1.75e-06,
+        "input_dbu_cost_per_token": 2.5e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.4e-05,
+        "output_dbu_cost_per_token": 0.0002,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
+    "databricks/databricks-gpt-5-3-codex": {
+        "input_cost_per_token": 1.75e-06,
+        "input_dbu_cost_per_token": 2.5e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.4e-05,
+        "output_dbu_cost_per_token": 0.0002,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
+    "databricks/databricks-gpt-5-4": {
+        "input_cost_per_token": 2.49998e-06,
+        "input_dbu_cost_per_token": 3.5714e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.5000020000000002e-05,
+        "output_dbu_cost_per_token": 0.000214286,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
+    "databricks/databricks-gpt-5-4-mini": {
+        "input_cost_per_token": 7.4998e-07,
+        "input_dbu_cost_per_token": 1.0714e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 4.50002e-06,
+        "output_dbu_cost_per_token": 6.4286e-05,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
+    "databricks/databricks-gpt-5-4-nano": {
+        "input_cost_per_token": 1.9999e-07,
+        "input_dbu_cost_per_token": 2.857e-06,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.24999e-06,
+        "output_dbu_cost_per_token": 1.7857e-05,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
     "databricks/databricks-gpt-5-mini": {
         "input_cost_per_token": 2.4997000000000006e-07,
         "input_dbu_cost_per_token": 3.571e-06,

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm"
-version = "1.98.0"
+version = "1.99.0"
 description = "Library to easily interface with LLM API providers"
 readme = "README.md"
 requires-python = ">=3.10, <3.15"
@@ -67,8 +67,8 @@ proxy = [
     "azure-identity>=1.25.2,<2.0",
     "azure-storage-blob>=12.28.0,<13.0",
     "mcp>=1.28.1,<2.0",
-    "litellm-proxy-extras==0.4.86",
-    "litellm-enterprise==0.1.56",
+    "litellm-proxy-extras==0.4.87",
+    "litellm-enterprise==0.1.57",
     "RestrictedPython>=8.1,<9.0",
     "rich>=13.9.4,<14.0",
     "InquirerPy>=0.3.4,<1.0",
@@ -306,7 +306,7 @@ members = ["enterprise", "litellm-proxy-extras"]
 profile = "black"
 
 [tool.commitizen]
-version = "1.98.0"
+version = "1.99.0"
 version_files = [
     "pyproject.toml:^version",
 ]

ruff-strict-budget.json

@@ -12,7 +12,7 @@
     "limit": 2017
   },
   "ANN202": {
-    "limit": 855
+    "limit": 854
   },
   "ANN204": {
     "limit": 711

tests/batches_tests/test_batch_rate_limits.py

@@ -27,6 +27,16 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import (
 from litellm.proxy.utils import InternalUsageCache
 
 
+def _build_batch_limiter() -> _PROXY_BatchRateLimiter:
+    internal_usage_cache = InternalUsageCache(dual_cache=DualCache())
+    return _PROXY_BatchRateLimiter(
+        internal_usage_cache=internal_usage_cache,
+        parallel_request_limiter=_PROXY_MaxParallelRequestsHandler_v3(
+            internal_usage_cache=internal_usage_cache
+        ),
+    )
+
+
 def get_expected_batch_file_usage(file_path: str) -> tuple[int, int]:
     """
     Helper function to calculate expected request count and token count from a batch JSONL file.
@@ -69,10 +79,7 @@ async def test_batch_rate_limits():
     """
     litellm._turn_on_debug()
     CUSTOM_LLM_PROVIDER = "openai"
-    BATCH_LIMITER = _PROXY_BatchRateLimiter(
-        internal_usage_cache=None,
-        parallel_request_limiter=None,
-    )
+    BATCH_LIMITER = _build_batch_limiter()
 
     file_name = "openai_batch_completions.jsonl"
     _current_dir = os.path.dirname(os.path.abspath(__file__))
@@ -580,10 +587,7 @@ async def test_batch_rate_limiter_without_user_context(tmp_path):
     CUSTOM_LLM_PROVIDER = "openai"
 
     # Setup
-    BATCH_LIMITER = _PROXY_BatchRateLimiter(
-        internal_usage_cache=None,
-        parallel_request_limiter=None,
-    )
+    BATCH_LIMITER = _build_batch_limiter()
 
     # Create a simple batch file
     batch_content = """{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hello"}]}}"""

tests/e2e/coverage_registry/reliability.yaml

@@ -18,6 +18,16 @@
 - {id: reliability.routing.usage_based.picks_under_tpm, module: reliability, tier: P0, behavior: routing, variant: usage_based, assertions: [picks_under_tpm], exercised_on: [chat_completions, messages], source: "router_strategy/lowest_tpm_rpm_v2.py", rationale: "Routes to lowest-TPM deployment; prevents over-allocation"}
 - {id: reliability.routing.least_busy.picks_lowest_traffic, module: reliability, tier: P1, behavior: routing, variant: least_busy, assertions: [picks_lowest_traffic], exercised_on: [chat_completions, messages], source: "router_strategy/least_busy.py", rationale: "Fewest in-flight requests"}
 - {id: reliability.routing.complexity_llm_classifier.routes_by_llm_tier, module: reliability, tier: P1, behavior: routing, variant: complexity_llm_classifier, assertions: [routes_by_llm_tier], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py", fail_before_fix: proven, rationale: "v2 auto-router LLM complexity classifier runs over the proxy and routes by semantic tier instead of silently crashing on absent litellm_metadata and falling back to heuristic scoring"}
+- {id: reliability.routing.tagged_marker.request_tag_selects_marker, module: reliability, tier: P0, behavior: routing, variant: tagged_marker, assertions: [request_tag_selects_marker], exercised_on: [chat_completions], source: "litellm/router.py:11445", rationale: "Tagged request selects the tagged strategy marker under a shared model_name instead of the plain deployment registered first (GitHub issue #36619)"}
+- {id: reliability.routing.tagged_marker.untagged_request_served_by_plain_deployment, module: reliability, tier: P0, behavior: routing, variant: tagged_marker, assertions: [untagged_request_served_by_plain_deployment], exercised_on: [chat_completions, messages, responses], source: "litellm/router.py:11445", rationale: "Untagged requests to a shared model_name are served by the plain deployment on every call, never captured or errored by the tagged marker (GitHub issue #36620)"}
+- {id: reliability.routing.tagged_marker.header_tag_selects_marker, module: reliability, tier: P0, behavior: routing, variant: tagged_marker, assertions: [header_tag_selects_marker], exercised_on: [messages], source: "litellm/router.py:11445", rationale: "A request tagged only via the x-litellm-tags header selects the tagged marker on Anthropic-native /v1/messages (GitHub issue #36621)"}
+- {id: reliability.routing.tagged_marker.untagged_tier_deployments_still_served, module: reliability, tier: P1, behavior: routing, variant: tagged_marker, assertions: [untagged_tier_deployments_still_served], exercised_on: [chat_completions, messages], source: "litellm/router_strategy/tag_based_routing.py:433", rationale: "Routing tags the marker consumed no longer constrain deployment selection inside the routed tier group, so untagged tier deployments serve the rewrite (GitHub issue #36621)"}
+- {id: reliability.routing.tagged_marker.tag_semantics_stay_strict, module: reliability, tier: P1, behavior: routing, variant: tagged_marker, assertions: [tag_semantics_stay_strict], exercised_on: [chat_completions], source: "litellm/router_strategy/tag_based_routing.py:299", rationale: "Tag consumption must not loosen strict semantics: a tagged call aimed straight at an untagged deployment still gets the 401 tags-configuration denial"}
+- {id: reliability.routing.tagged_marker.responses_input_routes_through_marker, module: reliability, tier: P0, behavior: routing, variant: tagged_marker, assertions: [responses_input_routes_through_marker], exercised_on: [responses], source: "litellm/router.py:11489", rationale: "Tagged /v1/responses (header or litellm_metadata.tags, string or list input) routes through the marker to its tier, extending the GitHub issues #36620/#36621 tag split to the Responses surface"}
+- {id: reliability.routing.tagged_marker.alias_connection_params_stay_with_tier, module: reliability, tier: P0, behavior: routing, variant: tagged_marker, assertions: [alias_connection_params_stay_with_tier], exercised_on: [chat_completions], source: "litellm/router.py:11567", rationale: "An api_key or api_base on the marker alias is never forwarded onto the routed request; the tier deployment calls its provider with its own credential (GitHub PR #36626)"}
+- {id: reliability.routing.semantic_auto_router.responses_input_routed, module: reliability, tier: P0, behavior: routing, variant: semantic_auto_router, assertions: [responses_input_routed], exercised_on: [responses], source: "litellm/router_strategy/auto_router/auto_router.py:131", fail_before_fix: proven, rationale: "/v1/responses input is resolved into messages for the semantic auto-router pre-routing hook instead of failing 400 Unmapped LLM provider auto_router (GitHub PR #37333)"}
+- {id: reliability.routing.strategy_alias.custom_pricing_ignored, module: reliability, tier: P1, behavior: routing, variant: strategy_alias, assertions: [custom_pricing_ignored], exercised_on: [chat_completions], source: "litellm/router.py:11489", rationale: "Custom pricing on a strategy-router alias never prices the routed request; spend logs at the routed tier deployment's own rate (GitHub PR #36691)"}
+- {id: reliability.routing.complexity_heuristic.scores_current_ask_only, module: reliability, tier: P1, behavior: routing, variant: complexity_heuristic, assertions: [scores_current_ask_only], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:942", rationale: "The heuristic complexity classifier scores the caller's current ask only, so a keyword-heavy agent system prompt cannot inflate the tier (GitHub PR #36721)"}
 - {id: reliability.cache.exact.returns_cached, module: reliability, tier: P1, behavior: cache, variant: exact, assertions: [returns_cached], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/caching.py", rationale: "Response cache returns cached on exact match"}
 - {id: reliability.cache.prompt_caching_model_select.returns_cached, module: reliability, tier: P1, behavior: cache, variant: prompt_caching_model_select, assertions: [returns_cached], exercised_on: [chat_completions], source: "router_utils/prompt_caching_cache.py", rationale: "Selects model supporting prompt caching for cacheable prefix"}
 - {id: reliability.circuit_breaker.redis.trips_then_recovers, module: reliability, tier: P0, behavior: circuit_breaker, variant: redis, assertions: [trips_then_recovers], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/redis_cache.py:99", rationale: "Redis breaker CLOSED->OPEN->HALF_OPEN; guards all cache/rate-limit ops"}

tests/e2e/e2e_http.py

@@ -75,6 +75,8 @@ class NetworkError(BaseModel):
 
 class UnauthorizedError(BaseModel):
     kind: Literal["unauthorized"] = "unauthorized"
+    # litellm 401s for key auth, model access, and tag routing alike, so keep the body to tell them apart.
+    body: str = ""
 
 
 class RateLimitedError(BaseModel):
@@ -289,7 +291,7 @@ def _classify[R: BaseModel](
     resp: requests.Response, response_type: type[R]
 ) -> Result[R]:
     if resp.status_code == 401:
-        return UnauthorizedError()
+        return UnauthorizedError(body=resp.text)
     if resp.status_code == 429:
         return RateLimitedError(body=resp.text)
     if not resp.ok:

tests/e2e/models.py

@@ -73,6 +73,7 @@ class KeyGenerateBody(BaseModel):
     allowed_passthrough_routes: list[str] | None = None
     metadata: KeyMetadata | None = None
     object_permission: ObjectPermission | None = None
+    router_settings: "RouterSettingsOverride | None" = None
 
 
 class KeyGenerateResponse(BaseModel):
@@ -234,16 +235,18 @@ class ChatBody(BaseModel):
 
 
 class RouterSettingsOverride(BaseModel):
-    """Per-request `router_settings_override` in a /chat/completions body: the
-    reliability knobs (fallbacks by trigger, retry count) the reliability suite
-    drives per call instead of via static router config. Serialized exclude_none, so
-    an override sets only the strategies a test exercises. Each fallbacks map is
-    model_name -> the ordered fallback model_names to try."""
+    """Router settings a test scopes below the global config: sent per request as
+    `router_settings_override` in a /chat/completions body (the reliability suite's
+    fallback and retry knobs) or stored on a key as `router_settings` at
+    /key/generate (the auto-router suite's tag filtering switch). Serialized
+    exclude_none, so an override sets only the knobs a test exercises. Each
+    fallbacks map is model_name -> the ordered fallback model_names to try."""
 
     fallbacks: list[dict[str, list[str]]] | None = None
     context_window_fallbacks: list[dict[str, list[str]]] | None = None
     content_policy_fallbacks: list[dict[str, list[str]]] | None = None
     num_retries: int | None = None
+    enable_tag_filtering: bool | None = None
 
 
 class ReliabilityChatBody(ChatBody):
@@ -744,6 +747,10 @@ class LiteLLMParamsBody(BaseModel):
     extra_headers: dict[str, str] | None = None
     use_in_pass_through: bool | None = None
     complexity_router_config: dict[str, object] | None = None
+    auto_router_config: str | None = None
+    auto_router_default_model: str | None = None
+    auto_router_embedding_model: str | None = None
+    tags: list[str] | None = None
     mock_response: str | None = None
     timeout: float | None = None
     tpm: int | None = None

tests/e2e/router/test_auto_router_regressions_e2e.py

@@ -0,0 +1,620 @@
+"""Live e2e regression pins for strategy-router (auto-router) routing.
+
+A strategy marker (an ``auto_router/complexity_router`` deployment) and a plain
+deployment can share one ``model_name``, split by tags once
+``enable_tag_filtering`` is on: tagged requests route through the marker to its
+tier models, untagged requests go to the plain deployment. That split, and the
+strategy-router alias behaviors around it, regressed repeatedly; each test here
+pins one fixed behavior:
+
+- GitHub issue #36619: a tagged request selects the tagged marker under a
+  shared name even when a plain deployment was registered first.
+- GitHub issue #36620: untagged requests keep being served by the plain
+  deployment on every call, never captured or 400'd by the tagged marker.
+- GitHub issue #36621: a request tagged via the ``x-litellm-tags`` header
+  routes through the marker even when the tier deployments carry no tags
+  (the marker consumes the routing tags before deployment selection), while a
+  tagged call aimed straight at an untagged deployment stays denied.
+- GitHub issues #36620/#36621 on /v1/responses: the same tag split holds for
+  string and list input, whether the tag arrives in litellm_metadata or the
+  x-litellm-tags header.
+- GitHub PR #37333: /v1/responses input is resolved into messages for a
+  semantic ``auto_router`` deployment's pre-routing hook; such requests used
+  to fail with 400 "Unmapped LLM provider auto_router" because only chat
+  messages fed the route matcher.
+- GitHub PR #36691: custom pricing on the marker alias never prices the routed
+  request; spend logs at the routed tier deployment's own rate.
+- GitHub PR #36721: the heuristic complexity classifier scores the caller's
+  current ask only, so a large agent system prompt cannot inflate the tier.
+- GitHub PR #36626: connection params on the marker alias (``api_key``,
+  ``api_base``) stay with the alias; the routed tier calls its provider with
+  its own credentials.
+
+Every deployment is registered via /model/new (stage has no static config for
+these) and ``enable_tag_filtering`` is enabled through key-level
+``router_settings`` on the keys the tag tests mint, so the switch rides only
+this module's own requests and the rest of the suite is never filtered.
+The served deployment is always read back from the spend log's ``model``,
+which stores either the registered alias or the provider-prefixed form.
+"""
+
+import json
+import os
+from collections.abc import Iterator
+from dataclasses import dataclass
+from typing import Final
+
+import pytest
+from pydantic import BaseModel, ConfigDict, Field
+
+from e2e_config import unique_marker
+from e2e_http import AnthropicHeaders, AuthHeaders, UnauthorizedError, unwrap
+from lifecycle import ResourceManager
+from models import (
+    AnthropicMessagesBody,
+    AnthropicMessagesResponse,
+    ChatBody,
+    ChatMessage,
+    ChatMetadata,
+    KeyGenerateBody,
+    LiteLLMParamsBody,
+    RouterSettingsOverride,
+    SpendLogRow,
+)
+from proxy_client import ProxyClient
+
+pytestmark = pytest.mark.e2e
+
+PLAIN_MODEL = "anthropic/claude-sonnet-5"
+CHEAP_MODEL = "anthropic/claude-haiku-4-5"
+STRONG_MODEL = "openai/gpt-5.6"
+MAX_TOKENS = 16
+TAG_DENIAL_MESSAGE = "Not allowed to access model due to tags configuration"
+PLAIN_SERVED = frozenset({PLAIN_MODEL, "claude-sonnet-5"})
+CHEAP_SERVED = frozenset({CHEAP_MODEL, "claude-haiku-4-5"})
+EMBEDDING_MODEL = "openai/text-embedding-3-small"
+SEMANTIC_ROUTE_UTTERANCE = "summarize this quarterly revenue report into three bullet points"
+
+KEYWORD_HEAVY_SYSTEM_PROMPT = (
+    "You are the principal architecture assistant for a distributed systems platform. "
+    "Analyze every request step by step: design the algorithm, prove its correctness, "
+    "evaluate time and space complexity, and reason about concurrency, consistency, and "
+    "fault tolerance tradeoffs. When asked, refactor and debug multi-threaded code, "
+    "optimize database query plans, derive mathematical proofs, and explain the theorem "
+    "or lemma behind each optimization. Think through edge cases rigorously before answering. "
+) * 4
+
+
+class TaggedAuthHeaders(AuthHeaders):
+    x_litellm_tags: str | None = Field(default=None, serialization_alias="x-litellm-tags")
+
+
+class TaggedAnthropicHeaders(AnthropicHeaders):
+    x_litellm_tags: str | None = Field(default=None, serialization_alias="x-litellm-tags")
+
+
+class ResponsesTagMetadata(BaseModel):
+    tags: list[str]
+
+
+class ResponsesInputItem(BaseModel):
+    role: str
+    content: str
+
+
+class ResponsesBody(BaseModel):
+    model: str
+    input: str | list[ResponsesInputItem]
+    max_output_tokens: int | None = None
+    litellm_metadata: ResponsesTagMetadata | None = None
+
+
+class ResponsesApiResponse(BaseModel):
+    """Minimal /v1/responses answer shape; routing is proven from spend logs,
+    so only the fields the assertions read are modeled."""
+
+    model_config = ConfigDict(extra="allow")
+    id: str | None = None
+    status: str | None = None
+    model: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class TagSplitDeployments:
+    """Scenario A mirrors the customer-shaped config from GitHub issue #36619:
+    plain deployment registered first, tier deployment and marker both tagged.
+    Scenario B flips both axes for GitHub issue #36621: marker registered first
+    and its tier deployment left untagged, so routing depends neither on
+    registration order nor on tier deployments carrying tags."""
+
+    tag_a: str
+    shared_a: str
+    tier_a: str
+    tag_b: str
+    shared_b: str
+    tier_b: str
+
+
+@dataclass(frozen=True, slots=True)
+class ZeroPricedAlias:
+    alias: str
+    tier: str
+
+
+@dataclass(frozen=True, slots=True)
+class HeuristicSplit:
+    alias: str
+    cheap: str
+    strong: str
+
+
+@dataclass(frozen=True, slots=True)
+class SemanticAutoRouter:
+    marker: str
+    target: str
+    fallback: str
+    embedding: str
+
+
+@dataclass(frozen=True, slots=True)
+class CredentialedAlias:
+    alias: str
+    tier: str
+
+
+def _provider_key(env_var: str) -> str:
+    return os.environ.get(env_var) or f"os.environ/{env_var}"
+
+
+def _uniform_tier_config(tier_model: str) -> dict[str, object]:
+    return {
+        "classifier_type": "heuristic",
+        "tiers": {"SIMPLE": tier_model, "MEDIUM": tier_model, "COMPLEX": tier_model, "REASONING": tier_model},
+    }
+
+
+def _key_for(
+    proxy: ProxyClient, resources: ResourceManager, models: list[str], tag_filtering: bool = False
+) -> str:
+    key: Final = proxy.generate_key(
+        KeyGenerateBody(
+            models=models,
+            user_id="e2e-auto-router-regressions",
+            router_settings=RouterSettingsOverride(enable_tag_filtering=True) if tag_filtering else None,
+        )
+    )
+    resources.defer(lambda: proxy.delete_key(key))
+    return key
+
+
+def _hello_chat_body(model: str, tags: list[str] | None = None) -> ChatBody:
+    return ChatBody(
+        model=model,
+        messages=[ChatMessage(role="user", content=f"say hello {unique_marker()}")],
+        max_tokens=MAX_TOKENS,
+        metadata=ChatMetadata(tags=tags) if tags is not None else None,
+    )
+
+
+def _hello_messages_body(model: str) -> AnthropicMessagesBody:
+    return AnthropicMessagesBody(
+        model=model,
+        messages=[ChatMessage(role="user", content=f"say hello {unique_marker()}")],
+        max_tokens=MAX_TOKENS,
+    )
+
+
+def _assert_served_only_by(rows: list[SpendLogRow], allowed: frozenset[str], context: str) -> None:
+    served: Final = tuple(row.model for row in rows)
+    assert served and all(model in allowed for model in served), (
+        f"{context}: expected every request to be served by one of {sorted(allowed)}, spend logs show {served}"
+    )
+
+
+@pytest.fixture(scope="module")
+def split(proxy: ProxyClient) -> Iterator[TagSplitDeployments]:
+    marker: Final = unique_marker()
+    deployments: Final = TagSplitDeployments(
+        tag_a=f"e2e-split-a-{marker}",
+        shared_a=f"e2e-autoroute-a-{marker}",
+        tier_a=f"e2e-tier-a-{marker}",
+        tag_b=f"e2e-split-b-{marker}",
+        shared_b=f"e2e-autoroute-b-{marker}",
+        tier_b=f"e2e-tier-b-{marker}",
+    )
+    anthropic_key: Final = _provider_key("ANTHROPIC_API_KEY")
+    marker_params_a: Final = LiteLLMParamsBody(
+        model="auto_router/complexity_router",
+        complexity_router_config=_uniform_tier_config(deployments.tier_a),
+        tags=[deployments.tag_a],
+    )
+    marker_params_b: Final = LiteLLMParamsBody(
+        model="auto_router/complexity_router",
+        complexity_router_config=_uniform_tier_config(deployments.tier_b),
+        tags=[deployments.tag_b],
+    )
+    registrations: Final[tuple[tuple[str, LiteLLMParamsBody], ...]] = (
+        (deployments.shared_a, LiteLLMParamsBody(model=PLAIN_MODEL, api_key=anthropic_key)),
+        (deployments.tier_a, LiteLLMParamsBody(model=CHEAP_MODEL, api_key=anthropic_key, tags=[deployments.tag_a])),
+        (deployments.shared_a, marker_params_a),
+        (deployments.shared_b, marker_params_b),
+        (deployments.tier_b, LiteLLMParamsBody(model=CHEAP_MODEL, api_key=anthropic_key)),
+        (deployments.shared_b, LiteLLMParamsBody(model=PLAIN_MODEL, api_key=anthropic_key)),
+    )
+    created: Final = tuple(proxy.create_model(name, params) for name, params in registrations)
+    try:
+        yield deployments
+    finally:
+        for model_id in created:
+            proxy.delete_model(model_id)
+
+
+@pytest.fixture(scope="module")
+def zero_priced_alias(proxy: ProxyClient) -> Iterator[ZeroPricedAlias]:
+    marker: Final = unique_marker()
+    named: Final = ZeroPricedAlias(alias=f"e2e-priced-alias-{marker}", tier=f"e2e-priced-tier-{marker}")
+    alias_params: Final = LiteLLMParamsBody(
+        model="auto_router/complexity_router",
+        complexity_router_config=_uniform_tier_config(named.tier),
+        input_cost_per_token=0.0,
+        output_cost_per_token=0.0,
+    )
+    registrations: Final[tuple[tuple[str, LiteLLMParamsBody], ...]] = (
+        (named.tier, LiteLLMParamsBody(model=CHEAP_MODEL, api_key=_provider_key("ANTHROPIC_API_KEY"))),
+        (named.alias, alias_params),
+    )
+    created: Final = tuple(proxy.create_model(name, params) for name, params in registrations)
+    try:
+        yield named
+    finally:
+        for model_id in created:
+            proxy.delete_model(model_id)
+
+
+@pytest.fixture(scope="module")
+def heuristic_split(proxy: ProxyClient) -> Iterator[HeuristicSplit]:
+    marker: Final = unique_marker()
+    named: Final = HeuristicSplit(
+        alias=f"e2e-heuristic-router-{marker}",
+        cheap=f"e2e-heuristic-cheap-{marker}",
+        strong=f"e2e-heuristic-strong-{marker}",
+    )
+    config: Final[dict[str, object]] = {
+        "classifier_type": "heuristic",
+        "token_thresholds": {"simple": 15, "complex": 400},
+        "tiers": {"SIMPLE": named.cheap, "MEDIUM": named.strong, "COMPLEX": named.strong, "REASONING": named.strong},
+    }
+    registrations: Final[tuple[tuple[str, LiteLLMParamsBody], ...]] = (
+        (named.cheap, LiteLLMParamsBody(model=CHEAP_MODEL, api_key=_provider_key("ANTHROPIC_API_KEY"))),
+        (named.strong, LiteLLMParamsBody(model=STRONG_MODEL, api_key=_provider_key("OPENAI_API_KEY"))),
+        (named.alias, LiteLLMParamsBody(model="auto_router/complexity_router", complexity_router_config=config)),
+    )
+    created: Final = tuple(proxy.create_model(name, params) for name, params in registrations)
+    try:
+        yield named
+    finally:
+        for model_id in created:
+            proxy.delete_model(model_id)
+
+
+@pytest.fixture(scope="module")
+def semantic_auto_router(proxy: ProxyClient) -> Iterator[SemanticAutoRouter]:
+    marker: Final = unique_marker()
+    named: Final = SemanticAutoRouter(
+        marker=f"e2e-semantic-router-{marker}",
+        target=f"e2e-semantic-target-{marker}",
+        fallback=f"e2e-semantic-fallback-{marker}",
+        embedding=f"e2e-semantic-embedding-{marker}",
+    )
+    router_config: Final = json.dumps(
+        {"routes": [{"name": named.target, "utterances": [SEMANTIC_ROUTE_UTTERANCE], "score_threshold": 0.3}]}
+    )
+    marker_params: Final = LiteLLMParamsBody(
+        model=f"auto_router/{named.marker}",
+        auto_router_config=router_config,
+        auto_router_default_model=named.fallback,
+        auto_router_embedding_model=named.embedding,
+    )
+    registrations: Final[tuple[tuple[str, LiteLLMParamsBody], ...]] = (
+        (named.embedding, LiteLLMParamsBody(model=EMBEDDING_MODEL, api_key=_provider_key("OPENAI_API_KEY"))),
+        (named.target, LiteLLMParamsBody(model=CHEAP_MODEL, api_key=_provider_key("ANTHROPIC_API_KEY"))),
+        (named.fallback, LiteLLMParamsBody(model=PLAIN_MODEL, api_key=_provider_key("ANTHROPIC_API_KEY"))),
+        (named.marker, marker_params),
+    )
+    created: Final = tuple(proxy.create_model(name, params) for name, params in registrations)
+    try:
+        yield named
+    finally:
+        for model_id in created:
+            proxy.delete_model(model_id)
+
+
+@pytest.fixture(scope="module")
+def credentialed_alias(proxy: ProxyClient) -> Iterator[CredentialedAlias]:
+    marker: Final = unique_marker()
+    named: Final = CredentialedAlias(alias=f"e2e-cred-alias-{marker}", tier=f"e2e-cred-tier-{marker}")
+    alias_params: Final = LiteLLMParamsBody(
+        model="auto_router/complexity_router",
+        complexity_router_config=_uniform_tier_config(named.tier),
+        api_key=f"sk-alias-never-used-{marker}",
+    )
+    registrations: Final[tuple[tuple[str, LiteLLMParamsBody], ...]] = (
+        (named.tier, LiteLLMParamsBody(model=CHEAP_MODEL, api_key=_provider_key("ANTHROPIC_API_KEY"))),
+        (named.alias, alias_params),
+    )
+    created: Final = tuple(proxy.create_model(name, params) for name, params in registrations)
+    try:
+        yield named
+    finally:
+        for model_id in created:
+            proxy.delete_model(model_id)
+
+
+class TestTagSplitRouting:
+    @pytest.mark.covers("reliability.routing.tagged_marker.request_tag_selects_marker")
+    def test_body_tagged_chat_routes_through_the_marker_to_its_tier(
+        self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments
+    ) -> None:
+        """Pins GitHub issue #36619: with tag filtering on, a chat request whose
+        body metadata tags match the tagged marker under a shared model name is
+        answered by the marker's tier deployment, not by the plain deployment
+        that was registered under the name first."""
+        key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True)
+        chat: Final = unwrap(proxy.chat(key, _hello_chat_body(split.shared_a, tags=[split.tag_a])))
+        assert chat.choices, "tagged chat through the shared name returned no choices"
+        rows: Final = proxy.poll_logs_for_key(key, min_rows=1)
+        _assert_served_only_by(rows, CHEAP_SERVED | {split.tier_a}, "body-tagged chat on the shared name")
+
+    @pytest.mark.covers("reliability.routing.tagged_marker.untagged_request_served_by_plain_deployment")
+    def test_untagged_chat_is_always_served_by_the_plain_deployment(
+        self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments
+    ) -> None:
+        """Pins GitHub issue #36620: untagged chat requests to the shared name
+        succeed on every call and are all served by the plain deployment; the
+        tagged marker never captures them, so no intermittent auto-router
+        errors and no tier hijacking."""
+        key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True)
+        for _ in range(5):
+            chat = unwrap(proxy.chat(key, _hello_chat_body(split.shared_a)))
+            assert chat.choices, "untagged chat through the shared name returned no choices"
+        rows: Final = proxy.poll_logs_for_key(key, min_rows=5)
+        _assert_served_only_by(rows, PLAIN_SERVED | {split.shared_a}, "untagged chat on the shared name")
+
+    @pytest.mark.covers("reliability.routing.tagged_marker.untagged_request_served_by_plain_deployment")
+    def test_untagged_messages_is_served_by_the_plain_deployment(
+        self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments
+    ) -> None:
+        """Pins GitHub issue #36620 on the /v1/messages surface: an untagged
+        Anthropic-native request to the shared name is served by the plain
+        deployment, not captured by the tagged marker."""
+        key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True)
+        answer: Final = unwrap(proxy.messages(key, _hello_messages_body(split.shared_a)))
+        assert answer.content or answer.choices, "untagged /v1/messages returned neither content nor choices"
+        rows: Final = proxy.poll_logs_for_key(key, min_rows=1)
+        _assert_served_only_by(rows, PLAIN_SERVED | {split.shared_a}, "untagged /v1/messages on the shared name")
+
+
+class TestUntaggedTierDeployments:
+    @pytest.mark.covers("reliability.routing.tagged_marker.header_tag_selects_marker")
+    def test_header_tagged_messages_routes_through_the_marker_to_an_untagged_tier(
+        self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments
+    ) -> None:
+        """Pins GitHub issue #36621: a /v1/messages request tagged only via the
+        x-litellm-tags header selects the tagged marker, and the rewrite still
+        lands on the tier deployment even though that deployment carries no
+        tags, because the marker consumed the routing tags."""
+        key: Final = _key_for(proxy, resources, [split.shared_b, split.tier_b], tag_filtering=True)
+        headers: Final = TaggedAnthropicHeaders(authorization=f"Bearer {key}", x_litellm_tags=split.tag_b)
+        answer: Final = unwrap(
+            proxy.transport.post(
+                "/v1/messages",
+                headers=headers,
+                json=_hello_messages_body(split.shared_b),
+                response_type=AnthropicMessagesResponse,
+            )
+        )
+        assert answer.content or answer.choices, "header-tagged /v1/messages returned neither content nor choices"
+        rows: Final = proxy.poll_logs_for_key(key, min_rows=1)
+        _assert_served_only_by(rows, CHEAP_SERVED | {split.tier_b}, "header-tagged /v1/messages on the shared name")
+
+    @pytest.mark.covers("reliability.routing.tagged_marker.untagged_tier_deployments_still_served")
+    def test_body_tagged_chat_reaches_the_untagged_tier_after_marker_rewrite(
+        self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments
+    ) -> None:
+        """Pins the tag-consumption half of GitHub issue #36621: after the
+        tagged marker rewrites the request to its tier model, the consumed
+        routing tags no longer constrain deployment selection, so the untagged
+        tier deployment serves the request instead of a strict-tag denial."""
+        key: Final = _key_for(proxy, resources, [split.shared_b, split.tier_b], tag_filtering=True)
+        chat: Final = unwrap(proxy.chat(key, _hello_chat_body(split.shared_b, tags=[split.tag_b])))
+        assert chat.choices, "body-tagged chat through the marker-first shared name returned no choices"
+        rows: Final = proxy.poll_logs_for_key(key, min_rows=1)
+        _assert_served_only_by(rows, CHEAP_SERVED | {split.tier_b}, "body-tagged chat with untagged tier")
+
+    @pytest.mark.covers("reliability.routing.tagged_marker.tag_semantics_stay_strict")
+    def test_tagged_call_straight_at_an_untagged_deployment_stays_denied(
+        self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments
+    ) -> None:
+        """The tag-consumption fix must not loosen strict tag semantics: a
+        tagged request aimed directly at an untagged deployment (no marker
+        involved) is still rejected with the 401 tags-configuration error."""
+        key: Final = _key_for(proxy, resources, [split.tier_b], tag_filtering=True)
+        result: Final = proxy.chat(key, _hello_chat_body(split.tier_b, tags=[split.tag_b]))
+        assert isinstance(result, UnauthorizedError), (
+            f"expected the tagged direct call to an untagged deployment to be denied with 401, got {result}"
+        )
+        assert TAG_DENIAL_MESSAGE in result.body, (
+            f"expected the denial to come from tag routing, got a 401 reading {result.body[:300]}"
+        )
+
+
+class TestResponsesApiTagRouting:
+    @pytest.mark.covers("reliability.routing.tagged_marker.responses_input_routes_through_marker")
+    def test_header_tagged_responses_with_string_input_routes_to_the_tier(
+        self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments
+    ) -> None:
+        """Pins the /v1/responses surface of the tag split (GitHub issues
+        #36620/#36621): a /v1/responses request with string input, tagged via
+        the x-litellm-tags header, succeeds and routes through the tagged
+        marker to its tier."""
+        key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True)
+        headers: Final = TaggedAuthHeaders(authorization=f"Bearer {key}", x_litellm_tags=split.tag_a)
+        body: Final = ResponsesBody(
+            model=split.shared_a, input=f"say hello {unique_marker()}", max_output_tokens=64
+        )
+        answer: Final = unwrap(
+            proxy.transport.post("/v1/responses", headers=headers, json=body, response_type=ResponsesApiResponse)
+        )
+        assert answer.id, "header-tagged /v1/responses returned no response id"
+        rows: Final = proxy.poll_logs_for_key(key, min_rows=1)
+        _assert_served_only_by(rows, CHEAP_SERVED | {split.tier_a}, "header-tagged /v1/responses string input")
+
+    @pytest.mark.covers("reliability.routing.tagged_marker.responses_input_routes_through_marker")
+    def test_body_tagged_responses_with_list_input_routes_to_the_tier(
+        self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments
+    ) -> None:
+        """Pins the body-tag and list-input combination of the same split:
+        /v1/responses with litellm_metadata.tags and structured input items
+        routes through the tagged marker to its tier."""
+        key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True)
+        body: Final = ResponsesBody(
+            model=split.shared_a,
+            input=[ResponsesInputItem(role="user", content=f"say hello {unique_marker()}")],
+            max_output_tokens=64,
+            litellm_metadata=ResponsesTagMetadata(tags=[split.tag_a]),
+        )
+        answer: Final = unwrap(
+            proxy.transport.post(
+                "/v1/responses",
+                headers=proxy.transport.bearer(key),
+                json=body,
+                response_type=ResponsesApiResponse,
+            )
+        )
+        assert answer.id, "body-tagged /v1/responses returned no response id"
+        rows: Final = proxy.poll_logs_for_key(key, min_rows=1)
+        _assert_served_only_by(rows, CHEAP_SERVED | {split.tier_a}, "body-tagged /v1/responses list input")
+
+    @pytest.mark.covers("reliability.routing.tagged_marker.untagged_request_served_by_plain_deployment")
+    def test_untagged_responses_is_served_by_the_plain_deployment(
+        self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments
+    ) -> None:
+        """Pins the untagged half of the /v1/responses tag split: an untagged
+        request to the shared name is served by the plain deployment, matching
+        the chat and messages surfaces."""
+        key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True)
+        body: Final = ResponsesBody(
+            model=split.shared_a, input=f"say hello {unique_marker()}", max_output_tokens=64
+        )
+        answer: Final = unwrap(
+            proxy.transport.post(
+                "/v1/responses",
+                headers=proxy.transport.bearer(key),
+                json=body,
+                response_type=ResponsesApiResponse,
+            )
+        )
+        assert answer.id, "untagged /v1/responses returned no response id"
+        rows: Final = proxy.poll_logs_for_key(key, min_rows=1)
+        _assert_served_only_by(rows, PLAIN_SERVED | {split.shared_a}, "untagged /v1/responses on the shared name")
+
+
+class TestStrategyAliasPricing:
+    @pytest.mark.covers("reliability.routing.strategy_alias.custom_pricing_ignored")
+    def test_zero_priced_alias_still_logs_spend_at_the_tier_rate(
+        self, proxy: ProxyClient, resources: ResourceManager, zero_priced_alias: ZeroPricedAlias
+    ) -> None:
+        """Pins GitHub PR #36691: custom pricing registered on a strategy-router
+        alias never prices the routed request. The alias here carries explicit
+        zero pricing, so any zero-spend row would prove the alias pricing was
+        applied; the routed tier deployment's real rate must produce spend > 0."""
+        key: Final = _key_for(proxy, resources, [zero_priced_alias.alias, zero_priced_alias.tier])
+        chat: Final = unwrap(proxy.chat(key, _hello_chat_body(zero_priced_alias.alias)))
+        assert chat.choices, "chat through the zero-priced alias returned no choices"
+        rows: Final = proxy.poll_logs_for_key(
+            key, min_rows=1, predicate=lambda logged: all((row.spend or 0.0) > 0.0 for row in logged)
+        )
+        _assert_served_only_by(rows, CHEAP_SERVED | {zero_priced_alias.tier}, "chat through the zero-priced alias")
+        priced: Final = tuple((row.model, row.spend) for row in rows)
+        assert all((row.spend or 0.0) > 0.0 for row in rows), (
+            f"expected spend at the tier deployment's own rate, got zero-spend rows: {priced}"
+        )
+
+
+class TestComplexityHeuristicScope:
+    @pytest.mark.covers("reliability.routing.complexity_heuristic.scores_current_ask_only")
+    def test_trivial_ask_behind_keyword_heavy_system_prompt_stays_on_the_cheap_tier(
+        self, proxy: ProxyClient, resources: ResourceManager, heuristic_split: HeuristicSplit
+    ) -> None:
+        """Pins GitHub PR #36721: the heuristic complexity classifier scores the
+        caller's current ask alone. The trivial ask scores SIMPLE on its own,
+        while the accompanying ~2KB agent system prompt is packed with enough
+        reasoning and complexity keywords that scoring the combined text lands
+        in REASONING; only ask-only scoring keeps this on the cheap tier."""
+        key: Final = _key_for(
+            proxy, resources, [heuristic_split.alias, heuristic_split.cheap, heuristic_split.strong]
+        )
+        body: Final = ChatBody(
+            model=heuristic_split.alias,
+            messages=[
+                ChatMessage(role="system", content=KEYWORD_HEAVY_SYSTEM_PROMPT),
+                ChatMessage(role="user", content=f"hi {unique_marker()}"),
+            ],
+            max_tokens=MAX_TOKENS,
+        )
+        chat: Final = unwrap(proxy.chat(key, body))
+        assert chat.choices, "chat through the heuristic router returned no choices"
+        rows: Final = proxy.poll_logs_for_key(key, min_rows=1)
+        _assert_served_only_by(
+            rows, CHEAP_SERVED | {heuristic_split.cheap}, "trivial ask behind a keyword-heavy system prompt"
+        )
+
+
+class TestSemanticAutoRouterResponses:
+    @pytest.mark.covers("reliability.routing.semantic_auto_router.responses_input_routed")
+    def test_responses_input_reaches_the_semantic_auto_router(
+        self, proxy: ProxyClient, resources: ResourceManager, semantic_auto_router: SemanticAutoRouter
+    ) -> None:
+        """Pins GitHub PR #37333: /v1/responses input is resolved into messages
+        for the semantic auto-router's pre-routing hook, so the marker embeds
+        the input, matches its route, and the target deployment serves the
+        request; before the fix the hook saw no messages and the request
+        failed with 400 "Unmapped LLM provider auto_router"."""
+        key: Final = _key_for(
+            proxy,
+            resources,
+            [semantic_auto_router.marker, semantic_auto_router.target, semantic_auto_router.fallback],
+        )
+        body: Final = ResponsesBody(
+            model=semantic_auto_router.marker, input=SEMANTIC_ROUTE_UTTERANCE, max_output_tokens=64
+        )
+        answer: Final = unwrap(
+            proxy.transport.post(
+                "/v1/responses",
+                headers=proxy.transport.bearer(key),
+                json=body,
+                response_type=ResponsesApiResponse,
+            )
+        )
+        assert answer.id, "/v1/responses through the semantic auto-router returned no response id"
+        rows: Final = proxy.poll_logs_for_key(key, min_rows=1)
+        _assert_served_only_by(
+            rows, CHEAP_SERVED | {semantic_auto_router.target}, "semantic auto-router /v1/responses string input"
+        )
+
+
+class TestAliasParamForwarding:
+    @pytest.mark.covers("reliability.routing.tagged_marker.alias_connection_params_stay_with_tier")
+    def test_alias_api_key_never_overrides_the_tier_credential(
+        self, proxy: ProxyClient, resources: ResourceManager, credentialed_alias: CredentialedAlias
+    ) -> None:
+        """Pins GitHub PR #36626: an api_key set on the marker alias entry is
+        never forwarded onto the routed request, so the tier deployment calls
+        its provider with its own credential. Before the fix the alias's key
+        was copied into the request, overriding the tier's credential, and
+        every routed call failed provider auth."""
+        key: Final = _key_for(proxy, resources, [credentialed_alias.alias, credentialed_alias.tier])
+        chat: Final = unwrap(proxy.chat(key, _hello_chat_body(credentialed_alias.alias)))
+        assert chat.choices, "chat through the credentialed alias returned no choices"
+        rows: Final = proxy.poll_logs_for_key(key, min_rows=1)
+        _assert_served_only_by(rows, CHEAP_SERVED | {credentialed_alias.tier}, "chat through the credentialed alias")

tests/e2e/ui/tests/mcp/mcpTools.spec.ts

@@ -63,7 +63,7 @@ test.describe("MCP Tools", () => {
 
     // The form is generated from the tool's inputSchema, so `repoName` proves the schema
     // round-tripped through the proxy instead of the panel falling back to a generic field.
-    const repoInput = page.locator('input[id="repoName"]');
+    const repoInput = page.getByLabel(/repoName/);
     await expect(repoInput).toBeVisible();
     await repoInput.fill(TOOL_ARG_REPO);
 

tests/e2e/ui/tests/modelsPage/addModel.spec.ts

@@ -338,8 +338,8 @@ test.describe("Add Model", () => {
 
       await page.getByRole("button", { name: "Add Model" }).last().click();
 
-      // Scope to antd's notification container so a stale toast can't satisfy this.
-      await expect(page.locator(".ant-notification").getByText("created successfully").last()).toBeVisible({
+      // Scope to the toast container so a stale toast can't satisfy this.
+      await expect(page.locator("[data-sonner-toast]").getByText("created successfully").last()).toBeVisible({
         timeout: 15_000,
       });
 

tests/e2e/ui/tests/modelsPage/clearCustomPricing.spec.ts

@@ -85,9 +85,9 @@ test.describe("Clear custom pricing on a deployment", () => {
     const inputCost = page.getByPlaceholder("Enter input cost");
     const outputCost = page.getByPlaceholder("Enter output cost");
     // Both cache fields share the same placeholder ("Defaults to Input Cost if blank"),
-    // so disambiguate via the Form.Item id (AntD assigns the `name` prop as input id).
-    const cacheReadCost = page.locator("#cache_read_cost");
-    const cacheWriteCost = page.locator("#cache_write_cost");
+    // so disambiguate via their labels.
+    const cacheReadCost = page.getByLabel(/Cache Read Cost/);
+    const cacheWriteCost = page.getByLabel(/Cache Write Cost/);
     await inputCost.waitFor({ timeout: 15_000 });
     for (const field of [inputCost, outputCost, cacheReadCost, cacheWriteCost]) {
       await field.click({ clickCount: 3 });

tests/e2e/ui/tests/proxy-admin/keys.spec.ts

@@ -36,9 +36,8 @@ test.describe("Proxy Admin - Keys", () => {
     // Wait for the key creation modal
     await expect(page.getByText("Key Ownership")).toBeVisible({ timeout: 10_000 });
 
-    // Fill key name (has data-testid="base-input" in the built UI)
     const keyName = `e2e-admin-key-${Date.now()}`;
-    await page.getByTestId("base-input").fill(keyName);
+    await page.getByLabel(/Key Name/).fill(keyName);
 
     // Select team
     const teamSelect = page.getByTestId("team-dropdown").getByRole("combobox");
@@ -192,7 +191,7 @@ test.describe("Proxy Admin - Keys", () => {
     await expect(page.getByText("Key Ownership")).toBeVisible({ timeout: 10_000 });
 
     const keyName = `e2e-admin-allproxy-${Date.now()}`;
-    await page.getByTestId("base-input").fill(keyName);
+    await page.getByLabel(/Key Name/).fill(keyName);
 
     // No team selection — leave team dropdown empty so the key is owned by the admin user
 
@@ -220,7 +219,7 @@ test.describe("Proxy Admin - Keys", () => {
     await expect(page.getByText("Key Ownership")).toBeVisible({ timeout: 10_000 });
 
     const keyName = `e2e-admin-specific-${Date.now()}`;
-    await page.getByTestId("base-input").fill(keyName);
+    await page.getByLabel(/Key Name/).fill(keyName);
 
     // Open the model multi-select and pick a single specific model. Use
     // getByRole("option", ...) to avoid the strict-mode collision between

tests/e2e/ui/tests/proxy-admin/teams.spec.ts

@@ -79,7 +79,7 @@ test.describe("Proxy Admin - Teams", () => {
     await expect(modal).toBeVisible({ timeout: 5_000 });
 
     // The email field is a Select — type to search, then select from dropdown
-    await modal.locator(".ant-select").first().click();
+    await modal.getByRole("combobox").first().click();
     await page.keyboard.type("invitable@test.local");
 
     // Wait for the option to appear, then select via keyboard (avoids viewport issues)

tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts

@@ -66,7 +66,7 @@ test.describe("Team Admin", () => {
 
     // Use a dedicated invitee user so this doesn't race with the proxy-admin
     // "Invite a user" test that adds invitable@test.local to the same team.
-    await modal.locator(".ant-select").first().click();
+    await modal.getByRole("combobox").first().click();
     await page.keyboard.type("invitable-team@test.local");
 
     const emailOption = page.getByRole("option", { name: "invitable-team@test.local" }).first();
@@ -136,7 +136,7 @@ test.describe("Team Admin", () => {
     await expect(page.getByText("Key Ownership")).toBeVisible({ timeout: 10_000 });
 
     const keyName = `e2e-team-admin-key-${Date.now()}`;
-    await page.getByTestId("base-input").fill(keyName);
+    await page.getByLabel(/Key Name/).fill(keyName);
 
     // Team selector — same locator pattern as the proxy-admin keys test.
     const teamSelect = page.getByTestId("team-dropdown").getByRole("combobox");

tests/image_gen_tests/test_image_generation.py

@@ -269,7 +269,7 @@ class TestAimlImageGeneration(BaseImageGenTest):
 
 class TestGoogleImageGen(BaseImageGenTest):
     def get_base_image_generation_call_args(self) -> dict:
-        return {"model": "gemini/imagen-4.0-generate-001"}
+        return {"model": "gemini/gemini-3.1-flash-image"}
 
 
 @pytest.mark.skip(reason="Runwayml image generation API only tested locally")

tests/llm_translation/test_groq.py

@@ -20,7 +20,7 @@ from litellm.llms.groq.chat.transformation import (
 class TestGroq(BaseLLMChatTest):
     def get_base_completion_call_args(self) -> dict:
         return {
-            "model": "groq/llama-3.3-70b-versatile",
+            "model": "groq/openai/gpt-oss-120b",
         }
 
     def test_tool_call_no_arguments(self, tool_call_no_arguments):

tests/local_testing/test_model_alias_map.py

@@ -15,7 +15,7 @@ from litellm import completion, embedding
 
 litellm.set_verbose = True
 
-model_alias_map = {"good-model": "groq/llama-3.1-8b-instant"}
+model_alias_map = {"good-model": "groq/openai/gpt-oss-120b"}
 
 
 def test_model_alias_map(caplog):
@@ -34,7 +34,7 @@ def test_model_alias_map(caplog):
             if rec.levelname == "ERROR" and rec.name.startswith("LiteLLM"):
                 pytest.fail(f"Unexpected litellm ERROR log: {rec.getMessage()}")
 
-        assert "llama-3.1-8b-instant" in response.model
+        assert "gpt-oss-120b" in response.model
     except litellm.ServiceUnavailableError:
         pass
     except Exception as e:

tests/local_testing/test_router.py

@@ -120,7 +120,7 @@ async def test_router_provider_wildcard_routing():
     print("response 2 = ", response2)
 
     response3 = await router.acompletion(
-        model="groq/llama-3.1-8b-instant",
+        model="groq/openai/gpt-oss-120b",
         messages=[{"role": "user", "content": "hello"}],
     )
 

tests/local_testing/test_router_batch_completion.py

@@ -44,7 +44,7 @@ async def test_batch_completion_multiple_models(mode):
             {
                 "model_name": "groq-llama",
                 "litellm_params": {
-                    "model": "groq/llama-3.1-8b-instant",
+                    "model": "groq/openai/gpt-oss-120b",
                 },
             },
         ]
@@ -143,7 +143,7 @@ async def test_batch_completion_fastest_response_streaming():
             {
                 "model_name": "groq-llama",
                 "litellm_params": {
-                    "model": "groq/llama-3.1-8b-instant",
+                    "model": "groq/openai/gpt-oss-120b",
                 },
             },
         ]
@@ -179,7 +179,7 @@ async def test_batch_completion_multiple_models_multiple_messages():
             {
                 "model_name": "groq-llama",
                 "litellm_params": {
-                    "model": "groq/llama-3.1-8b-instant",
+                    "model": "groq/openai/gpt-oss-120b",
                 },
             },
         ]

tests/local_testing/test_stream_chunk_builder.py

@@ -871,7 +871,7 @@ def load_env():
     }
     LLAMA3_3 = {
         "messages": messages,
-        "model": "groq/llama-3.3-70b-versatile",
+        "model": "groq/openai/gpt-oss-120b",
         "api_base": "https://api.groq.com/openai/v1",
         "temperature": 0.0,
         "tools": tools,

tests/logging_callback_tests/test_logging_redaction_e2e_test.py

@@ -317,110 +317,69 @@ async def test_redaction_responses_api_stream():
 @pytest.mark.asyncio
 async def test_redaction_responses_api_with_reasoning_summary():
     """Test that reasoning summary in ResponsesAPIResponse output is properly redacted"""
+    import litellm
     from litellm.litellm_core_utils.redact_messages import perform_redaction
 
-    # Create a simple mock object with output items that have reasoning summaries
-    class MockResponsesAPIResponse:
-        def __init__(self):
-            self.output = [
-                # Reasoning item with summary
-                type(
-                    "obj",
-                    (object,),
+    response = litellm.ResponsesAPIResponse(
+        id="resp_123",
+        created_at=1234567890,
+        output=[
+            {
+                "type": "reasoning",
+                "id": "rs_123",
+                "summary": [
                     {
-                        "type": "reasoning",
-                        "id": "rs_123",
-                        "summary": [
-                            type(
-                                "obj",
-                                (object,),
-                                {
-                                    "text": "This is a detailed reasoning summary that should be redacted",
-                                    "type": "summary_text",
-                                },
-                            )()
-                        ],
-                    },
-                )(),
-                # Message item with content
-                type(
-                    "obj",
-                    (object,),
+                        "type": "summary_text",
+                        "text": "This is a detailed reasoning summary that should be redacted",
+                    }
+                ],
+            },
+            {
+                "type": "message",
+                "id": "msg_123",
+                "status": "completed",
+                "role": "assistant",
+                "content": [
                     {
-                        "type": "message",
-                        "id": "msg_123",
-                        "content": [
-                            type(
-                                "obj",
-                                (object,),
-                                {
-                                    "text": "This is the actual message content",
-                                    "type": "output_text",
-                                },
-                            )()
-                        ],
-                    },
-                )(),
-            ]
-            self.reasoning = {"effort": "low", "summary": "auto"}
+                        "type": "output_text",
+                        "text": "This is the actual message content",
+                        "annotations": [],
+                    }
+                ],
+            },
+        ],
+        reasoning={"effort": "low", "summary": "auto"},
+    )
 
-    # Mock as ResponsesAPIResponse so perform_redaction recognizes it
-    mock_response = MockResponsesAPIResponse()
-    mock_response.__class__.__name__ = "ResponsesAPIResponse"
+    model_call_details = {
+        "messages": [{"role": "user", "content": "test"}],
+        "prompt": "test prompt",
+        "input": "test input",
+    }
 
-    # Patch isinstance to recognize our mock as ResponsesAPIResponse
-    import litellm
+    redacted_result = perform_redaction(model_call_details, response)
 
-    original_isinstance = isinstance
+    assert isinstance(
+        redacted_result, litellm.ResponsesAPIResponse
+    ), "Redaction should preserve the ResponsesAPIResponse type"
 
-    def patched_isinstance(obj, cls):
-        if (
-            cls == litellm.ResponsesAPIResponse
-            and obj.__class__.__name__ == "ResponsesAPIResponse"
-        ):
-            return True
-        return original_isinstance(obj, cls)
+    reasoning_item = redacted_result.output[0]
+    assert (
+        reasoning_item.summary[0].text == "redacted-by-litellm"
+    ), "Reasoning summary text should be redacted"
 
-    import builtins
+    message_item = redacted_result.output[1]
+    assert (
+        message_item.content[0].text == "redacted-by-litellm"
+    ), "Message content text should be redacted"
 
-    builtins.isinstance = patched_isinstance
+    assert (
+        redacted_result.reasoning is None
+    ), "Top-level reasoning field should be None"
 
-    try:
-        model_call_details = {
-            "messages": [{"role": "user", "content": "test"}],
-            "prompt": "test prompt",
-            "input": "test input",
-        }
-
-        # Perform redaction
-        redacted_result = perform_redaction(model_call_details, mock_response)
-
-        # Verify reasoning summary text is redacted
-        reasoning_item = redacted_result.output[0]
-        assert (
-            reasoning_item.summary[0].text == "redacted-by-litellm"
-        ), "Reasoning summary text should be redacted"
-
-        # Verify message content is also redacted
-        message_item = redacted_result.output[1]
-        assert (
-            message_item.content[0].text == "redacted-by-litellm"
-        ), "Message content text should be redacted"
-
-        # Verify top-level reasoning field is removed
-        assert (
-            redacted_result.reasoning is None
-        ), "Top-level reasoning field should be None"
-
-        # Verify input messages are redacted
-        assert (
-            model_call_details["messages"][0]["content"] == "redacted-by-litellm"
-        ), "Input messages should be redacted"
-
-        print("✓ Reasoning summary redaction test passed")
-    finally:
-        # Restore original isinstance
-        builtins.isinstance = original_isinstance
+    assert (
+        model_call_details["messages"][0]["content"] == "redacted-by-litellm"
+    ), "Input messages should be redacted"
 
 
 @pytest.mark.asyncio

tests/ocr_tests/test_ocr_azure_document_intelligence.py

@@ -62,7 +62,7 @@ class TestAzureDocumentIntelligencePagesParam:
         return AzureDocumentIntelligenceOCRConfig()
 
     def test_get_supported_ocr_params_includes_pages_and_features(self, cfg):
-        assert cfg.get_supported_ocr_params("prebuilt-layout") == ["pages", "features"]
+        assert cfg.get_supported_ocr_params("prebuilt-layout") == ["pages", "features", "req_format"]
 
     def test_map_ocr_params_mistral_zero_based_int_list(self, cfg):
         mapped = cfg.map_ocr_params({"pages": [0, 1, 2]}, {}, "prebuilt-layout")

tests/pass_through_unit_tests/test_pass_through_unit_tests.py

@@ -413,35 +413,54 @@ async def test_pass_through_request_logging_failure_with_stream(
             assert response.body == b'{"mock": "response"}'
 
 
+PROTOCOL_CONSTRAINED_PASS_THROUGH_ROUTES = {
+    "/comprehendmedical": {"POST"},
+    "/comprehendmedical/{operation}": {"POST"},
+}
+
+
 def test_pass_through_routes_support_all_methods():
     """
-    Test that all pass-through routes support GET, POST, PUT, DELETE, PATCH methods
+    A pass-through route fronts a whole provider API, so narrowing its method
+    set turns a request the upstream would have accepted into a 405. The
+    exceptions are providers whose wire protocol admits only one method: Amazon
+    Comprehend Medical speaks AWS JSON 1.1, which is POST-only, so there is no
+    other method to forward.
     """
-    # Import the routers
     from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
         router as llm_router,
     )
 
-    # Expected HTTP methods
     expected_methods = {"GET", "POST", "PUT", "DELETE", "PATCH"}
 
-    # Function to check routes in a router
     def check_router_methods(router):
         for route in router.routes:
             if isinstance(route, APIRoute):
-                # Get path and methods for this route
                 path = route.path
                 methods = set(route.methods)
-                print("supported methods for route", path, "are", methods)
-                # Assert all expected methods are supported
+                allowed = PROTOCOL_CONSTRAINED_PASS_THROUGH_ROUTES.get(path, expected_methods)
                 assert (
-                    methods == expected_methods
-                ), f"Route {path} does not support all methods. Supported: {methods}, Expected: {expected_methods}"
+                    methods == allowed
+                ), f"Route {path} does not support all methods. Supported: {methods}, Expected: {allowed}"
 
-    # Check both routers
     check_router_methods(llm_router)
 
 
+def test_protocol_constrained_pass_through_exemptions_are_not_stale():
+    """
+    The exemption list above weakens the method contract, so it must not
+    outlive the routes it covers: a renamed or deleted route has to fail here
+    rather than sit in the list silently exempting nothing.
+    """
+    from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
+        router as llm_router,
+    )
+
+    registered_paths = {route.path for route in llm_router.routes if isinstance(route, APIRoute)}
+    unmatched = set(PROTOCOL_CONSTRAINED_PASS_THROUGH_ROUTES) - registered_paths
+    assert not unmatched, f"Exempted pass-through routes no longer exist: {sorted(unmatched)}"
+
+
 def test_is_bedrock_agent_runtime_route():
     """
     Test that _is_bedrock_agent_runtime_route correctly identifies bedrock agent runtime endpoints

tests/search_tests/test_tinyfish_search.py

@@ -35,11 +35,16 @@ MOCK_TINYFISH_RESPONSE = {
 
 
 def _make_mock_response(
-    json_data: dict, status_code: int = 200, request_url: str | None = None
+    json_data: dict,
+    status_code: int = 200,
+    request_url: str | None = None,
+    headers: dict | None = None,
 ) -> MagicMock:
     mock = MagicMock()
     mock.status_code = status_code
     mock.json.return_value = json_data
+    # httpx.Headers normalizes keys to lowercase — mirror production behavior.
+    mock.headers = httpx.Headers(headers or {})
     if request_url:
         mock.request = MagicMock()
         mock.request.url = httpx.URL(request_url)
@@ -163,7 +168,7 @@ class TestTinyfishSearch:
 
     @pytest.mark.asyncio
     async def test_fetch_param_round_trip(self):
-        # End-to-end check: caller passes `fetch=...` (JSON-encoded tf-fetch
+        # End-to-end check: caller passes `fetch=...` (JSON-encoded fetch
         # config); param reaches TinyFish on the request side and the nested
         # `fetch` object on each result surfaces back to the SearchResult on the
         # response side. No LiteLLM-side support code is required.
@@ -235,6 +240,58 @@ class TestTinyfishSearch:
         assert result.results[0].title == "Result 0"
         assert result.results[2].title == "Result 2"
 
+    @pytest.mark.asyncio
+    async def test_top_level_extras_surface_end_to_end(self):
+        # Envelope extras (`query`, `total_results`, `page`) must survive the
+        # full asearch dispatch — proves LiteLLM's entry-point plumbing outside
+        # our transformer doesn't accidentally strip them.
+        os.environ["TINYFISH_API_KEY"] = "sk-tinyfish-test"
+
+        mock_response = _make_mock_response(MOCK_TINYFISH_RESPONSE)
+
+        with patch(
+            "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get",
+            new_callable=AsyncMock,
+        ) as mock_get:
+            mock_get.return_value = mock_response
+
+            response = await litellm.asearch(
+                query="web automation tools",
+                search_provider="tinyfish",
+            )
+
+            assert getattr(response, "query", None) == "web automation tools"
+            assert getattr(response, "total_results", None) == 2
+            assert getattr(response, "page", None) == 0
+
+    @pytest.mark.asyncio
+    async def test_response_headers_surface_end_to_end(self):
+        # Response headers must land on `_hidden_params` after the full
+        # asearch dispatch (both raw and sanitized channels).
+        os.environ["TINYFISH_API_KEY"] = "sk-tinyfish-test"
+
+        mock_response = _make_mock_response(
+            MOCK_TINYFISH_RESPONSE,
+            headers={"X-Request-ID": "req-e2e-1"},
+        )
+
+        with patch(
+            "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get",
+            new_callable=AsyncMock,
+        ) as mock_get:
+            mock_get.return_value = mock_response
+
+            response = await litellm.asearch(
+                query="test",
+                search_provider="tinyfish",
+            )
+
+            raw = response._hidden_params["headers"]
+            add = response._hidden_params["additional_headers"]
+            # httpx lowercases; both channels agree on the value.
+            assert raw["x-request-id"] == "req-e2e-1"
+            assert add["llm_provider-x-request-id"] == "req-e2e-1"
+
     @pytest.mark.asyncio
     async def test_empty_results(self):
         os.environ["TINYFISH_API_KEY"] = "sk-tinyfish-test"

tests/test_litellm/batches/test_batch_utils.py

@@ -959,8 +959,14 @@ async def test_handle_completed_vertex_batch_computes_cost_usage_and_models(monk
         litellm_params={"vertex_project": "proj-1", "vertex_location": "us-central1"},
     )
 
+    pricing = litellm.model_cost["vertex_ai/gemini-3.6-flash"]
+    batch_input = pricing["input_cost_per_token_batches"]
+    batch_output = pricing["output_cost_per_token_batches"]
+
+    assert batch_input < pricing["input_cost_per_token"]
+    assert batch_output < pricing["output_cost_per_token"]
     assert result.cost > 0
-    assert result.cost == pytest.approx(30 * 7.5e-07 + 15 * 3.75e-06)
+    assert result.cost == pytest.approx(30 * batch_input + 15 * batch_output)
     assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (30, 15, 45)
     assert result.models == ["gemini-3.6-flash", "gemini-3.6-flash"]
     assert result.successful_requests == 2

tests/test_litellm/conftest.py

@@ -100,6 +100,12 @@ def isolate_host_aws_config(monkeypatch, isolated_aws_credentials_dir):
     monkeypatch.delenv("AWS_DEFAULT_REGION", raising=False)
 
 
+@pytest.fixture(scope="function", autouse=True)
+def isolate_host_proxy_base_url(monkeypatch):
+    """Prevent a host PROXY_BASE_URL from outranking request-derived URLs during unit tests."""
+    monkeypatch.delenv("PROXY_BASE_URL", raising=False)
+
+
 def _run_coroutine_if_needed(result):
     if not asyncio.iscoroutine(result):
         return

tests/test_litellm/integrations/otel/test_db_endpoint.py

@@ -0,0 +1,312 @@
+"""Tests for litellm/integrations/otel/model/db_endpoint.py
+
+Prisma talks to PostgreSQL through a loopback query engine, so a DB span with no
+``server.address`` gets attributed to ``localhost`` by the backend. These cover
+the endpoint derivation that names the real server, for the local engine and for
+remote and read-replica deployments, and pin the rule that no credential is ever
+exported.
+"""
+
+import os
+from unittest.mock import patch
+
+import pytest
+
+from litellm.integrations.otel.model.db_endpoint import (
+    DatabaseEndpoint,
+    db_span_attributes,
+    parse_database_endpoint,
+    postgres_endpoint,
+)
+
+LOCAL_DSN = "postgresql://llmproxy:dbpassword9090@localhost:5432/litellm"
+REMOTE_DSN = "postgresql://llmproxy:s3cr3t@litellm-prod.abc123.us-east-1.rds.amazonaws.com:6432/litellm?schema=reporting&sslmode=require"
+REPLICA_DSN = "postgresql://reader:r3ad0nly@litellm-prod-ro.abc123.us-east-1.rds.amazonaws.com/litellm_replica"
+
+
+def _resolve(service, call_type=None, database_url=None, read_replica_url=None):
+    """Resolve attributes with the two DB env vars set, as the proxy sets them."""
+    env = {k: v for k, v in (("DATABASE_URL", database_url), ("DATABASE_URL_READ_REPLICA", read_replica_url)) if v}
+    with patch.dict(os.environ, env, clear=False):
+        for absent in {"DATABASE_URL", "DATABASE_URL_READ_REPLICA"} - set(env):
+            os.environ.pop(absent, None)
+        return dict(db_span_attributes(service, call_type))
+
+
+def test_local_prisma_engine_endpoint_is_the_postgres_server_not_the_engine():
+    assert parse_database_endpoint(LOCAL_DSN) == DatabaseEndpoint(
+        address="localhost", port=5432, namespace="litellm"
+    )
+
+
+def test_remote_endpoint_keeps_host_port_and_schema_qualified_namespace():
+    assert parse_database_endpoint(REMOTE_DSN) == DatabaseEndpoint(
+        address="litellm-prod.abc123.us-east-1.rds.amazonaws.com",
+        port=6432,
+        namespace="litellm|reporting",
+    )
+
+
+def test_read_replica_dsn_parses_to_the_replica_host_and_database():
+    assert parse_database_endpoint(REPLICA_DSN) == DatabaseEndpoint(
+        address="litellm-prod-ro.abc123.us-east-1.rds.amazonaws.com",
+        port=5432,
+        namespace="litellm_replica",
+    )
+
+
+def test_default_schema_is_not_spelled_out_in_the_namespace():
+    """``?schema=public`` and no schema at all are the same deployment, so they
+    must not split a group-by on db.namespace."""
+    assert parse_database_endpoint("postgresql://u:p@db.internal/litellm?schema=public") == parse_database_endpoint(
+        "postgresql://u:p@db.internal/litellm"
+    )
+
+
+def test_unix_socket_host_parameter_wins_over_the_netloc():
+    """libpq and the Cloud SQL connector both put the real target in ``host=``
+    behind a localhost netloc, which is the attribution this module removes."""
+    assert parse_database_endpoint(
+        "postgresql://u:p@localhost:5432/litellm?host=/cloudsql/proj:us-east1:inst"
+    ) == DatabaseEndpoint(address="/cloudsql/proj:us-east1:inst", port=5432, namespace="litellm")
+
+
+def test_socket_only_dsn_without_a_netloc_host_still_resolves():
+    assert parse_database_endpoint("postgresql:///litellm?host=/var/run/postgresql") == DatabaseEndpoint(
+        address="/var/run/postgresql", port=5432, namespace="litellm"
+    )
+
+
+def test_percent_encoded_database_name_is_decoded():
+    endpoint = parse_database_endpoint("postgresql://u:p@db.internal/litellm%20prod")
+    assert endpoint is not None and endpoint.namespace == "litellm prod"
+
+
+MISPARSED_AUTHORITY_DSNS = (
+    ("postgresql://litellm:/kJ8xQz+9wT@db.internal:5432/litellm", "kJ8xQz+9wT"),
+    ("postgresql://litellm:12345/aBcD@db.internal:5432/litellm", "aBcD"),
+    # '#' sends the tail to the fragment and '?' to the query, so the path is
+    # empty and only the stranded userinfo '@' reveals the mis-split.
+    ("postgresql://litellm:12345#aBcD@db.internal/litellm", "aBcD"),
+    ("postgresql://litellm:12345?aBcD@db.internal/litellm", "aBcD"),
+    # A '?'-stranded tail that happens to parse as parameters, including one
+    # that hijacks the host= parameter into server.address.
+    ("postgresql://litellm:12345?a=aBcD@db.internal/litellm", "aBcD"),
+    ("postgresql://litellm:12345?host=aBcD@db.internal/litellm", "aBcD"),
+    # Both '/' and '?key=value' together: the slash leaves a clean path holding
+    # the password remainder and the query still parses, so only the stranded
+    # at-sign gives it away.
+    ("postgresql://litellm:12345/aBcD?x=1@db.internal/litellm", "aBcD"),
+)
+
+
+@pytest.mark.parametrize(("dsn", "secret"), MISPARSED_AUTHORITY_DSNS)
+def test_unencoded_slash_in_password_never_yields_an_endpoint(dsn, secret):
+    """An unencoded '/' truncates the authority, so urlparse reports the username
+    as the host and the password tail as the database. Postgres drivers reject
+    such a DSN outright, so the only safe reading is no endpoint at all."""
+    assert parse_database_endpoint(dsn) is None
+
+
+@pytest.mark.parametrize(("dsn", "secret"), MISPARSED_AUTHORITY_DSNS)
+def test_unencoded_slash_in_password_never_reaches_a_span(dsn, secret):
+    attrs = _resolve("postgres", "get_data", database_url=dsn)
+    exported = " ".join(str(value) for value in attrs.values())
+    assert secret not in exported
+    assert "db.namespace" not in attrs
+    assert "server.address" not in attrs
+
+
+def test_extra_path_segment_yields_no_endpoint():
+    """A database name cannot hold an unencoded '/', so a second path segment
+    means the authority was mis-split even when no '@' survived into the path."""
+    assert parse_database_endpoint("postgresql://db.internal:5432/litellm/extra") is None
+
+
+@pytest.mark.parametrize("dsn", [d for d, _ in MISPARSED_AUTHORITY_DSNS])
+def test_a_mis_split_authority_never_exports_the_database_username(dsn):
+    """The username lands in ``parsed.hostname`` when the authority truncates, so
+    a span would name the DB user as the server."""
+    attrs = _resolve("postgres", "get_data", database_url=dsn)
+    assert "server.address" not in attrs
+    assert "litellm" not in " ".join(str(v) for v in attrs.values())
+
+
+@pytest.mark.parametrize(
+    "dsn",
+    [
+        "postgresql://db.internal:5432/litellm?application_name=svc@prod",
+        "postgresql://db.internal:5432/litellm?user=admin@company.com",
+    ],
+)
+def test_an_unencoded_at_sign_in_a_query_forfeits_the_endpoint(dsn):
+    """This shape is byte-for-byte indistinguishable from a mis-split password,
+    so it resolves to no endpoint rather than risking a credential fragment.
+    Percent-encoding the at-sign restores the attributes."""
+    assert parse_database_endpoint(dsn) is None
+    assert parse_database_endpoint(dsn.replace("@", "%40")) is not None
+
+
+def test_host_and_port_query_parameters_are_honoured_together():
+    assert parse_database_endpoint("postgresql://ignored/litellm?host=real.internal&port=6543") == DatabaseEndpoint(
+        address="real.internal", port=6543, namespace="litellm"
+    )
+
+
+def test_percent_encoded_password_still_resolves_the_endpoint():
+    """The encoded spelling is the one a driver accepts, so it must keep working."""
+    assert parse_database_endpoint("postgresql://litellm:pa%2Fssw0rd@db.internal:5432/litellm") == DatabaseEndpoint(
+        address="db.internal", port=5432, namespace="litellm"
+    )
+
+
+def test_hostless_socket_dsn_still_names_the_database():
+    """``postgresql:///litellm`` is a valid local-socket DSN that Prisma accepts,
+    so the database is knowable even though no server address is."""
+    assert parse_database_endpoint("postgresql:///litellm") == DatabaseEndpoint(
+        address=None, port=None, namespace="litellm"
+    )
+
+
+def test_hostless_socket_dsn_emits_namespace_without_a_server():
+    attrs = _resolve("postgres", "get_data", database_url="postgresql:///litellm")
+    assert attrs["db.namespace"] == "litellm"
+    assert "server.address" not in attrs
+    assert "server.port" not in attrs
+
+
+def test_dsn_with_neither_host_nor_database_yields_no_endpoint():
+    assert parse_database_endpoint("postgresql://") is None
+
+
+def test_prisma_default_schema_is_left_implicit():
+    endpoint = parse_database_endpoint("postgresql://u:p@db.internal/litellm?schema=public")
+    assert endpoint is not None and endpoint.namespace == "litellm"
+
+
+@pytest.mark.parametrize("spelling", ["PUBLIC", "Public", "reporting"])
+def test_a_non_default_schema_stays_in_the_namespace(spelling):
+    """Prisma quotes the schema name, so ``?schema=PUBLIC`` provisions a second
+    schema alongside ``public`` with its own tables. Case-folding them into one
+    namespace would report two different schemas as the same database."""
+    endpoint = parse_database_endpoint(f"postgresql://u:p@db.internal/litellm?schema={spelling}")
+    assert endpoint is not None and endpoint.namespace == f"litellm|{spelling}"
+
+
+def test_postgres_scheme_alias_is_accepted():
+    assert parse_database_endpoint("postgres://u:p@db.internal/litellm") == DatabaseEndpoint(
+        address="db.internal", port=5432, namespace="litellm"
+    )
+
+
+@pytest.mark.parametrize(
+    "dsn",
+    [
+        None,
+        "",
+        "mysql://u:p@db.internal:3306/litellm",
+        "postgresql://u:p@db.internal:not-a-port/litellm",
+        "not a url at all",
+    ],
+)
+def test_unusable_dsn_degrades_to_no_endpoint(dsn):
+    assert parse_database_endpoint(dsn) is None
+
+
+def test_database_without_name_or_schema_has_no_namespace():
+    assert parse_database_endpoint("postgresql://u:p@db.internal:5432/") == DatabaseEndpoint(
+        address="db.internal", port=5432, namespace=None
+    )
+
+
+def test_postgres_service_span_carries_system_operation_and_endpoint():
+    assert _resolve("postgres", "get_data", database_url=REMOTE_DSN) == {
+        "db.system.name": "postgresql",
+        "db.system": "postgresql",
+        "db.operation.name": "get_data",
+        "server.address": "litellm-prod.abc123.us-east-1.rds.amazonaws.com",
+        "server.port": 6432,
+        "db.namespace": "litellm|reporting",
+    }
+
+
+def test_legacy_db_system_is_dual_emitted_for_datadog():
+    """Datadog's OTLP intake infers the database span type from ``db.system``,
+    not from the semconv-current ``db.system.name``."""
+    assert _resolve("postgres", "get_data", database_url=LOCAL_DSN)["db.system"] == "postgresql"
+    assert _resolve("redis", "set")["db.system"] == "redis"
+
+
+def test_batch_write_service_is_also_attributed_to_postgres():
+    attrs = _resolve("batch_write_to_db", "_PROXY_track_cost_callback", database_url=REMOTE_DSN)
+    assert attrs["db.system.name"] == "postgresql"
+    assert attrs["server.address"] == "litellm-prod.abc123.us-east-1.rds.amazonaws.com"
+
+
+def test_redis_service_never_borrows_the_postgres_endpoint():
+    assert _resolve("redis", "set", database_url=REMOTE_DSN) == {
+        "db.system.name": "redis",
+        "db.system": "redis",
+        "db.operation.name": "set",
+    }
+
+
+def test_non_datastore_service_gets_no_db_attributes():
+    assert _resolve("reset_budget_job", "reset_budget", database_url=REMOTE_DSN) == {}
+
+
+def test_configured_read_replica_suppresses_the_endpoint_rather_than_naming_the_primary():
+    """Reads are routed to the replica per Prisma call, underneath the span, so
+    naming the writer would pin replica latency onto the primary."""
+    attrs = _resolve("postgres", "get_data", database_url=REMOTE_DSN, read_replica_url=REPLICA_DSN)
+    assert attrs == {
+        "db.system.name": "postgresql",
+        "db.system": "postgresql",
+        "db.operation.name": "get_data",
+    }
+
+
+def test_endpoint_attributes_are_omitted_when_database_url_is_unset():
+    assert _resolve("postgres", "get_data") == {
+        "db.system.name": "postgresql",
+        "db.system": "postgresql",
+        "db.operation.name": "get_data",
+    }
+
+
+def test_blank_call_type_does_not_emit_an_empty_operation_attribute():
+    assert "db.operation.name" not in _resolve("postgres", "")
+    assert "db.operation.name" not in _resolve("postgres", None)
+
+
+@pytest.mark.parametrize(
+    ("dsn", "secrets"),
+    [
+        (LOCAL_DSN, ("dbpassword9090", "llmproxy")),
+        (REMOTE_DSN, ("s3cr3t", "llmproxy", "sslmode")),
+        (REPLICA_DSN, ("r3ad0nly", "reader")),
+    ],
+)
+def test_no_credential_reaches_any_exported_attribute(dsn, secrets):
+    attrs = _resolve("postgres", "get_data", database_url=dsn)
+    assert attrs["server.address"]
+    exported = " ".join(str(value) for value in attrs.values())
+    for secret in secrets:
+        assert secret not in exported
+
+
+def test_a_runtime_endpoint_change_is_reflected_on_the_next_span():
+    """The RDS IAM refresh, the reconnect path and the DB-backed
+    environment_variables overlay can all rewrite DATABASE_URL after startup, so
+    a value cached for the process lifetime would report a server the process no
+    longer talks to."""
+    first = _resolve("postgres", "get_data", database_url=LOCAL_DSN)
+    assert first["server.address"] == "localhost"
+    moved = _resolve("postgres", "get_data", database_url=REMOTE_DSN)
+    assert moved["server.address"] == "litellm-prod.abc123.us-east-1.rds.amazonaws.com"
+
+
+def test_a_replica_configured_after_the_first_span_suppresses_the_endpoint():
+    assert _resolve("postgres", "get_data", database_url=REMOTE_DSN)["server.address"]
+    later = _resolve("postgres", "get_data", database_url=REMOTE_DSN, read_replica_url=REPLICA_DSN)
+    assert "server.address" not in later

tests/test_litellm/integrations/otel/test_otel_v2_logger.py

@@ -8,6 +8,8 @@ hooks, proxy SERVER span lifecycle (start + setters), parent-context resolution
 
 import asyncio
 import contextlib
+import os
+from unittest.mock import patch
 from datetime import datetime, timedelta, timezone
 
 import pytest
@@ -1521,6 +1523,36 @@ def test_async_service_success_hook_emits_service_span():
     assert span.status.status_code is StatusCode.UNSET
 
 
+def test_postgres_db_span_names_the_database_server_not_the_prisma_engine():
+    """Prisma reaches Postgres over loopback, so without server.address the
+    backend attributes the wait to localhost."""
+    dsn = "postgresql://llmproxy:dbpassword9090@litellm-prod.abc123.us-east-1.rds.amazonaws.com:6432/litellm?schema=reporting"
+    logger, exporter = _logger()
+    parent = _service_parent(logger)
+    try:
+        with patch.dict(os.environ, {"DATABASE_URL": dsn}, clear=False):
+            os.environ.pop("DATABASE_URL_READ_REPLICA", None)
+            asyncio.run(
+                logger.async_service_success_hook(
+                    payload=_ServicePayload("postgres", "get_data"),
+                    parent_otel_span=parent,
+                )
+            )
+    finally:
+        parent.end()
+    span = {s.name: s for s in exporter.get_finished_spans()}["postgres get_data"]
+    assert span.kind is SpanKind.CLIENT
+    assert span.attributes["db.system.name"] == "postgresql"
+    assert span.attributes["db.operation.name"] == "get_data"
+    assert span.attributes["server.address"] == "litellm-prod.abc123.us-east-1.rds.amazonaws.com"
+    assert span.attributes["server.port"] == 6432
+    assert span.attributes["db.namespace"] == "litellm|reporting"
+    assert span.attributes["db.system"] == "postgresql"
+    exported = " ".join(str(value) for value in span.attributes.values())
+    assert "dbpassword9090" not in exported
+    assert "llmproxy" not in exported
+
+
 def test_async_service_failure_hook_marks_error_status():
     logger, exporter = _logger()
     parent = _service_parent(logger)

tests/test_litellm/integrations/test_opentelemetry.py

@@ -34,6 +34,7 @@ from litellm.integrations.opentelemetry import (
     _normalize_team_metadata_keys,
 )
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
+from litellm.types.services import ServiceLoggerPayload, ServiceTypes
 
 
 class TestOpenTelemetryGuardrails(unittest.TestCase):
@@ -6212,3 +6213,110 @@ class TestDynamicTracerProviderCache(unittest.TestCase):
 
         self.assertTrue(entry.owns_exporter)
         self.assertIsNotNone(entry.provider._atexit_handler)
+class TestOpenTelemetryDatabaseSemconvAttributes(unittest.TestCase):
+    """A Postgres service span must name the PostgreSQL server it reached.
+
+    Without ``db.system`` and ``server.address``, the only host in the trace is
+    the loopback address of Prisma's local query engine, so the backend
+    attributes the wait to ``localhost`` and it cannot be correlated with the
+    database's own metrics.
+    """
+
+    DSN = "postgresql://llmproxy:dbpassword9090@litellm-prod.abc123.us-east-1.rds.amazonaws.com:6432/litellm?schema=reporting"
+    REPLICA_DSN = "postgresql://reader:r3ad0nly@litellm-prod-ro.abc123.us-east-1.rds.amazonaws.com/litellm"
+
+    def _service_span(self, service, call_type, dsn, error=None, replica_dsn=None):
+        exporter = InMemorySpanExporter()
+        provider = TracerProvider()
+        provider.add_span_processor(SimpleSpanProcessor(exporter))
+        otel = OpenTelemetry()
+        otel.tracer = provider.get_tracer(__name__)
+        parent = otel.tracer.start_span("Received Proxy Server Request")
+        payload = ServiceLoggerPayload(
+            is_error=error is not None,
+            error=error,
+            service=service,
+            duration=0.25,
+            call_type=call_type,
+            event_metadata=None,
+        )
+        hook = otel.async_service_failure_hook if error else otel.async_service_success_hook
+        kwargs = {"error": error} if error else {}
+        env = {k: v for k, v in (("DATABASE_URL", dsn), ("DATABASE_URL_READ_REPLICA", replica_dsn)) if v}
+        with patch.dict(os.environ, env, clear=False):
+            for absent in {"DATABASE_URL", "DATABASE_URL_READ_REPLICA"} - set(env):
+                os.environ.pop(absent, None)
+            asyncio.run(
+                hook(
+                    payload=payload,
+                    parent_otel_span=parent,
+                    start_time=datetime.now(),
+                    end_time=datetime.now(),
+                    **kwargs,
+                )
+            )
+        parent.end()
+        return next(s for s in exporter.get_finished_spans() if s.name == service.value)
+
+    def test_postgres_span_names_the_database_server(self):
+        span = self._service_span(ServiceTypes.DB, "get_data", self.DSN)
+        self.assertEqual(span.attributes["db.system.name"], "postgresql")
+        self.assertEqual(span.attributes["db.operation.name"], "get_data")
+        self.assertEqual(
+            span.attributes["server.address"],
+            "litellm-prod.abc123.us-east-1.rds.amazonaws.com",
+        )
+        self.assertEqual(span.attributes["server.port"], 6432)
+        self.assertEqual(span.attributes["db.namespace"], "litellm|reporting")
+
+    def test_datastore_span_is_a_client_span_carrying_the_legacy_db_system(self):
+        """Datadog types a span as a database call from CLIENT kind plus
+        ``db.system``; an INTERNAL span is classified as custom work."""
+        span = self._service_span(ServiceTypes.DB, "get_data", self.DSN)
+        self.assertEqual(span.kind, trace.SpanKind.CLIENT)
+        self.assertEqual(span.attributes["db.system"], "postgresql")
+
+    def test_internal_service_span_stays_internal(self):
+        span = self._service_span(ServiceTypes.RESET_BUDGET_JOB, "reset_budget", self.DSN)
+        self.assertEqual(span.kind, trace.SpanKind.INTERNAL)
+        self.assertNotIn("db.system.name", span.attributes)
+        self.assertNotIn("server.address", span.attributes)
+
+    def test_existing_service_and_call_type_attributes_are_unchanged(self):
+        span = self._service_span(ServiceTypes.DB, "get_data", self.DSN)
+        self.assertEqual(span.attributes["service"], "postgres")
+        self.assertEqual(span.attributes["call_type"], "get_data")
+
+    def test_failed_postgres_span_also_names_the_database_server(self):
+        span = self._service_span(ServiceTypes.DB, "get_data", self.DSN, error="connection refused")
+        self.assertEqual(span.attributes["db.system.name"], "postgresql")
+        self.assertEqual(span.kind, trace.SpanKind.CLIENT)
+        self.assertEqual(
+            span.attributes["server.address"],
+            "litellm-prod.abc123.us-east-1.rds.amazonaws.com",
+        )
+        self.assertEqual(span.attributes["error"], "connection refused")
+
+    def test_no_credential_from_the_dsn_lands_on_the_span(self):
+        span = self._service_span(ServiceTypes.DB, "get_data", self.DSN)
+        exported = " ".join(str(value) for value in span.attributes.values())
+        self.assertIn("litellm-prod.abc123.us-east-1.rds.amazonaws.com", exported)
+        self.assertNotIn("dbpassword9090", exported)
+        self.assertNotIn("llmproxy", exported)
+
+    def test_redis_span_does_not_borrow_the_postgres_endpoint(self):
+        span = self._service_span(ServiceTypes.REDIS, "async_set_cache", self.DSN)
+        self.assertEqual(span.attributes["db.system.name"], "redis")
+        self.assertEqual(span.kind, trace.SpanKind.CLIENT)
+        self.assertNotIn("server.address", span.attributes)
+
+    def test_configured_read_replica_suppresses_the_endpoint(self):
+        span = self._service_span(ServiceTypes.DB, "get_data", self.DSN, replica_dsn=self.REPLICA_DSN)
+        self.assertEqual(span.attributes["db.system.name"], "postgresql")
+        self.assertNotIn("server.address", span.attributes)
+        self.assertNotIn("db.namespace", span.attributes)
+
+    def test_unset_database_url_leaves_the_span_without_endpoint_attributes(self):
+        span = self._service_span(ServiceTypes.DB, "get_data", None)
+        self.assertEqual(span.attributes["db.system.name"], "postgresql")
+        self.assertNotIn("server.address", span.attributes)

tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py

@@ -241,10 +241,32 @@ def test_split_concatenated_json_non_dict_value():
     assert result == [{}]
 
 
-def test_split_concatenated_json_invalid_raises():
-    """Completely invalid JSON raises JSONDecodeError."""
-    with pytest.raises(json.JSONDecodeError):
-        split_concatenated_json_objects("not json at all")
+def test_split_concatenated_json_wholly_invalid_returns_empty():
+    """
+    Wholly unparseable JSON degrades to an empty list instead of raising.
+
+    Regression for https://github.com/BerriAI/litellm/issues/18667: a raise
+    here propagated out of `_convert_to_bedrock_tool_call_invoke` and turned
+    every replayed conversation into a 500.
+    """
+    assert split_concatenated_json_objects("not json at all") == []
+
+
+def test_split_concatenated_json_malformed_object_returns_empty():
+    """
+    A single malformed object (missing comma between keys) degrades to an
+    empty list rather than raising `Expecting ',' delimiter`.
+    """
+    assert split_concatenated_json_objects('{"location": "Boston" "unit": "celsius"}') == []
+
+
+def test_split_concatenated_json_salvages_prefix_before_truncated_tail():
+    """
+    Complete objects parsed before an unparseable/truncated tail are kept;
+    only the bad tail is discarded.
+    """
+    result = split_concatenated_json_objects('{"a": 1}{"b": 2}{"c":')
+    assert result == [{"a": 1}, {"b": 2}]
 
 
 # ---------------------------------------------------------------------------

tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py

@@ -2287,6 +2287,116 @@ def test_bedrock_tool_call_invoke_non_dict_arguments():
     assert result[0]["toolUse"]["input"] == {}
 
 
+def test_bedrock_tool_call_invoke_malformed_json_does_not_raise():
+    """
+    Regression for https://github.com/BerriAI/litellm/issues/18667.
+
+    When the model emits malformed JSON in tool-call arguments (here a
+    missing comma between keys), replaying that history must NOT raise
+    `Unable to convert openai tool calls ... Expecting ',' delimiter`.
+    It degrades to an empty-object input so the conversation can continue.
+    """
+    tool_calls = [
+        {
+            "id": "toolu_abc123",
+            "type": "function",
+            "function": {
+                "name": "get_weather",
+                "arguments": '{"location": "Boston" "unit": "celsius"}',
+            },
+        }
+    ]
+    result = _convert_to_bedrock_tool_call_invoke(tool_calls)
+    assert len(result) == 1
+    assert result[0]["toolUse"]["toolUseId"] == "toolu_abc123"
+    assert result[0]["toolUse"]["name"] == "get_weather"
+    assert result[0]["toolUse"]["input"] == {}
+
+
+def test_bedrock_tool_call_invoke_salvages_valid_prefix_before_truncated_tail():
+    """
+    A valid leading object followed by a truncated tail keeps the valid
+    object rather than dropping everything or raising.
+    """
+    tool_calls = [
+        {
+            "id": "call_partial",
+            "type": "function",
+            "function": {"name": "shell", "arguments": '{"cmd": "ls"}{"cmd":'},
+        }
+    ]
+    result = _convert_to_bedrock_tool_call_invoke(tool_calls)
+    assert len(result) == 1
+    assert result[0]["toolUse"]["input"] == {"cmd": "ls"}
+
+
+def test_bedrock_tool_call_invoke_mixed_turn_survives_one_malformed_call():
+    """
+    Regression for LIT-4574: an assistant turn with several tool calls where only one
+    has malformed/truncated arguments must keep the valid calls intact and degrade just
+    the bad one to empty input, instead of killing the entire turn.
+    """
+    tool_calls = [
+        {
+            "id": "t_good",
+            "type": "function",
+            "function": {
+                "name": "good_tool",
+                "arguments": '{"item_type": "email", "item_id": "AAMkAD=="}',
+            },
+        },
+        {
+            "id": "t_bad",
+            "type": "function",
+            "function": {"name": "bad_tool", "arguments": '{"item_type": "email"'},
+        },
+    ]
+    result = _convert_to_bedrock_tool_call_invoke(tool_calls)
+    tool_uses = [block["toolUse"] for block in result if "toolUse" in block]
+    assert len(tool_uses) == 2
+    by_name = {tool_use["name"]: tool_use for tool_use in tool_uses}
+    assert by_name["good_tool"]["input"] == {"item_type": "email", "item_id": "AAMkAD=="}
+    assert by_name["bad_tool"]["input"] == {}
+
+
+def test_bedrock_tool_call_invoke_truncated_json_arguments():
+    """
+    Truncated tool call arguments (issue #35303) must not raise. A client replaying a
+    partially streamed tool call would otherwise trigger a pre-network exception that the
+    router maps to a retryable APIConnectionError and retries through the fallback graph.
+    """
+    tool_calls = [
+        {
+            "id": "tooluse_MAh2QLVjBRkvi5QJkLQ08V",
+            "type": "function",
+            "function": {
+                "name": "replace_note_content",
+                "arguments": '{"note_id": "999af35c-4061-4ece-8581-7d43fc988ba4", "title": "WG"',
+            },
+        }
+    ]
+    result = _convert_to_bedrock_tool_call_invoke(tool_calls)
+    assert len(result) == 1
+    assert result[0]["toolUse"]["toolUseId"] == "tooluse_MAh2QLVjBRkvi5QJkLQ08V"
+    assert result[0]["toolUse"]["input"] == {}
+
+
+def test_bedrock_tool_call_invoke_unconvertible_raises_non_retryable_bad_request():
+    """
+    Conversion failures are client input errors, so they must surface as a non-retryable
+    BadRequestError instead of a bare Exception that maps to APIConnectionError, and the
+    message must not embed the tool call payload (issue #35303).
+    """
+    tool_calls = [{"id": "call_bad", "type": "function", "function": None}]
+
+    with pytest.raises(litellm.BadRequestError) as exc_info:
+        _convert_to_bedrock_tool_call_invoke(tool_calls)
+
+    assert exc_info.value.status_code == 400
+    assert "call_bad" in str(exc_info.value)
+    assert "function" not in str(exc_info.value).split("Received error=")[0]
+
+
 def test_make_valid_bedrock_tool_name_preserves_hyphens():
     assert make_valid_bedrock_tool_name("my-tool") == "my-tool"
     assert (

tests/test_litellm/litellm_core_utils/test_litellm_logging.py

@@ -4918,3 +4918,51 @@ def test_payload_without_guardrail_cost_is_unchanged(logging_obj):
     assert payload is not None
     assert payload["response_cost"] == pytest.approx(0.0000429)
     assert payload["cost_breakdown"] is None
+
+
+_AWS_SECRET = "wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY"
+_GEMINI_KEY = "AIzaSyC0000000000000000000000000000000"
+
+
+def test_empty_api_base_does_not_dump_call_state(logging_obj):
+    """Direct (non-HTTP) providers pass api_base='', which used to echo model_call_details."""
+    logging_obj.model_call_details["litellm_params"] = {
+        "api_key": "sk-proj-hunter2hunter2hunter2hunter2",
+        "aws_secret_access_key": _AWS_SECRET,
+    }
+
+    curl_command = logging_obj._get_request_curl_command(
+        api_base="",
+        headers={},
+        additional_args={},
+        data={"model": "some-model"},
+    )
+
+    assert "litellm_call_id" not in curl_command
+    assert _AWS_SECRET not in curl_command
+    assert "hunter2" not in curl_command
+
+
+def test_pre_call_redacts_and_masks_raw_request(logging_obj):
+    """log_raw_request_response echoes the request body and api_base back to loggers/UI."""
+    metadata = {"user_api_key_alias": "qa-key"}
+    logging_obj.model_call_details["litellm_params"] = {"metadata": metadata}
+    logging_obj.log_raw_request_response = True
+
+    logging_obj.pre_call(
+        input="hi",
+        api_key="",
+        additional_args={
+            "api_base": f"https://generativelanguage.googleapis.com/v1beta/models/x:generateContent?key={_GEMINI_KEY}",
+            "headers": {},
+            "complete_input_dict": {"aws_secret_access_key": _AWS_SECRET},
+        },
+    )
+
+    raw_request = metadata["raw_request"]
+    assert _AWS_SECRET not in raw_request
+    assert "REDACTED" in raw_request
+
+    raw_api_base = logging_obj.model_call_details["raw_request_typed_dict"]["raw_request_api_base"]
+    assert _GEMINI_KEY not in raw_api_base
+    assert "key=*****" in raw_api_base

tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py

@@ -6043,3 +6043,12 @@ def test_streaming_usage_chunk_is_transformed():
     assert chunk.usage.prompt_tokens == 11
     assert chunk.usage.completion_tokens == 4
     assert chunk.usage.total_tokens == 15
+
+
+def test_update_optional_params_with_thinking_tokens_bool_thinking_does_not_crash():
+    config = AmazonConverseConfig()
+    optional_params = {"thinking": True}
+    config.update_optional_params_with_thinking_tokens(
+        non_default_params={"thinking": True}, optional_params=optional_params
+    )
+    assert "maxTokens" not in optional_params

tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py

@@ -1,7 +1,9 @@
 import asyncio
 import json
+import logging
 import os
 import sys
+import time
 from unittest.mock import AsyncMock, Mock, patch
 
 import httpx
@@ -9,6 +11,7 @@ import pytest
 
 sys.path.insert(0, os.path.abspath("../../../.."))  # Adds the parent directory to the system path
 import litellm
+from litellm._logging import verbose_logger
 from litellm.integrations.code_interpreter_interception.handler import (
     CodeInterpreterInterceptionLogger,
     LITELLM_CODE_EXECUTION_TOOL_NAME,
@@ -2158,3 +2161,68 @@ async def test_vector_store_search_handler_direct_config_async_skips_http():
     pre_call_args = logging_obj.pre_call.call_args.kwargs["additional_args"]
     assert pre_call_args["query"] == ["q1", "q2"]
     assert pre_call_args["vector_store_id"] == "vs_direct"
+
+
+def _direct_vector_store_debug_logging_obj():
+    from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
+
+    logging_obj = LitellmLogging(
+        model="valkey",
+        messages=[{"role": "user", "content": "q"}],
+        stream=False,
+        call_type="vector_store_search",
+        start_time=time.time(),
+        litellm_call_id="vs-debug-call-id",
+        function_id="vs-debug-function-id",
+        log_raw_request_response=True,
+    )
+    logging_obj.update_environment_variables(
+        model="valkey",
+        optional_params={"vector_store_id": "vs_direct", "query": "q"},
+        litellm_params={
+            "litellm_call_id": "vs-debug-call-id",
+            "vector_store_id": "vs_direct",
+            "litellm_request_debug": True,
+            "metadata": {"user_api_key_alias": "vs-test-key"},
+            "valkey_host": "valkey.internal",
+            "valkey_password": "sup3r-s3cret-valkey-pw",
+            "litellm_embedding_config": {"api_key": "sk-embedding-s3cret"},
+        },
+    )
+    return logging_obj
+
+
+@pytest.mark.parametrize("is_async", [False, True])
+def test_direct_vector_store_search_debug_log_omits_stored_credentials(caplog, is_async):
+    """Regression: an empty api_base made pre_call dump the whole model_call_details, so every
+    search shipped the stored valkey_password / embedding api_key into the raw_request metadata."""
+    handler = BaseLLMHTTPHandler()
+    stub_response = {"object": "vector_store.search_results.page", "search_query": "q", "data": []}
+    config = _make_stub_direct_vector_store_config(stub_response)
+    logging_obj = _direct_vector_store_debug_logging_obj()
+
+    with caplog.at_level(logging.DEBUG, logger=verbose_logger.name):
+        result = handler.vector_store_search_handler(
+            vector_store_id="vs_direct",
+            query="q",
+            vector_store_search_optional_params={"max_num_results": 4},
+            vector_store_provider_config=config,
+            custom_llm_provider="valkey",
+            litellm_params=GenericLiteLLMParams(
+                valkey_host="valkey.internal",
+                valkey_password="sup3r-s3cret-valkey-pw",
+            ),
+            logging_obj=logging_obj,
+            _is_async=is_async,
+        )
+        if is_async:
+            result = asyncio.run(result)
+
+    assert result is stub_response
+    raw_request = logging_obj.model_call_details["litellm_params"]["metadata"]["raw_request"]
+    assert "sup3r-s3cret-valkey-pw" not in raw_request
+    assert "sk-embedding-s3cret" not in raw_request
+    assert "valkey://vs_direct" in raw_request
+    logged = "\n".join(record.getMessage() for record in caplog.records)
+    assert "sup3r-s3cret-valkey-pw" not in logged
+    assert "sk-embedding-s3cret" not in logged

tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py

@@ -271,6 +271,47 @@ class TestDashscopeCostCalculator:
 
         assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
 
+    def test_dashscope_nested_cache_creation_input_tokens_bill_at_cache_write_rate(self):
+        """
+        Regression (LIT-5757): DashScope nests cache_creation_input_tokens inside
+        prompt_tokens_details; those tokens must bill at the tier's cache-creation
+        rate instead of being folded into text tokens at the input rate.
+        """
+        self._register_tiered_model(
+            "dashscope/qwen-nested-cache-write-test",
+            [
+                {
+                    "range": [0, 128000],
+                    "input_cost_per_token": 4e-07,
+                    "cache_read_input_token_cost": 1.6e-07,
+                    "cache_creation_input_token_cost": 5e-07,
+                    "output_cost_per_token": 1.6e-06,
+                }
+            ],
+        )
+
+        usage = Usage(
+            prompt_tokens=2059,
+            completion_tokens=201,
+            total_tokens=2260,
+            prompt_tokens_details={
+                "cached_tokens": 0,
+                "text_tokens": 2059,
+                "cache_type": "ephemeral",
+                "cache_creation_input_tokens": 2048,
+                "cache_creation": {"ephemeral_5m_input_tokens": 2048},
+            },
+            completion_tokens_details={"reasoning_tokens": 170},
+        )
+
+        prompt_cost, _ = dashscope_cost_per_token(
+            model="qwen-nested-cache-write-test", usage=usage
+        )
+
+        assert math.isclose(
+            prompt_cost, (2048 * 5e-07) + (11 * 4e-07), rel_tol=1e-10
+        )
+
     def test_dashscope_tiered_cache_creation_falls_back_to_tier_input_rate(self):
         """
         Tiers without a cache_creation_input_token_cost bill cache-creation tokens at

tests/test_litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py

@@ -101,3 +101,8 @@ async def test_async_transform_request_strips_unsupported_tools_from_body():
 
     assert [tool["type"] for tool in body["tools"]] == ["function"]
     assert body["tools"][0]["function"]["name"] == "shell"
+
+
+def test_thinking_mode_active_bool_thinking_returns_false_without_crashing():
+    config = DeepSeekChatConfig()
+    assert config._thinking_mode_active(model="deepseek-reasoner", optional_params={"thinking": True}) is False

tests/test_litellm/llms/tinyfish/test_tinyfish_search.py

@@ -47,7 +47,9 @@ def _make_mock_response(
 
     mock = MagicMock()
     mock.status_code = status_code
-    mock.headers = headers or {}
+    # httpx.Headers normalizes keys to lowercase — mirror production so tests
+    # assert what callers actually see.
+    mock.headers = httpx.Headers(headers or {})
     if json_data is not None:
         mock.json.return_value = json_data
         mock.text = text if text is not None else _json.dumps(json_data)
@@ -222,7 +224,7 @@ class TestTransformSearchRequest:
             assert param not in result["_tinyfish_params"]
 
     def test_arbitrary_param_passed_through(self):
-        # `fetch` is a TinyFish-specific param (JSON-encoded tf-fetch config).
+        # `fetch` is a TinyFish-specific param (JSON-encoded fetch config).
         # The passthrough loop should forward it verbatim without LiteLLM needing
         # to know about it.
         config = TinyfishSearchConfig()
@@ -237,26 +239,49 @@ class TestTransformSearchRequest:
         config = TinyfishSearchConfig()
         result = config.transform_search_request(
             query="test",
-            optional_params={"fetch": {"format": "html", "fetch_path": "fast"}},
-        )
-        assert (
-            result["_tinyfish_params"]["fetch"]
-            == '{"format":"html","fetch_path":"fast"}'
+            optional_params={"fetch": {"format": "html"}},
         )
+        assert result["_tinyfish_params"]["fetch"] == '{"format":"html"}'
 
     def test_bool_param_serialized_as_lowercase(self):
-        # urlencode renders Python bool as capitalized "True"/"False"; ux-labs
-        # rejects those (e.g. include_thumbnail must be literal "true"/"false").
-        # Normalize before passing through.
+        # urlencode renders Python bool as capitalized "True"/"False"; TinyFish
+        # Search's bool params require lowercase "true"/"false" strings on the
+        # wire. Normalize before passing through.
         config = TinyfishSearchConfig()
         true_result = config.transform_search_request(
-            query="test", optional_params={"include_thumbnail": True}
+            query="test", optional_params={"some_bool_param": True}
         )
         false_result = config.transform_search_request(
-            query="test", optional_params={"include_thumbnail": False}
+            query="test", optional_params={"some_bool_param": False}
         )
-        assert true_result["_tinyfish_params"]["include_thumbnail"] == "true"
-        assert false_result["_tinyfish_params"]["include_thumbnail"] == "false"
+        assert true_result["_tinyfish_params"]["some_bool_param"] == "true"
+        assert false_result["_tinyfish_params"]["some_bool_param"] == "false"
+
+    def test_float_param_passes_through(self):
+        # Float values pass the urlencode adapter and land on the wire as
+        # their decimal string form. If TinyFish's server rejects a float
+        # for a param it expects as int, the server's 400 response is
+        # attributed via _wrap_error (`TinyFish Search: ...`) — better than
+        # a client-side pydantic ValidationError with no context.
+        config = TinyfishSearchConfig()
+        result = config.transform_search_request(
+            query="test",
+            optional_params={"some_float_param": 0.5},
+        )
+        assert result["_tinyfish_params"]["some_float_param"] == 0.5
+
+    def test_list_param_auto_json_encoded(self):
+        # TinyFish Search's JSON-array params arrive on the wire as JSON-
+        # encoded strings. Accept the natural Python list form and serialize
+        # so the caller doesn't have to pre-stringify. Params whose wire
+        # format is a plain comma-separated string are the caller's
+        # responsibility to pass as a Python str.
+        config = TinyfishSearchConfig()
+        result = config.transform_search_request(
+            query="test",
+            optional_params={"some_list_param": ["a.example", "b.example"]},
+        )
+        assert result["_tinyfish_params"]["some_list_param"] == '["a.example","b.example"]'
 
     def test_pre_stringified_param_passed_unchanged(self):
         # If the caller already JSON-encoded, don't re-encode.
@@ -422,10 +447,104 @@ class TestTransformSearchResponse:
         assert getattr(first, "position", None) == 1
         assert getattr(first, "site_name", None) == "tinyfish.ai"
 
+    def test_top_level_extras_flow_through(self):
+        # TinyFish returns `query`, `total_results`, `page` at the envelope
+        # level. These must ride through to the caller via SearchResponse's
+        # extra="allow" so pagination logic, echo checks, etc. work.
+        config = TinyfishSearchConfig()
+        mock_response = _make_mock_response(MOCK_TINYFISH_RESPONSE)
+        result = config.transform_search_response(
+            raw_response=mock_response, logging_obj=None
+        )
+        assert getattr(result, "query", None) == "web automation tools"
+        assert getattr(result, "total_results", None) == 2
+        assert getattr(result, "page", None) == 0
+
+    def test_top_level_future_extras_flow_through(self):
+        # Any future TinyFish top-level field must ride through unchanged
+        # (design contract: no LiteLLM code change needed for new fields).
+        config = TinyfishSearchConfig()
+        body = {
+            "results": [
+                {"title": "x", "url": "https://x", "snippet": "x"},
+            ],
+            "query": "test",
+            "example_int_extra": 123,  # hypothetical future field
+            "example_str_extra": "value",  # hypothetical future field
+            "example_id_extra": "abc-def",  # hypothetical future field
+        }
+        result = config.transform_search_response(
+            raw_response=_make_mock_response(body), logging_obj=None
+        )
+        assert getattr(result, "example_int_extra", None) == 123
+        assert getattr(result, "example_str_extra", None) == "value"
+        assert getattr(result, "example_id_extra", None) == "abc-def"
+
+    def test_response_headers_stashed_on_hidden_params(self):
+        # TinyFish Search sets X-Request-ID on every success response. Confirm it
+        # lands on both `_hidden_params["headers"]` (raw) and
+        # `_hidden_params["additional_headers"]` (sanitized/prefixed).
+        # httpx.Headers lowercases every key, so assertions use lowercase.
+        config = TinyfishSearchConfig()
+        mock_response = _make_mock_response(
+            MOCK_TINYFISH_RESPONSE,
+            headers={"X-Request-ID": "req-abc-123", "Content-Type": "application/json"},
+        )
+        result = config.transform_search_response(
+            raw_response=mock_response, logging_obj=None
+        )
+        # Raw copy — httpx has normalized keys to lowercase.
+        assert result._hidden_params["headers"]["x-request-id"] == "req-abc-123"
+        # process_response_headers prefixes non-OpenAI-standard keys with "llm_provider-".
+        assert result._hidden_params["additional_headers"]["llm_provider-x-request-id"] == "req-abc-123"
+
+    def test_response_headers_future_headers_flow_through(self):
+        # "Accept extra": any header TinyFish Search adds later must ride
+        # through without a LiteLLM code change.
+        config = TinyfishSearchConfig()
+        mock_response = _make_mock_response(
+            MOCK_TINYFISH_RESPONSE,
+            headers={
+                "X-Request-ID": "req-1",
+                "X-Example-Header-A": "value-a",  # hypothetical future header
+                "X-Example-Header-B": "value-b",  # hypothetical future header
+            },
+        )
+        result = config.transform_search_response(
+            raw_response=mock_response, logging_obj=None
+        )
+        raw = result._hidden_params["headers"]
+        # httpx lowercases header names on read.
+        assert raw["x-example-header-a"] == "value-a"
+        assert raw["x-example-header-b"] == "value-b"
+
+    def test_response_headers_strips_x_litellm_spoof(self):
+        # A provider setting `x-litellm-*` in its response must not be able to
+        # spoof LiteLLM-internal markers via _hidden_params["additional_headers"].
+        # The raw copy preserves the header (opt-in debug view); the sanitized
+        # copy prefixes it with `llm_provider-` so bare `x-litellm-*` markers
+        # can't be spoofed (values still survive under the prefixed key for
+        # observability).
+        config = TinyfishSearchConfig()
+        mock_response = _make_mock_response(
+            MOCK_TINYFISH_RESPONSE,
+            headers={"x-litellm-attempted-fallbacks": "spoofed", "X-Request-ID": "r1"},
+        )
+        result = config.transform_search_response(
+            raw_response=mock_response, logging_obj=None
+        )
+        # Raw view still has the spoof.
+        assert result._hidden_params["headers"]["x-litellm-attempted-fallbacks"] == "spoofed"
+        # Sanitized view: the spoof survives only under the llm_provider- prefix
+        # (never under the bare x-litellm-* key that LiteLLM downstream trusts).
+        additional = result._hidden_params["additional_headers"]
+        assert "x-litellm-attempted-fallbacks" not in additional
+        assert additional.get("llm_provider-x-litellm-attempted-fallbacks") == "spoofed"
+
     def test_fetch_field_rides_through_to_search_result(self):
-        # Mirrors browser-search's per-result `fetch` nested object (see
-        # api/src/parser.rs SearchResult.fetch). Confirms `fetch=...` requests
-        # surface their content to LiteLLM callers without provider changes.
+        # Mirrors TinyFish Search's per-result `fetch` nested object.
+        # Confirms `fetch=...` requests surface their content to LiteLLM
+        # callers without provider changes.
         config = TinyfishSearchConfig()
         fetched = {
             "results": [
@@ -568,7 +687,7 @@ class TestTransformSearchResponse:
 
 class TestErrorHandling:
     def test_4xx_response_raises_with_attribution_and_unwrapped_message(self):
-        # Reproduces ux-labs' error envelope shape for an INVALID_INPUT response.
+        # Reproduces TinyFish Search's error envelope shape for an INVALID_INPUT response.
         config = TinyfishSearchConfig()
         body = {
             "error": {
@@ -590,7 +709,7 @@ class TestErrorHandling:
 
     def test_429_preserves_status_code_and_headers(self):
         config = TinyfishSearchConfig()
-        body = {"error": {"code": "RATE_LIMIT_EXCEEDED", "message": "60 rpm"}}
+        body = {"error": {"code": "RATE_LIMIT_EXCEEDED", "message": "rate limit exceeded"}}
         mock_response = _make_mock_response(
             body, status_code=429, headers={"Retry-After": "60"}
         )
@@ -600,10 +719,12 @@ class TestErrorHandling:
             )
         assert getattr(exc_info.value, "status_code", None) == 429
         headers = getattr(exc_info.value, "headers", {}) or {}
-        assert headers.get("Retry-After") == "60"
+        # httpx lowercases; the exception carries the same dict shape.
+        assert headers.get("retry-after") == "60"
 
-    def test_5xx_with_non_ux_labs_body_falls_back_to_raw_text(self):
-        # Cloudflare-style JSON or any other envelope: unwrap fails, fall back to raw.
+    def test_5xx_with_non_tinyfish_envelope_shape_falls_back_to_raw_text(self):
+        # A JSON body that doesn't match TinyFish Search's error envelope shape:
+        # unwrap fails, fall back to the raw body text.
         config = TinyfishSearchConfig()
         body = {"errors": [{"code": "10000", "message": "Internal"}]}
         mock_response = _make_mock_response(body, status_code=502)

tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py

@@ -6406,7 +6406,7 @@ async def test_execute_mcp_tool_rest_server_id_injects_requested_server_credenti
     with (
         patch.dict(
             mcp_module.global_mcp_server_manager.tool_name_to_mcp_server_name_mapping,
-            {"echo": collision_server.name},
+            {"echo": collision_server.name, "echo_requested-echo": requested_server.name},
         ),
         patch.object(
             mcp_module.global_mcp_server_manager,
@@ -8430,3 +8430,72 @@ class TestListFiltersHonorThePrefixBoundary:
 
         assert listed == callable_, f"grants={grants!r} listed={listed} callable={callable_}"
         assert listed is expected, f"grants={grants!r} expected={expected} got={listed}"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "auth_type",
+    [MCPAuth.authorization, MCPAuth.bearer_token, MCPAuth.api_key, MCPAuth.basic, MCPAuth.token],
+)
+async def test_list_tools_injects_byok_credential_for_non_oauth2_auth_types(auth_type):
+    """Regression for BYOK servers on a non-oauth2 auth_type: the stored per-user credential must be
+    attached when listing tools, otherwise the upstream 401 is absorbed and the server lists nothing."""
+    try:
+        from litellm.proxy._experimental.mcp_server.server import (
+            _get_tools_from_mcp_servers,
+            set_auth_context,
+        )
+    except ImportError:
+        pytest.skip("MCP server not available")
+
+    user_api_key_auth = UserAPIKeyAuth(api_key="test_key", user_id="byok_user")
+    set_auth_context(user_api_key_auth)
+
+    server = MagicMock()
+    server.server_id = "byok_server"
+    server.name = "byok"
+    server.alias = "byok"
+    server.server_name = "byok"
+    server.auth_type = auth_type
+    server.is_byok = True
+    server.allowed_tools = None
+    server.disallowed_tools = None
+    server.extra_headers = None
+    server.tool_name_to_display_name = None
+    server.tool_name_to_description = None
+
+    seen_auth_headers = []
+
+    async def mock_get_tools_from_server(server, mcp_auth_header=None, add_prefix=False, **kwargs):
+        seen_auth_headers.append(mcp_auth_header)
+        tool = MagicMock()
+        tool.name = f"{server.alias}-toolA" if add_prefix else "toolA"
+        tool.description = "desc"
+        tool.inputSchema = {}
+        return [tool]
+
+    mock_manager = MagicMock()
+    mock_manager.get_allowed_mcp_servers = AsyncMock(return_value=[server.server_id])
+    mock_manager.get_mcp_server_by_id = MagicMock(return_value=server)
+    mock_manager.filter_server_ids_by_ip_with_info = lambda server_ids, client_ip: (server_ids, 0)
+    mock_manager._get_tools_from_server = mock_get_tools_from_server
+
+    with (
+        patch(
+            "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+            mock_manager,
+        ),
+        patch(
+            "litellm.proxy._experimental.mcp_server.server._get_byok_credential",
+            AsyncMock(return_value="personal-api-key"),
+        ),
+    ):
+        listing = await _get_tools_from_mcp_servers(
+            user_api_key_auth=user_api_key_auth,
+            mcp_auth_header=None,
+            mcp_servers=None,
+            mcp_server_auth_headers=None,
+        )
+
+    assert seen_auth_headers == ["personal-api-key"]
+    assert [tool.name for tool in listing.tools] == ["byok-toolA"]

tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py

@@ -42,6 +42,7 @@ from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
     _deserialize_json_list,
     _normalize_mcp_server_cost_info,
     _obo_retry_applies,
+    _resolve_openapi_tool_auth,
     _should_strip_caller_authorization,
     _without_authorization,
 )
@@ -4833,6 +4834,74 @@ class TestMCPServerManager:
         with pytest.raises(ValueError, match="Tool missing_tool not found"):
             manager._resolve_mcp_server_for_tool_call("github", "missing_tool")
 
+    @staticmethod
+    def _manager_with_deepwiki_and_huggingface() -> MCPServerManager:
+        manager = MCPServerManager()
+        deepwiki = MCPServer(server_id="deepwiki-id", name="deepwiki", server_name="deepwiki", transport=MCPTransport.http)
+        huggingface = MCPServer(
+            server_id="huggingface-id", name="huggingface", server_name="huggingface", transport=MCPTransport.http
+        )
+        manager.registry = {"deepwiki-id": deepwiki, "huggingface-id": huggingface}
+        manager.tool_name_to_mcp_server_name_mapping = {
+            "read_wiki_structure": "deepwiki",
+            "deepwiki-read_wiki_structure": "deepwiki",
+            "hub_repo_search": "huggingface",
+            "huggingface-hub_repo_search": "huggingface",
+        }
+        return manager
+
+    def test_resolve_mcp_server_for_tool_call_rejects_tool_exposed_only_by_another_server(self):
+        manager = self._manager_with_deepwiki_and_huggingface()
+
+        with pytest.raises(ValueError, match="Tool read_wiki_structure not found"):
+            manager._resolve_mcp_server_for_tool_call("huggingface", "read_wiki_structure")
+        with pytest.raises(ValueError, match="Tool hub_repo_search not found"):
+            manager._resolve_mcp_server_for_tool_call("deepwiki", "hub_repo_search")
+
+        assert manager._resolve_mcp_server_for_tool_call("deepwiki", "read_wiki_structure") is manager.registry["deepwiki-id"]
+        assert manager._resolve_mcp_server_for_tool_call("huggingface", "hub_repo_search") is manager.registry["huggingface-id"]
+
+    def test_get_mcp_server_from_tool_name_rejects_other_servers_prefix(self):
+        manager = self._manager_with_deepwiki_and_huggingface()
+
+        assert manager._get_mcp_server_from_tool_name("huggingface-read_wiki_structure") is None
+        assert manager._get_mcp_server_from_tool_name("deepwiki-hub_repo_search") is None
+        assert manager._get_mcp_server_from_tool_name("deepwiki-read_wiki_structure") is manager.registry["deepwiki-id"]
+        assert manager._get_mcp_server_from_tool_name("huggingface-hub_repo_search") is manager.registry["huggingface-id"]
+
+    def test_resolve_mcp_server_for_tool_call_shared_bare_name_resolves_via_own_prefixed_spelling(self):
+        manager = MCPServerManager()
+        zapier = MCPServer(server_id="zapier-id", name="zapier", alias="zapier-alias", transport=MCPTransport.http)
+        other = MCPServer(server_id="other-id", name="other", server_name="other", transport=MCPTransport.http)
+        manager.registry = {"zapier-id": zapier, "other-id": other}
+        manager.tool_name_to_mcp_server_name_mapping = {
+            "create_zap": "other",
+            "other-create_zap": "other",
+            "zapier-alias-create_zap": "zapier-alias",
+        }
+
+        assert manager._resolve_mcp_server_for_tool_call("zapier", "create_zap") is zapier
+        assert manager._resolve_mcp_server_for_tool_call("other", "create_zap") is other
+
+    def test_remove_server_drops_only_its_own_tool_mapping_rows(self):
+        manager = self._manager_with_deepwiki_and_huggingface()
+
+        manager.remove_server(
+            LiteLLM_MCPServerTable(
+                server_id="huggingface-id",
+                alias="huggingface",
+                url="https://huggingface.co/mcp",
+                transport=MCPTransport.http,
+                created_at=datetime.now(),
+                updated_at=datetime.now(),
+            )
+        )
+
+        assert manager.tool_name_to_mcp_server_name_mapping == {
+            "read_wiki_structure": "deepwiki",
+            "deepwiki-read_wiki_structure": "deepwiki",
+        }
+
     @pytest.mark.asyncio
     async def test_resolve_oauth2_headers_skipped_when_not_user_oauth(self):
         """Returns input headers unchanged when server does not need user OAuth."""
@@ -10457,3 +10526,242 @@ class TestSessionResourceScopeIntersect:
         ):
             fallback = await manager.get_allowed_mcp_servers(auth)
         assert fallback == ["granted-id"]
+
+
+class TestClientForwardedDiscoveryFailureIsNotFatal:
+    """A failed OAuth metadata discovery may only brick the flows the gateway runs itself.
+
+    ``true_passthrough`` / ``oauth_delegate`` forward the caller's own bearer and mint nothing, so an
+    upstream that publishes no RFC 9728 metadata (an internal API, or any IdP unreachable from the
+    pod) must still serve sessions instead of 503-ing before the upstream is ever contacted.
+    """
+
+    @staticmethod
+    def _config(auth_type: MCPAuthType, dcr_bridge: bool | None) -> dict[str, dict[str, object]]:
+        entry: Final[dict[str, object]] = {
+            "url": "https://up.example.com/mcp",
+            "transport": MCPTransport.http,
+            "auth_type": auth_type,
+            **({"oauth2_flow": "authorization_code"} if auth_type == MCPAuth.oauth2 else {}),
+            **({"dcr_bridge": dcr_bridge} if dcr_bridge is not None else {}),
+        }
+        return {"upstream": entry}
+
+    async def _registered(self, manager: MCPServerManager, auth_type: MCPAuthType, dcr_bridge: bool | None):
+        with (
+            patch.object(manager, "_discover_oauth_metadata_for_server", new=AsyncMock(return_value=None)),
+            patch.object(manager, "initialize_tool_name_to_mcp_server_name_mapping"),
+        ):
+            await manager.load_servers_from_config(self._config(auth_type, dcr_bridge))
+        return next(iter(manager.config_mcp_servers.values()))
+
+    @pytest.mark.parametrize(
+        "auth_type, dcr_bridge, serves_without_endpoints",
+        [
+            (MCPAuth.true_passthrough, None, True),
+            (MCPAuth.true_passthrough, True, True),
+            (MCPAuth.oauth_delegate, None, True),
+            (MCPAuth.oauth_delegate, True, True),
+            (MCPAuth.oauth2, None, False),
+            (MCPAuth.oauth2_token_exchange, None, False),
+        ],
+    )
+    @pytest.mark.parametrize("failure", ["incomplete", "timed_out"])
+    @pytest.mark.asyncio
+    async def test_discovery_failure_blocks_only_gateway_run_flows(
+        self,
+        auth_type: MCPAuthType,
+        dcr_bridge: bool | None,
+        serves_without_endpoints: bool,
+        failure: str,
+    ):
+        manager = MCPServerManager()
+        server = await self._registered(manager, auth_type, dcr_bridge)
+        manager._set_oauth_discovery_deferred(server.server_id, True)
+
+        async def never_returns(_server):
+            await asyncio.Future()
+
+        discovery_patch: Final = (
+            {"new": AsyncMock(return_value=None)} if failure == "incomplete" else {"side_effect": never_returns}
+        )
+        with (
+            patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.MCP_METADATA_TIMEOUT", 0.01),
+            patch.object(manager, "_discover_oauth_metadata_for_server", **discovery_patch),
+        ):
+            if not serves_without_endpoints:
+                with pytest.raises(HTTPException) as exc:
+                    await manager.ensure_oauth_metadata_discovered(server)
+                assert exc.value.status_code == 503
+                return
+
+            resolved = await manager.ensure_oauth_metadata_discovered(server)
+
+        assert resolved is manager.config_mcp_servers[server.server_id]
+        assert resolved.authorization_url is None
+        assert resolved.token_url is None
+        assert manager._oauth_discovery_slot(server.server_id) is not None
+
+    @pytest.mark.parametrize(
+        "auth_type, serves_the_listing",
+        [(MCPAuth.true_passthrough, True), (MCPAuth.oauth_delegate, True), (MCPAuth.oauth2, False)],
+    )
+    @pytest.mark.asyncio
+    async def test_listing_leg_serves_a_forwarding_server_whose_discovery_failed(
+        self, auth_type: MCPAuthType, serves_the_listing: bool
+    ):
+        """The listing leg is where the 503 became an empty tool list, so pin the fix there too.
+
+        ``_get_tools_from_server`` is the per-server leg the aggregate absorbs: a failure here is what
+        the fan-out turns into HTTP 200 with ``tools: []``, which is why the outage carried no
+        diagnostic. A forwarding server must now reach its upstream, and a gateway-run flow must still
+        surface the fault rather than be silently listed as empty.
+        """
+        manager = MCPServerManager()
+        server = await self._registered(manager, auth_type, None)
+        manager._set_oauth_discovery_deferred(server.server_id, True)
+        manager._fetch_tools_with_timeout = AsyncMock(
+            return_value=[MCPTool(name="list_reports", description="d", inputSchema={"type": "object"})]
+        )
+
+        with patch.object(manager, "_discover_oauth_metadata_for_server", new=AsyncMock(return_value=None)):
+            if not serves_the_listing:
+                with pytest.raises(MCPServerListError):
+                    await manager._get_tools_from_server(server=server)
+                return
+            tools = await manager._get_tools_from_server(server=server)
+
+        assert [tool.name for tool in tools] == ["upstream-list_reports"]
+        manager._fetch_tools_with_timeout.assert_awaited_once()
+
+    @pytest.mark.parametrize("auth_type", [MCPAuth.true_passthrough, MCPAuth.oauth_delegate])
+    @pytest.mark.asyncio
+    async def test_client_forwarded_servers_keep_discovering_their_front_door_endpoints(
+        self, auth_type: MCPAuthType
+    ):
+        """Exempting these modes from the FAILURE must not exempt them from discovery itself.
+
+        ``/authorize``, ``/token`` and ``/register`` read the discovered endpoints for these servers
+        (``_resolve_ephemeral_dcr_client`` mints for ``true_passthrough`` whatever ``dcr_bridge``
+        says), so an exemption written into the unresolved-endpoints predicate would disarm the slot
+        and silently drop a working front door.
+        """
+        manager = MCPServerManager()
+        metadata: Final = MCPOAuthMetadata(
+            authorization_url="https://idp.example.com/authorize",
+            token_url="https://idp.example.com/token",
+            registration_url="https://idp.example.com/register",
+        )
+        with (
+            patch.object(manager, "_discover_oauth_metadata_for_server", new=AsyncMock(return_value=metadata)),
+            patch.object(manager, "initialize_tool_name_to_mcp_server_name_mapping"),
+        ):
+            await manager.load_servers_from_config(self._config(auth_type, None))
+            server = next(iter(manager.config_mcp_servers.values()))
+            resolved = await manager.ensure_oauth_metadata_discovered(server)
+
+        assert resolved.authorization_url == "https://idp.example.com/authorize"
+        assert resolved.token_url == "https://idp.example.com/token"
+        assert resolved.registration_url == "https://idp.example.com/register"
+        assert manager.config_mcp_servers[server.server_id].authorization_url == "https://idp.example.com/authorize"
+        assert manager._oauth_discovery_slot(server.server_id) is None
+
+
+class TestResolveOpenapiToolAuth:
+    """The credential matrix for a ``spec_path`` server's two OpenAPI dispatch arms.
+
+    A per-server ``x-mcp-{alias}-authorization`` is already a complete header value and is forwarded
+    verbatim; a BYOK credential is a raw secret that takes the auth-type prefix. Conflating them
+    ships ``Bearer Bearer <token>``, so every cell pins which of the two a value came from.
+    """
+
+    @staticmethod
+    def _server(auth_type: MCPAuthType = MCPAuth.oauth_delegate) -> MCPServer:
+        return MCPServer(
+            server_id="srv-openapi",
+            name="report_api",
+            server_name="report_api",
+            alias="report_api",
+            url="https://api.internal.example.com",
+            transport=MCPTransport.http,
+            auth_type=auth_type,
+            spec_path="https://api.internal.example.com/openapi.json",
+            extra_headers=["X-Tenant"],
+        )
+
+    @pytest.mark.parametrize(
+        "per_server, byok, expected_auth, expected_extra_keys, expected_credential",
+        [
+            (
+                {"report_api": "Bearer caller-token"},
+                None,
+                "Bearer caller-token",
+                {"X-Tenant"},
+                "Bearer caller-token",
+            ),
+            (
+                {"report_api": {"Authorization": "Bearer caller-token", "X-Trace": "abc"}},
+                None,
+                "Bearer caller-token",
+                {"X-Tenant", "X-Trace"},
+                {"Authorization": "Bearer caller-token", "X-Trace": "abc"},
+            ),
+            (
+                {"report_api": {"X-Api-Key": "k1"}},
+                "byok-secret",
+                "Bearer byok-secret",
+                {"X-Tenant", "X-Api-Key"},
+                "byok-secret",
+            ),
+            ({"report_api": {"X-Api-Key": "k1"}}, None, None, {"X-Tenant", "X-Api-Key"}, None),
+            (None, "byok-secret", "Bearer byok-secret", {"X-Tenant"}, "byok-secret"),
+            (None, None, None, {"X-Tenant"}, None),
+            ({"other_server": "Bearer wrong"}, None, None, {"X-Tenant"}, None),
+        ],
+    )
+    def test_credential_matrix(
+        self,
+        per_server: dict | None,
+        byok: str | None,
+        expected_auth: str | None,
+        expected_extra_keys: set,
+        expected_credential: object,
+    ):
+        auth_value, forwarded, credential = _resolve_openapi_tool_auth(
+            mcp_server=self._server(),
+            mcp_auth_header=byok,
+            mcp_server_auth_headers=per_server,
+            raw_headers={"x-tenant": "acme", "authorization": "Bearer admission-key"},
+            user_api_key_auth=None,
+        )
+
+        assert auth_value == expected_auth
+        assert set(forwarded or {}) == expected_extra_keys
+        assert credential == expected_credential
+
+    def test_per_server_value_is_never_re_prefixed(self):
+        """The regression that a naive wiring produces: the caller already sent ``Bearer <token>``."""
+        auth_value, _, credential = _resolve_openapi_tool_auth(
+            mcp_server=self._server(auth_type=MCPAuth.api_key),
+            mcp_auth_header="byok-secret",
+            mcp_server_auth_headers={"report_api": "Bearer caller-token"},
+            raw_headers=None,
+            user_api_key_auth=None,
+        )
+
+        assert auth_value == "Bearer caller-token"
+        assert credential == "Bearer caller-token"
+        assert not auth_value.startswith("ApiKey ")
+
+    def test_per_server_authorization_is_not_also_left_in_forwarded_headers(self):
+        """``resolve_openapi_upstream_auth`` pops Authorization out of the forwarded headers, so a
+        second copy there would give the passthrough arm two sources to reconcile."""
+        _, forwarded, _ = _resolve_openapi_tool_auth(
+            mcp_server=self._server(),
+            mcp_auth_header=None,
+            mcp_server_auth_headers={"report_api": {"Authorization": "Bearer caller-token"}},
+            raw_headers={"x-tenant": "acme"},
+            user_api_key_auth=None,
+        )
+
+        assert "Authorization" not in (forwarded or {})

tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py

@@ -189,9 +189,11 @@ async def test_openapi_local_tool_denied_when_server_not_resolvable():
 
     pre_call = AsyncMock(return_value={})
     handle_local = AsyncMock(return_value=[])
+    resolve_auth = MagicMock()
 
     # `_get_mcp_server_from_tool_name` returns None — no server context.
     with (
+        patch.object(mcp_module, "_resolve_openapi_tool_auth", new=resolve_auth),
         patch.object(
             mcp_module.global_mcp_server_manager,
             "_get_mcp_server_from_tool_name",
@@ -228,6 +230,9 @@ async def test_openapi_local_tool_denied_when_server_not_resolvable():
     assert exc.value.status_code == 503
     pre_call.assert_not_awaited()
     handle_local.assert_not_awaited()
+    # The credential resolver takes a non-optional server, so the 503 guard above it is what keeps
+    # a missing server from ever reaching it. Pinned here so moving the guard reds this test.
+    resolve_auth.assert_not_called()
 
 
 @pytest.mark.asyncio
@@ -549,3 +554,101 @@ async def test_unknown_tool_name_still_reports_not_found():
 
     assert exc.value.status_code == 404
     assert "not found" in str(exc.value.detail)
+
+
+OPENAPI_PER_SERVER_TOKEN = "Bearer per-server-upstream-token"
+
+
+def _spec_path_server() -> MCPServer:
+    return MCPServer(
+        server_id="srv-reports",
+        name="report_api",
+        server_name="report_api",
+        alias="report_api",
+        url="https://api.internal.example.com",
+        transport=MCPTransport.http,
+        auth_type=MCPAuth.oauth_delegate,
+        spec_path="https://api.internal.example.com/openapi.json",
+    )
+
+
+@pytest.mark.parametrize("dispatch_arm", ["local_registry", "call_tool"])
+@pytest.mark.asyncio
+async def test_per_server_auth_header_reaches_both_openapi_dispatch_arms(dispatch_arm: str):
+    """`x-mcp-{alias}-authorization` must survive on BOTH OpenAPI dispatch arms.
+
+    OpenAPI tools live in the local tool registry, so `execute_mcp_tool` serves MCP-protocol and REST
+    tool calls while `MCPServerManager.call_tool` serves the responses-API handler. Both arms sourced
+    the upstream credential only from the deprecated global / BYOK `mcp_auth_header`, so the
+    per-server header was dropped and the upstream API saw no Authorization at all.
+
+    Asserting on the resolver kwarg as well as the ContextVar is deliberate: for the client-forwarded
+    modes the credential has to reach `resolve_openapi_upstream_auth`, whose passthrough arm outranks
+    the ContextVar when it materializes a header.
+    """
+    from litellm.proxy._experimental.mcp_server import server as mcp_module
+    from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import (
+        _request_auth_header,
+    )
+
+    server = _spec_path_server()
+    auth_headers = {"report_api": {"Authorization": OPENAPI_PER_SERVER_TOKEN}}
+    user = UserAPIKeyAuth(api_key="sk-user", user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER.value)
+    captured: dict = {}
+
+    async def fake_resolver(**kwargs):
+        captured["resolver_credential"] = kwargs["mcp_auth_header"]
+        return None, kwargs["forwarded_headers"]
+
+    async def capture_local(_name, _arguments):
+        captured["injected"] = _request_auth_header.get()
+        return []
+
+    async def capture_openapi_handler(_server, _name, _arguments):
+        captured["injected"] = _request_auth_header.get()
+        return []
+
+    manager = mcp_module.global_mcp_server_manager
+    with (
+        patch.object(manager, "resolve_openapi_upstream_auth", new=fake_resolver),
+        patch.object(manager, "pre_call_tool_check", new=AsyncMock(return_value={})),
+    ):
+        if dispatch_arm == "local_registry":
+            fake_tool = MagicMock()
+            fake_tool.name = "list_reports"
+            with (
+                patch.object(manager, "_get_mcp_server_from_tool_name", return_value=server),
+                patch.object(mcp_module.global_mcp_tool_registry, "get_tool", return_value=fake_tool),
+                patch(
+                    "litellm.proxy._experimental.mcp_server.server._handle_local_mcp_tool",
+                    new=capture_local,
+                ),
+                patch(
+                    "litellm.proxy._experimental.mcp_server.server.MCPRequestHandler.is_tool_allowed",
+                    return_value=True,
+                ),
+            ):
+                await mcp_module.execute_mcp_tool(
+                    name="list_reports",
+                    arguments={},
+                    allowed_mcp_servers=[server],
+                    start_time=datetime.now(timezone.utc),
+                    user_api_key_auth=user,
+                    mcp_server_auth_headers=auth_headers,
+                )
+        else:
+            with (
+                patch.object(manager, "_resolve_mcp_server_for_tool_call", return_value=server),
+                patch.object(manager, "_call_openapi_tool_handler", new=capture_openapi_handler),
+            ):
+                await manager.call_tool(
+                    server_name="report_api",
+                    name="list_reports",
+                    arguments={},
+                    user_api_key_auth=user,
+                    mcp_server_auth_headers=auth_headers,
+                )
+
+    assert captured["resolver_credential"] == {"Authorization": OPENAPI_PER_SERVER_TOKEN}
+    assert captured["injected"] == OPENAPI_PER_SERVER_TOKEN
+    assert _request_auth_header.get() is None

tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py

@@ -164,6 +164,41 @@ class TestProxyExceptionPassthrough:
         mock_logging.post_call_failure_hook.assert_awaited_once()
 
 
+class TestFailureHookRequestData:
+    @pytest.mark.asyncio
+    async def test_failure_hook_gets_post_setup_data_with_logging_obj(self):
+        """Request setup replaces the processor's data dict (adding the logging
+        object the failure hook needs to lift token usage from); the exception
+        handler must pass that replaced dict, not the raw request body dict."""
+        import litellm.proxy.anthropic_endpoints.endpoints as ep
+        import litellm.proxy.proxy_server as proxy_server
+        from litellm.proxy._types import ProxyException, UserAPIKeyAuth
+
+        captured = {}
+
+        async def fake_process(self, **kwargs):
+            self.data = {**self.data, "litellm_logging_obj": "logging-obj-sentinel"}
+            captured["processor_data"] = self.data
+            raise RuntimeError("provider timeout")
+
+        with (
+            patch.object(ep, "_read_request_body", new=AsyncMock(return_value={"model": "claude-sonnet"})),
+            patch.object(ep.ProxyBaseLLMRequestProcessing, "base_process_llm_request", new=fake_process),
+            patch.object(proxy_server, "proxy_logging_obj") as mock_logging,
+        ):
+            mock_logging.post_call_failure_hook = AsyncMock()
+            with pytest.raises(ProxyException):
+                await ep.anthropic_response(
+                    fastapi_response=MagicMock(),
+                    request=MagicMock(),
+                    user_api_key_dict=UserAPIKeyAuth(),
+                )
+
+        hook_request_data = mock_logging.post_call_failure_hook.await_args.kwargs["request_data"]
+        assert hook_request_data is captured["processor_data"]
+        assert hook_request_data["litellm_logging_obj"] == "logging-obj-sentinel"
+
+
 class TestEventLoggingBatchEndpoint:
     """Test the stubbed event logging batch endpoint"""
 

tests/test_litellm/proxy/common_utils/test_registry_read_through.py

@@ -0,0 +1,476 @@
+import asyncio
+from typing import Final
+
+import pytest
+
+from litellm.proxy.common_utils.registry_read_through import RegistryReadThrough
+
+
+class ResyncSpy:
+    def __init__(self, found: bool = True, error: Exception | None = None) -> None:
+        self.found = found
+        self.error = error
+        self.calls: list[str] = []
+
+    async def __call__(self, key: str) -> bool:
+        self.calls.append(key)
+        if self.error is not None:
+            raise self.error
+        return self.found
+
+
+@pytest.mark.asyncio
+async def test_attempt_returns_true_when_resync_finds_object():
+    spy: Final = ResyncSpy(found=True)
+    read_through: Final = RegistryReadThrough(resync=spy)
+
+    assert await read_through.attempt("new-model") is True
+    assert spy.calls == ["new-model"]
+
+
+@pytest.mark.asyncio
+async def test_attempt_found_key_is_not_negative_cached():
+    spy: Final = ResyncSpy(found=True)
+    read_through: Final = RegistryReadThrough(resync=spy)
+
+    assert await read_through.attempt("new-model") is True
+    assert await read_through.attempt("new-model") is True
+    assert spy.calls == ["new-model", "new-model"]
+
+
+@pytest.mark.asyncio
+async def test_missing_key_is_negative_cached_within_ttl():
+    spy: Final = ResyncSpy(found=False)
+    read_through: Final = RegistryReadThrough(resync=spy, miss_ttl_seconds=60.0)
+
+    assert await read_through.attempt("ghost-model") is False
+    assert await read_through.attempt("ghost-model") is False
+    assert spy.calls == ["ghost-model"]
+
+
+@pytest.mark.asyncio
+async def test_negative_cache_expires_and_resync_runs_again():
+    spy: Final = ResyncSpy(found=False)
+    read_through: Final = RegistryReadThrough(resync=spy, miss_ttl_seconds=0.05)
+
+    assert await read_through.attempt("ghost-model") is False
+    await asyncio.sleep(0.1)
+    assert await read_through.attempt("ghost-model") is False
+    assert spy.calls == ["ghost-model", "ghost-model"]
+
+
+@pytest.mark.asyncio
+async def test_resync_exception_returns_false_without_negative_caching():
+    spy: Final = ResyncSpy(error=RuntimeError("db down"))
+    read_through: Final = RegistryReadThrough(resync=spy)
+
+    assert await read_through.attempt("new-model") is False
+    assert await read_through.attempt("new-model") is False
+    assert spy.calls == ["new-model", "new-model"]
+
+
+@pytest.mark.asyncio
+async def test_concurrent_attempts_for_missing_key_resync_once():
+    class SlowResyncSpy(ResyncSpy):
+        async def __call__(self, key: str) -> bool:
+            await asyncio.sleep(0.05)
+            return await super().__call__(key)
+
+    spy: Final = SlowResyncSpy(found=False)
+    read_through: Final = RegistryReadThrough(resync=spy, miss_ttl_seconds=60.0)
+
+    results: Final = await asyncio.gather(*(read_through.attempt("ghost-model") for _ in range(5)))
+    assert results == [False] * 5
+    assert spy.calls == ["ghost-model"]
+
+
+@pytest.mark.asyncio
+async def test_distinct_keys_do_not_share_negative_cache():
+    spy: Final = ResyncSpy(found=False)
+    read_through: Final = RegistryReadThrough(resync=spy, miss_ttl_seconds=60.0)
+
+    assert await read_through.attempt("ghost-a") is False
+    assert await read_through.attempt("ghost-b") is False
+    assert spy.calls == ["ghost-a", "ghost-b"]
+
+
+@pytest.mark.asyncio
+async def test_resync_budget_exhausted_blocks_resync_without_negative_caching():
+    spy: Final = ResyncSpy(found=False)
+    read_through: Final = RegistryReadThrough(
+        resync=spy, miss_ttl_seconds=60.0, max_resyncs_per_window=2, resync_window_seconds=60.0
+    )
+
+    assert await read_through.attempt("ghost-a") is False
+    assert await read_through.attempt("ghost-b") is False
+    assert await read_through.attempt("ghost-c") is False
+    assert spy.calls == ["ghost-a", "ghost-b"]
+    assert read_through._recent_misses.get_cache("ghost-c") is None
+
+
+@pytest.mark.asyncio
+async def test_resync_budget_replenishes_after_window():
+    spy: Final = ResyncSpy(found=True)
+    read_through: Final = RegistryReadThrough(resync=spy, max_resyncs_per_window=1, resync_window_seconds=0.05)
+
+    assert await read_through.attempt("model-a") is True
+    assert await read_through.attempt("model-b") is False
+    await asyncio.sleep(0.1)
+    assert await read_through.attempt("model-b") is True
+    assert spy.calls == ["model-a", "model-b"]
+
+
+class FakeAgentRow:
+    def __init__(self, agent_id: str, agent_name: str) -> None:
+        self.agent_id = agent_id
+        self.agent_name = agent_name
+        self.object_permission = None
+        self.spend = 0.0
+
+    def model_dump(self):
+        return {
+            "agent_id": self.agent_id,
+            "agent_name": self.agent_name,
+            "agent_card_params": {"name": self.agent_name, "url": "http://db-agent"},
+            "litellm_params": {},
+            "object_permission": None,
+            "spend": self.spend,
+        }
+
+
+@pytest.fixture
+def clean_agent_registry():
+    from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
+
+    original_agents: Final = list(global_agent_registry.agent_list)
+    original_config_agents: Final = getattr(global_agent_registry, "config_agents", ())
+    global_agent_registry.agent_list = []
+    global_agent_registry.config_agents = ()
+    try:
+        yield global_agent_registry
+    finally:
+        global_agent_registry.agent_list = original_agents
+        global_agent_registry.config_agents = original_config_agents
+
+
+@pytest.mark.asyncio
+async def test_get_agent_with_read_through_recovers_agent_created_on_sibling_replica(
+    clean_agent_registry, monkeypatch
+):
+    from unittest.mock import AsyncMock, MagicMock
+
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.proxy.common_utils.registry_read_through import get_agent_with_read_through
+
+    agent_id: Final = "read-through-db-agent-id"
+    prisma_client: Final = MagicMock()
+    prisma_client.db.litellm_agentstable.find_unique = AsyncMock(
+        return_value=FakeAgentRow(agent_id, "read-through-db-agent")
+    )
+    monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+    monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+
+    assert clean_agent_registry.get_agent_by_id(agent_id=agent_id) is None
+    agent: Final = await get_agent_with_read_through(agent_id)
+
+    assert agent is not None
+    assert agent.agent_id == agent_id
+    prisma_client.db.litellm_agentstable.find_unique.assert_awaited_once_with(
+        where={"agent_id": agent_id},
+        include={"object_permission": True},
+    )
+
+
+@pytest.mark.asyncio
+async def test_get_agent_with_read_through_recovers_agent_by_name(clean_agent_registry, monkeypatch):
+    from unittest.mock import AsyncMock, MagicMock
+
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.proxy.common_utils.registry_read_through import get_agent_with_read_through
+
+    agent_name: Final = "read-through-db-agent-by-name"
+    prisma_client: Final = MagicMock()
+    prisma_client.db.litellm_agentstable.find_unique = AsyncMock(
+        side_effect=[None, FakeAgentRow("read-through-name-lookup-id", agent_name)]
+    )
+    monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+    monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+
+    agent: Final = await get_agent_with_read_through(agent_name)
+
+    assert agent is not None
+    assert agent.agent_name == agent_name
+    prisma_client.db.litellm_agentstable.find_unique.assert_awaited_with(
+        where={"agent_name": agent_name},
+        include={"object_permission": True},
+    )
+
+
+@pytest.mark.asyncio
+async def test_get_agent_with_read_through_returns_none_for_unknown_agent(clean_agent_registry, monkeypatch):
+    from unittest.mock import AsyncMock, MagicMock
+
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.proxy.common_utils.registry_read_through import get_agent_with_read_through
+
+    prisma_client: Final = MagicMock()
+    prisma_client.db.litellm_agentstable.find_unique = AsyncMock(return_value=None)
+    monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+    monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+
+    assert await get_agent_with_read_through("agent-nobody-created") is None
+    assert prisma_client.db.litellm_agentstable.find_unique.await_count == 2
+
+
+@pytest.mark.asyncio
+async def test_resync_agents_already_registered_skips_db(clean_agent_registry, monkeypatch):
+    from unittest.mock import AsyncMock, MagicMock
+
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.proxy.common_utils.registry_read_through import _resync_agents
+
+    agent_id: Final = "read-through-dedup-agent-id"
+    prisma_client: Final = MagicMock()
+    prisma_client.db.litellm_agentstable.find_unique = AsyncMock(
+        return_value=FakeAgentRow(agent_id, "read-through-dedup-agent")
+    )
+    monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+    monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+
+    assert await _resync_agents(agent_id) is True
+    assert await _resync_agents(agent_id) is True
+    assert prisma_client.db.litellm_agentstable.find_unique.await_count == 1
+    assert len(clean_agent_registry.agent_list) == 1
+
+
+class FakeGuardrailRow:
+    def __init__(self, guardrail_id: str, guardrail_name: str) -> None:
+        self.guardrail_id = guardrail_id
+        self.guardrail_name = guardrail_name
+
+    def __iter__(self):
+        return iter(
+            {
+                "guardrail_id": self.guardrail_id,
+                "guardrail_name": self.guardrail_name,
+                "litellm_params": {
+                    "guardrail": "litellm_content_filter",
+                    "mode": "pre_call",
+                    "default_on": True,
+                    "blocked_words": [{"keyword": "secret", "action": "BLOCK"}],
+                },
+                "guardrail_info": {},
+                "status": "active",
+            }.items()
+        )
+
+
+@pytest.mark.asyncio
+async def test_get_guardrail_with_read_through_recovers_guardrail_created_on_sibling_replica(monkeypatch):
+    from unittest.mock import AsyncMock, MagicMock
+
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.proxy.common_utils.registry_read_through import (
+        get_initialized_guardrail_with_read_through,
+    )
+    from litellm.proxy.guardrails.guardrail_registry import IN_MEMORY_GUARDRAIL_HANDLER
+
+    guardrail_id: Final = "read-through-db-guardrail-id"
+    guardrail_name: Final = "read-through-db-guardrail"
+    prisma_client: Final = MagicMock()
+    prisma_client.db.litellm_guardrailstable.find_first = AsyncMock(
+        return_value=FakeGuardrailRow(guardrail_id, guardrail_name)
+    )
+    prisma_client.db.litellm_guardrailstable.find_many = AsyncMock(
+        side_effect=AssertionError("full-table guardrail scan on read-through miss")
+    )
+    monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+    monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+
+    try:
+        guardrail: Final = await get_initialized_guardrail_with_read_through(guardrail_name=guardrail_name)
+        assert guardrail is not None
+        assert guardrail.guardrail_name == guardrail_name
+        prisma_client.db.litellm_guardrailstable.find_first.assert_awaited_once_with(
+            where={"guardrail_name": guardrail_name, "status": "active"}
+        )
+    finally:
+        IN_MEMORY_GUARDRAIL_HANDLER.delete_in_memory_guardrail(guardrail_id)
+
+
+@pytest.mark.asyncio
+async def test_get_guardrail_with_read_through_returns_none_for_unknown_guardrail(monkeypatch):
+    from unittest.mock import AsyncMock, MagicMock
+
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.proxy.common_utils.registry_read_through import (
+        get_initialized_guardrail_with_read_through,
+    )
+
+    prisma_client: Final = MagicMock()
+    prisma_client.db.litellm_guardrailstable.find_first = AsyncMock(return_value=None)
+    monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+    monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+
+    assert await get_initialized_guardrail_with_read_through(guardrail_name="guardrail-nobody-created") is None
+
+
+@pytest.mark.asyncio
+async def test_resync_guardrails_never_loads_non_active_rows(monkeypatch):
+    from unittest.mock import AsyncMock, MagicMock
+
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.proxy.common_utils.registry_read_through import _resync_guardrails
+
+    pending_name: Final = "pending-review-guardrail"
+    prisma_client: Final = MagicMock()
+    prisma_client.db.litellm_guardrailstable.find_first = AsyncMock(return_value=None)
+    monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+    monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+
+    assert await _resync_guardrails(pending_name) is False
+    prisma_client.db.litellm_guardrailstable.find_first.assert_awaited_once_with(
+        where={"guardrail_name": pending_name, "status": "active"}
+    )
+
+
+@pytest.mark.asyncio
+async def test_resync_guardrails_syncs_under_guardrail_reconcile_lock(monkeypatch):
+    from unittest.mock import AsyncMock, MagicMock
+
+    import litellm.proxy.common_utils.registry_read_through as read_through_module
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.proxy.common_utils.registry_read_through import _resync_guardrails
+    from litellm.proxy.guardrails.guardrail_registry import (
+        GUARDRAIL_RECONCILE_LOCK,
+        IN_MEMORY_GUARDRAIL_HANDLER,
+    )
+
+    guardrail_name: Final = "lock-scope-guardrail"
+    prisma_client: Final = MagicMock()
+    prisma_client.db.litellm_guardrailstable.find_first = AsyncMock(
+        return_value=FakeGuardrailRow("lock-scope-guardrail-id", guardrail_name)
+    )
+    lock_states: list[bool] = []
+
+    def record_sync(guardrail) -> None:
+        lock_states.append(GUARDRAIL_RECONCILE_LOCK.locked())
+
+    monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+    monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+    monkeypatch.setattr(IN_MEMORY_GUARDRAIL_HANDLER, "sync_guardrail_from_db", record_sync)
+    monkeypatch.setattr(read_through_module, "_initialized_guardrail", lambda guardrail_name: MagicMock())
+
+    assert await _resync_guardrails(guardrail_name) is True
+    assert lock_states == [True]
+    assert not GUARDRAIL_RECONCILE_LOCK.locked()
+
+
+@pytest.mark.asyncio
+async def test_resync_model_deployments_mutates_router_under_model_reconcile_lock(monkeypatch):
+    from unittest.mock import AsyncMock, MagicMock
+
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.proxy.common_utils.registry_read_through import _resync_model_deployments
+
+    prisma_client: Final = MagicMock()
+    prisma_client.db.litellm_proxymodeltable.find_many = AsyncMock(return_value=[MagicMock()])
+    router: Final = MagicMock()
+    router.get_model_list.return_value = []
+    lock_states: list[bool] = []
+
+    def record_add_deployment(db_models) -> None:
+        lock_states.append(proxy_server.MODEL_RECONCILE_LOCK.locked())
+
+    monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+    monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+    monkeypatch.setattr(proxy_server, "llm_router", router)
+    monkeypatch.setattr(proxy_server, "llm_model_list", None)
+    monkeypatch.setattr(proxy_server.proxy_config, "_add_deployment", record_add_deployment)
+
+    assert await _resync_model_deployments("lock-scope-model") is True
+    assert lock_states == [True]
+    assert not proxy_server.MODEL_RECONCILE_LOCK.locked()
+
+
+@pytest.mark.asyncio
+async def test_resync_model_deployments_respects_supported_db_objects(monkeypatch):
+    from unittest.mock import AsyncMock, MagicMock
+
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.proxy.common_utils.registry_read_through import _resync_model_deployments
+
+    prisma_client: Final = MagicMock()
+    prisma_client.db.litellm_proxymodeltable.find_many = AsyncMock(
+        side_effect=AssertionError("db hit for an object type this replica does not load")
+    )
+    monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+    monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+    monkeypatch.setattr(proxy_server, "general_settings", {"supported_db_objects": ["guardrails"]})
+
+    assert await _resync_model_deployments("gated-out-model") is False
+
+
+@pytest.mark.asyncio
+async def test_resync_guardrails_respects_supported_db_objects(monkeypatch):
+    from unittest.mock import AsyncMock, MagicMock
+
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.proxy.common_utils.registry_read_through import _resync_guardrails
+
+    prisma_client: Final = MagicMock()
+    prisma_client.db.litellm_guardrailstable.find_unique = AsyncMock(
+        side_effect=AssertionError("db hit for an object type this replica does not load")
+    )
+    monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+    monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+    monkeypatch.setattr(proxy_server, "general_settings", {"supported_db_objects": ["models"]})
+
+    assert await _resync_guardrails("gated-out-guardrail") is False
+
+
+@pytest.mark.asyncio
+async def test_resync_agents_respects_supported_db_objects(clean_agent_registry, monkeypatch):
+    from unittest.mock import AsyncMock, MagicMock
+
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.proxy.common_utils.registry_read_through import _resync_agents
+
+    prisma_client: Final = MagicMock()
+    prisma_client.db.litellm_agentstable.find_unique = AsyncMock(
+        side_effect=AssertionError("db hit for an object type this replica does not load")
+    )
+    monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+    monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+    monkeypatch.setattr(proxy_server, "general_settings", {"supported_db_objects": ["models"]})
+
+    assert await _resync_agents("gated-out-agent") is False
+
+
+@pytest.mark.asyncio
+async def test_resync_agents_waits_for_agent_reload_and_skips_duplicate_registration(clean_agent_registry, monkeypatch):
+    from unittest.mock import AsyncMock, MagicMock
+
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.proxy.agent_endpoints.agent_registry import AGENT_RECONCILE_LOCK
+    from litellm.proxy.common_utils.registry_read_through import _resync_agents
+    from litellm.types.agents import AgentResponse
+
+    agent_id: Final = "reload-race-agent-id"
+    prisma_client: Final = MagicMock()
+    prisma_client.db.litellm_agentstable.find_unique = AsyncMock(
+        side_effect=AssertionError("db hit while the agent reload held the reconcile lock")
+    )
+    monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+    monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+
+    async with AGENT_RECONCILE_LOCK:
+        resync_task: Final = asyncio.ensure_future(_resync_agents(agent_id))
+        await asyncio.sleep(0.05)
+        assert not resync_task.done()
+        clean_agent_registry.register_agent(
+            agent_config=AgentResponse.model_validate(FakeAgentRow(agent_id, "reload-race-agent").model_dump())
+        )
+
+    assert await resync_task is True
+    assert len(clean_agent_registry.agent_list) == 1

tests/test_litellm/proxy/hooks/test_batch_file_validation.py

@@ -1382,6 +1382,315 @@ async def test_check_and_increment_computes_descriptors_when_not_passed():
     parallel_request_limiter._create_rate_limit_descriptors.assert_called_once()
 
 
+@pytest.mark.asyncio
+async def test_pre_call_enforces_project_otpm_limit_for_batch():
+    """VERIA regression: ``_create_batch_rate_limit_descriptors`` only asked
+    for the generic key/user/team/model descriptors, so a project caller
+    could submit a batch that consumed none of its configured project OTPM
+    quota. The project OTPM descriptor must now be present and charged with
+    the batch's estimated *output* tokens, not its input tokens."""
+    from litellm import DualCache
+    from litellm.proxy.hooks.batch_rate_limiter import _PROXY_BatchRateLimiter
+    from litellm.proxy.hooks.parallel_request_limiter_v3 import (
+        _PROXY_MaxParallelRequestsHandler_v3,
+    )
+    from litellm.proxy.utils import InternalUsageCache
+
+    local_cache = DualCache()
+    parallel_request_limiter = _PROXY_MaxParallelRequestsHandler_v3(
+        internal_usage_cache=InternalUsageCache(local_cache)
+    )
+    rate_limiter = _PROXY_BatchRateLimiter(
+        internal_usage_cache=InternalUsageCache(local_cache),
+        parallel_request_limiter=parallel_request_limiter,
+    )
+
+    user = UserAPIKeyAuth(
+        api_key="sk-project-batch-otpm",
+        models=["*"],
+        project_id="proj-mantle-batch",
+        project_metadata={"model_otpm_limit": {"gpt-4o-mini": 50}},
+    )
+
+    # Two rows each declaring max_tokens=40: 80 output tokens total, over the
+    # configured 50-token project OTPM limit but negligible input tokens.
+    mock_content = MagicMock()
+    mock_content.content = (
+        b'{"body": {"model": "gpt-4o-mini", "max_tokens": 40, '
+        b'"messages": [{"role": "user", "content": "hi"}]}}\n'
+        b'{"body": {"model": "gpt-4o-mini", "max_tokens": 40, '
+        b'"messages": [{"role": "user", "content": "hi"}]}}\n'
+    )
+
+    with (
+        patch("litellm.proxy.proxy_server.general_settings", {}),
+        patch("litellm.proxy.proxy_server.llm_router", MagicMock()),
+        patch(
+            "litellm.proxy.openai_files_endpoints.common_utils.get_credentials_for_model",
+            return_value={"custom_llm_provider": "openai"},
+        ),
+        patch("litellm.afile_content", new=AsyncMock(return_value=mock_content)),
+    ):
+        with pytest.raises(HTTPException) as exc:
+            await rate_limiter.async_pre_call_hook(
+                user_api_key_dict=user,
+                cache=local_cache,
+                data={"input_file_id": "file-abc123", "model": "gpt-4o-mini"},
+                call_type="acreate_batch",
+            )
+
+    assert exc.value.status_code == 429
+    assert "model_per_project_otpm" in str(exc.value.detail)
+
+
+@pytest.mark.asyncio
+async def test_pre_call_enforces_project_itpm_limit_for_batch():
+    """Companion to the OTPM regression above: a project's ITPM quota must
+    also apply to batch submissions."""
+    from litellm import DualCache
+    from litellm.proxy.hooks.batch_rate_limiter import _PROXY_BatchRateLimiter
+    from litellm.proxy.hooks.parallel_request_limiter_v3 import (
+        _PROXY_MaxParallelRequestsHandler_v3,
+    )
+    from litellm.proxy.utils import InternalUsageCache
+
+    local_cache = DualCache()
+    parallel_request_limiter = _PROXY_MaxParallelRequestsHandler_v3(
+        internal_usage_cache=InternalUsageCache(local_cache)
+    )
+    rate_limiter = _PROXY_BatchRateLimiter(
+        internal_usage_cache=InternalUsageCache(local_cache),
+        parallel_request_limiter=parallel_request_limiter,
+    )
+
+    user = UserAPIKeyAuth(
+        api_key="sk-project-batch-itpm",
+        models=["*"],
+        project_id="proj-mantle-batch",
+        project_metadata={"model_itpm_limit": {"gpt-4o-mini": 1}},
+    )
+
+    mock_content = MagicMock()
+    mock_content.content = (
+        b'{"body": {"model": "gpt-4o-mini", "max_tokens": 1, '
+        b'"messages": [{"role": "user", "content": "well over one token of input"}]}}\n'
+    )
+
+    with (
+        patch("litellm.proxy.proxy_server.general_settings", {}),
+        patch("litellm.proxy.proxy_server.llm_router", MagicMock()),
+        patch(
+            "litellm.proxy.openai_files_endpoints.common_utils.get_credentials_for_model",
+            return_value={"custom_llm_provider": "openai"},
+        ),
+        patch("litellm.afile_content", new=AsyncMock(return_value=mock_content)),
+    ):
+        with pytest.raises(HTTPException) as exc:
+            await rate_limiter.async_pre_call_hook(
+                user_api_key_dict=user,
+                cache=local_cache,
+                data={"input_file_id": "file-abc123", "model": "gpt-4o-mini"},
+                call_type="acreate_batch",
+            )
+
+    assert exc.value.status_code == 429
+    assert "model_per_project_itpm" in str(exc.value.detail)
+
+
+@pytest.mark.asyncio
+async def test_pre_call_enforces_project_otpm_limit_for_non_routing_row_model():
+    """VERIA regression: project ITPM/OTPM descriptors were built only for the
+    file-bound/top-level routing model, so a caller could bind the batch file
+    to an unlimited model while a JSONL row's own `body.model` named a
+    different, quota-limited model. That row's tokens must still be charged
+    against its own model's project OTPM quota."""
+    from litellm import DualCache
+    from litellm.proxy.hooks.batch_rate_limiter import _PROXY_BatchRateLimiter
+    from litellm.proxy.hooks.parallel_request_limiter_v3 import (
+        _PROXY_MaxParallelRequestsHandler_v3,
+    )
+    from litellm.proxy.utils import InternalUsageCache
+
+    local_cache = DualCache()
+    parallel_request_limiter = _PROXY_MaxParallelRequestsHandler_v3(
+        internal_usage_cache=InternalUsageCache(local_cache)
+    )
+    rate_limiter = _PROXY_BatchRateLimiter(
+        internal_usage_cache=InternalUsageCache(local_cache),
+        parallel_request_limiter=parallel_request_limiter,
+    )
+
+    # The routing model ("unlimited-model") has no configured quota; only
+    # "quota-limited-model" -- named inside the JSONL row, not the routing
+    # model -- has a project OTPM limit.
+    user = UserAPIKeyAuth(
+        api_key="sk-project-batch-cross-model",
+        models=["*"],
+        project_id="proj-mantle-batch",
+        project_metadata={"model_otpm_limit": {"quota-limited-model": 50}},
+    )
+
+    mock_content = MagicMock()
+    mock_content.content = (
+        b'{"body": {"model": "quota-limited-model", "max_tokens": 80, '
+        b'"messages": [{"role": "user", "content": "hi"}]}}\n'
+    )
+
+    with (
+        patch("litellm.proxy.proxy_server.general_settings", {}),
+        patch("litellm.proxy.proxy_server.llm_router", MagicMock()),
+        patch(
+            "litellm.proxy.openai_files_endpoints.common_utils.get_credentials_for_model",
+            return_value={"custom_llm_provider": "openai"},
+        ),
+        patch("litellm.afile_content", new=AsyncMock(return_value=mock_content)),
+    ):
+        with pytest.raises(HTTPException) as exc:
+            await rate_limiter.async_pre_call_hook(
+                user_api_key_dict=user,
+                cache=local_cache,
+                data={"input_file_id": "file-abc123", "model": "unlimited-model"},
+                call_type="acreate_batch",
+            )
+
+    assert exc.value.status_code == 429
+    assert "model_per_project_otpm" in str(exc.value.detail)
+    assert "quota-limited-model" in str(exc.value.detail)
+
+
+@pytest.mark.asyncio
+async def test_pre_call_charges_each_row_model_against_its_own_project_quota():
+    """A batch whose rows target two different project-quota-limited models
+    must charge each row's tokens only against its own model's quota, never
+    the other model's or the whole batch's combined total. The under-limit
+    model's request must succeed even though the over-limit model's row
+    would fail on its own."""
+    from litellm import DualCache
+    from litellm.proxy.hooks.batch_rate_limiter import _PROXY_BatchRateLimiter
+    from litellm.proxy.hooks.parallel_request_limiter_v3 import (
+        PROJECT_OTPM_DESCRIPTOR_KEY,
+        _PROXY_MaxParallelRequestsHandler_v3,
+    )
+    from litellm.proxy.utils import InternalUsageCache
+
+    local_cache = DualCache()
+    parallel_request_limiter = _PROXY_MaxParallelRequestsHandler_v3(
+        internal_usage_cache=InternalUsageCache(local_cache)
+    )
+    rate_limiter = _PROXY_BatchRateLimiter(
+        internal_usage_cache=InternalUsageCache(local_cache),
+        parallel_request_limiter=parallel_request_limiter,
+    )
+
+    user = UserAPIKeyAuth(
+        api_key="sk-project-batch-two-models",
+        models=["*"],
+        project_id="proj-mantle-batch",
+        project_metadata={
+            "model_otpm_limit": {"model-a": 1000, "model-b": 10},
+        },
+    )
+
+    # model-a stays comfortably under its 1000 OTPM limit; model-b's single
+    # row alone exceeds its 10 OTPM limit. If the two were combined into one
+    # counter (the pre-fix behavior for the routing model), model-a's ample
+    # headroom would mask model-b's overage.
+    mock_content = MagicMock()
+    mock_content.content = (
+        b'{"body": {"model": "model-a", "max_tokens": 5, '
+        b'"messages": [{"role": "user", "content": "hi"}]}}\n'
+        b'{"body": {"model": "model-b", "max_tokens": 40, '
+        b'"messages": [{"role": "user", "content": "hi"}]}}\n'
+    )
+
+    with (
+        patch("litellm.proxy.proxy_server.general_settings", {}),
+        patch("litellm.proxy.proxy_server.llm_router", MagicMock()),
+        patch(
+            "litellm.proxy.openai_files_endpoints.common_utils.get_credentials_for_model",
+            return_value={"custom_llm_provider": "openai"},
+        ),
+        patch("litellm.afile_content", new=AsyncMock(return_value=mock_content)),
+    ):
+        with pytest.raises(HTTPException) as exc:
+            await rate_limiter.async_pre_call_hook(
+                user_api_key_dict=user,
+                cache=local_cache,
+                data={"input_file_id": "file-abc123", "model": "model-a"},
+                call_type="acreate_batch",
+            )
+
+    assert exc.value.status_code == 429
+    assert "model-b" in str(exc.value.detail)
+
+    # model-a's own counter was not touched by model-b's rejection: a
+    # follow-up model-a-only batch well within its own limit must still pass.
+    model_a_status = await parallel_request_limiter.should_rate_limit(
+        descriptors=[
+            {
+                "key": PROJECT_OTPM_DESCRIPTOR_KEY,
+                "value": "proj-mantle-batch:model-a",
+                "rate_limit": {
+                    "requests_per_unit": None,
+                    "tokens_per_unit": 1000,
+                    "window_size": parallel_request_limiter.window_size,
+                },
+            }
+        ],
+        read_only=True,
+    )
+    assert model_a_status["overall_code"] == "OK"
+
+
+def test_should_not_skip_when_project_has_io_limit_for_non_routing_model():
+    """The no-limits skip must not fire just because the file-bound/top-level
+    routing model itself has no configured quota: a JSONL row can name a
+    different model that the project *does* quota, and that isn't knowable
+    without downloading and parsing the file."""
+    rate_limiter = _make_rate_limiter()
+    # No key/team/model-level limits at all -- only a project OTPM limit for a
+    # model unrelated to the routing model below.
+    rate_limiter.parallel_request_limiter._create_rate_limit_descriptors.return_value = [
+        {"rate_limit": {}}
+    ]
+    user = UserAPIKeyAuth(
+        api_key="sk",
+        models=["*"],
+        project_id="proj-mantle-batch",
+        project_metadata={"model_otpm_limit": {"some-other-model": 50}},
+    )
+    with patch("litellm.proxy.proxy_server.general_settings", {}):
+        should_skip, descriptors = rate_limiter._should_skip_batch_input_file_processing(
+            data={"model": "unlimited-model", "input_file_id": "file-abc"},
+            user_api_key_dict=user,
+        )
+    assert should_skip is False
+    assert descriptors is not None
+
+
+def test_should_skip_when_project_has_no_io_limits_and_no_other_limits():
+    """Sanity check for the new project-limits carve-out: a project caller
+    with no ITPM/OTPM configuration anywhere must still get the fast-path
+    skip when no other rate limits apply, exactly as before this fix."""
+    rate_limiter = _make_rate_limiter()
+    rate_limiter.parallel_request_limiter._create_rate_limit_descriptors.return_value = [
+        {"rate_limit": {}}
+    ]
+    user = UserAPIKeyAuth(
+        api_key="sk",
+        models=["*"],
+        project_id="proj-mantle-batch",
+        project_metadata={},
+    )
+    with patch("litellm.proxy.proxy_server.general_settings", {}):
+        should_skip, descriptors = rate_limiter._should_skip_batch_input_file_processing(
+            data={"model": "unlimited-model", "input_file_id": "file-abc"},
+            user_api_key_dict=user,
+        )
+    assert should_skip is True
+    assert descriptors is None
+
+
 @pytest.mark.asyncio
 async def test_count_input_file_usage_raises_on_non_bytes_content():
     from litellm.proxy.hooks.batch_rate_limiter import _PROXY_BatchRateLimiter
@@ -1671,3 +1980,117 @@ async def test_count_input_file_usage_collects_models_after_malformed_line():
             )
 
     assert exc.value.status_code == 403
+
+
+# ---------------------------------------------------------------------------
+# VERIA-Low regression: Responses batch rows must not bypass project OTPM
+# ---------------------------------------------------------------------------
+
+
+def _output_estimator():
+    """A `_PROXY_BatchRateLimiter` whose output-token floor is observable:
+    the no-`max_tokens` floor mock returns a distinctive sentinel so tests can
+    tell "floor was used" apart from "an explicit cap was read"."""
+    from litellm.proxy.hooks.batch_rate_limiter import _PROXY_BatchRateLimiter
+    from litellm.proxy.hooks.parallel_request_limiter_v3 import (
+        _PROXY_MaxParallelRequestsHandler_v3,
+    )
+
+    limiter = MagicMock()
+    limiter.no_max_tokens_output_floor.return_value = 999
+    limiter.get_output_candidate_count = _PROXY_MaxParallelRequestsHandler_v3.get_output_candidate_count
+    return _PROXY_BatchRateLimiter(
+        internal_usage_cache=MagicMock(),
+        parallel_request_limiter=limiter,
+    )
+
+
+def test_estimate_entry_output_tokens_zero_for_embeddings_url():
+    """A real `/v1/embeddings` row reserves zero output tokens."""
+    rate_limiter = _output_estimator()
+    entry = {
+        "url": "/v1/embeddings",
+        "body": {"model": "text-embedding-3-small", "input": "hello world"},
+    }
+
+    assert rate_limiter._estimate_entry_output_tokens(entry, None) == 0
+
+
+def test_estimate_entry_output_tokens_does_not_zero_responses_row_with_input():
+    """Pre-fix: a `/v1/responses` row carries `body.input` with no `messages`/
+    `prompt`, so the old body-shape heuristic misclassified it as embeddings
+    and reserved zero output tokens -- a project caller could submit large
+    Responses generations against a quota-limited model without consuming
+    OTPM. The row's own `url` (not body shape) must decide this."""
+    rate_limiter = _output_estimator()
+    entry = {
+        "url": "/v1/responses",
+        "body": {"model": "gpt-4o", "input": "write me an essay"},
+    }
+
+    # No explicit cap on the row, so it must fall back to the no-max-tokens
+    # floor -- never straight to zero.
+    assert rate_limiter._estimate_entry_output_tokens(entry, None) == 999
+    rate_limiter.parallel_request_limiter.no_max_tokens_output_floor.assert_called_once_with(None)
+
+
+def test_estimate_entry_output_tokens_uses_max_output_tokens_for_responses():
+    """`/v1/responses` caps output with `max_output_tokens`, not `max_tokens`/
+    `max_completion_tokens`. Pre-fix this field was never inspected, so a
+    capped Responses row still fell through to the (possibly larger) floor
+    estimate instead of the caller's own declared cap."""
+    rate_limiter = _output_estimator()
+    entry = {
+        "url": "/v1/responses",
+        "body": {"model": "gpt-4o", "input": "hi", "max_output_tokens": 123},
+    }
+
+    assert rate_limiter._estimate_entry_output_tokens(entry, None) == 123
+    rate_limiter.parallel_request_limiter.no_max_tokens_output_floor.assert_not_called()
+
+
+def test_estimate_entry_output_tokens_prefers_max_tokens_over_max_output_tokens():
+    """When a row somehow carries both fields, the chat-style cap wins first --
+    `max_output_tokens` is only consulted once the chat-style caps are absent."""
+    rate_limiter = _output_estimator()
+    entry = {
+        "url": "/v1/chat/completions",
+        "body": {
+            "model": "gpt-4o",
+            "messages": [],
+            "max_tokens": 50,
+            "max_output_tokens": 500,
+        },
+    }
+
+    assert rate_limiter._estimate_entry_output_tokens(entry, None) == 50
+
+
+@pytest.mark.parametrize(
+    ("body_extra", "expected"),
+    [
+        ({"max_tokens": 40, "n": 10}, 400),
+        ({"max_tokens": 40, "best_of": 5}, 200),
+        ({"max_tokens": 40, "n": 3, "best_of": 5}, 200),
+        ({"max_tokens": 40, "n": 0}, 40),
+        ({"max_tokens": 40, "n": -2}, 40),
+        ({"max_tokens": 40, "n": 5.0}, 200),
+        ({"max_tokens": 40, "n": "10"}, 400),
+        ({"max_tokens": 40, "n": "not-a-number"}, 40),
+        ({"max_tokens": 40, "n": 1e309}, 40),
+        ({"max_tokens": 1e309, "n": 3}, 2997),
+        ({"n": 3}, 2997),
+    ],
+)
+def test_estimate_entry_output_tokens_multiplies_candidate_count(body_extra, expected):
+    """A row generating n / best_of candidates consumes that many completions'
+    worth of output tokens, so the OTPM reservation must scale with the
+    effective candidate count. Pre-fix a `max_tokens: 40, n: 10` row consumed
+    up to 400 output tokens while reserving only 40."""
+    rate_limiter = _output_estimator()
+    entry = {
+        "url": "/v1/chat/completions",
+        "body": {"model": "gpt-4o", "messages": [], **body_extra},
+    }
+
+    assert rate_limiter._estimate_entry_output_tokens(entry, None) == expected

tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py

@@ -3116,6 +3116,192 @@ async def test_project_model_rate_limits_not_triggered_for_other_model_v3():
     ), f"model_per_project should not be added for unrelated model, got: {descriptor_keys}"
 
 
+@pytest.mark.asyncio
+async def test_project_model_itpm_otpm_limits_enforced_v3():
+    """
+    Project-level model_itpm_limit/model_otpm_limit must produce distinct
+    Bedrock Mantle-style input and output token descriptors.
+    """
+    _api_key = hash_token("sk-project-io-test")
+    local_cache = DualCache()
+    parallel_request_handler = _PROXY_MaxParallelRequestsHandler(
+        internal_usage_cache=InternalUsageCache(local_cache)
+    )
+
+    captured_descriptors = []
+
+    async def mock_should_rate_limit(descriptors, **kwargs):
+        captured_descriptors.extend(descriptors)
+        return {"overall_code": "OK", "statuses": []}
+
+    parallel_request_handler.should_rate_limit = mock_should_rate_limit
+
+    user_api_key_dict = UserAPIKeyAuth(
+        api_key=_api_key,
+        project_id="proj-mantle",
+        project_metadata={
+            "model_itpm_limit": {"bedrock_mantle/claude-opus": 20000000},
+            "model_otpm_limit": {"bedrock_mantle/claude-opus": 4000000},
+        },
+    )
+
+    await parallel_request_handler.async_pre_call_hook(
+        user_api_key_dict=user_api_key_dict,
+        cache=local_cache,
+        data={"model": "bedrock_mantle/claude-opus"},
+        call_type="",
+    )
+
+    descriptor_keys = [d["key"] for d in captured_descriptors]
+    assert "model_per_project_itpm" in descriptor_keys
+    assert "model_per_project_otpm" in descriptor_keys
+    assert "model_per_project" not in descriptor_keys
+
+    itpm_descriptor = next(
+        d for d in captured_descriptors if d["key"] == "model_per_project_itpm"
+    )
+    otpm_descriptor = next(
+        d for d in captured_descriptors if d["key"] == "model_per_project_otpm"
+    )
+    assert itpm_descriptor["value"] == "proj-mantle:bedrock_mantle/claude-opus"
+    assert itpm_descriptor["rate_limit"]["tokens_per_unit"] == 20000000
+    assert otpm_descriptor["value"] == "proj-mantle:bedrock_mantle/claude-opus"
+    assert otpm_descriptor["rate_limit"]["tokens_per_unit"] == 4000000
+
+
+@pytest.mark.asyncio
+async def test_project_model_itpm_otpm_limits_not_triggered_for_other_model_v3():
+    """Split project limits must not apply to an unrelated model."""
+    _api_key = hash_token("sk-project-io-test-2")
+    local_cache = DualCache()
+    parallel_request_handler = _PROXY_MaxParallelRequestsHandler(
+        internal_usage_cache=InternalUsageCache(local_cache)
+    )
+
+    captured_descriptors = []
+
+    async def mock_should_rate_limit(descriptors, **kwargs):
+        captured_descriptors.extend(descriptors)
+        return {"overall_code": "OK", "statuses": []}
+
+    parallel_request_handler.should_rate_limit = mock_should_rate_limit
+
+    user_api_key_dict = UserAPIKeyAuth(
+        api_key=_api_key,
+        project_id="proj-mantle",
+        project_metadata={
+            "model_itpm_limit": {"bedrock_mantle/claude-opus": 20000000},
+        },
+    )
+
+    await parallel_request_handler.async_pre_call_hook(
+        user_api_key_dict=user_api_key_dict,
+        cache=local_cache,
+        data={"model": "gpt-4"},
+        call_type="",
+    )
+
+    descriptor_keys = [d["key"] for d in captured_descriptors]
+    assert "model_per_project_itpm" not in descriptor_keys
+    assert "model_per_project_otpm" not in descriptor_keys
+
+
+@pytest.mark.asyncio
+async def test_project_model_itpm_and_tpm_limits_coexist_v3():
+    """Combined project TPM and split ITPM/OTPM limits are enforced together."""
+    _api_key = hash_token("sk-project-io-test-3")
+    local_cache = DualCache()
+    parallel_request_handler = _PROXY_MaxParallelRequestsHandler(
+        internal_usage_cache=InternalUsageCache(local_cache)
+    )
+
+    captured_descriptors = []
+
+    async def mock_should_rate_limit(descriptors, **kwargs):
+        captured_descriptors.extend(descriptors)
+        return {"overall_code": "OK", "statuses": []}
+
+    parallel_request_handler.should_rate_limit = mock_should_rate_limit
+
+    user_api_key_dict = UserAPIKeyAuth(
+        api_key=_api_key,
+        project_id="proj-mantle",
+        project_metadata={
+            "model_tpm_limit": {"bedrock_mantle/claude-opus": 1000},
+            "model_itpm_limit": {"bedrock_mantle/claude-opus": 20000000},
+            "model_otpm_limit": {"bedrock_mantle/claude-opus": 4000000},
+        },
+    )
+
+    await parallel_request_handler.async_pre_call_hook(
+        user_api_key_dict=user_api_key_dict,
+        cache=local_cache,
+        data={"model": "bedrock_mantle/claude-opus"},
+        call_type="",
+    )
+
+    descriptor_keys = [d["key"] for d in captured_descriptors]
+    assert "model_per_project" in descriptor_keys
+    assert "model_per_project_itpm" in descriptor_keys
+    assert "model_per_project_otpm" in descriptor_keys
+
+
+@pytest.mark.asyncio
+async def test_enforce_project_io_token_quota_for_frame_blocks_over_limit_otpm():
+    """VERIA regression: the Responses WebSocket connection-level pre-call
+    hook only runs once, but a connection accepts many response.create
+    frames. enforce_project_io_token_quota_for_frame is the per-frame check
+    that closes that gap; it must reserve against the caller's project OTPM
+    limit and reject once a frame's estimated output tokens exceed it."""
+    _api_key = hash_token("sk-ws-frame-otpm")
+    local_cache = DualCache()
+    handler = _PROXY_MaxParallelRequestsHandler(
+        internal_usage_cache=InternalUsageCache(local_cache)
+    )
+    user_api_key_dict = UserAPIKeyAuth(
+        api_key=_api_key,
+        project_id="proj-mantle-ws",
+        project_metadata={"model_otpm_limit": {"gpt-4o": 50}},
+    )
+
+    await handler.enforce_project_io_token_quota_for_frame(
+        user_api_key_dict=user_api_key_dict,
+        requested_model="gpt-4o",
+        estimated_input_tokens=1,
+        estimated_output_tokens=30,
+    )
+
+    with pytest.raises(HTTPException) as exc:
+        await handler.enforce_project_io_token_quota_for_frame(
+            user_api_key_dict=user_api_key_dict,
+            requested_model="gpt-4o",
+            estimated_input_tokens=1,
+            estimated_output_tokens=30,
+        )
+
+    assert exc.value.status_code == 429
+    assert "model_per_project_otpm" in str(exc.value.detail)
+
+
+@pytest.mark.asyncio
+async def test_enforce_project_io_token_quota_for_frame_noop_without_project_limits():
+    """A key with no project ITPM/OTPM configured must never be blocked by
+    the per-frame check (no descriptors to reserve against)."""
+    _api_key = hash_token("sk-ws-frame-no-limits")
+    local_cache = DualCache()
+    handler = _PROXY_MaxParallelRequestsHandler(
+        internal_usage_cache=InternalUsageCache(local_cache)
+    )
+    user_api_key_dict = UserAPIKeyAuth(api_key=_api_key)
+
+    await handler.enforce_project_io_token_quota_for_frame(
+        user_api_key_dict=user_api_key_dict,
+        requested_model="gpt-4o",
+        estimated_input_tokens=10_000_000,
+        estimated_output_tokens=10_000_000,
+    )
+
+
 @pytest.mark.asyncio
 async def test_pre_call_hook_keeps_internal_stash_out_of_request_body():
     """Regression for #27001 / #35197: the limiter's per-request bookkeeping
@@ -3192,7 +3378,7 @@ async def test_responses_route_body_untouched_by_pre_call_hook(caller_metadata):
     _api_key = hash_token("sk-responses-regression")
     user_api_key_dict = UserAPIKeyAuth(
         api_key=_api_key,
-        tpm_limit=1000,
+        tpm_limit=100000,
         rpm_limit=5,
     )
     local_cache = DualCache()
@@ -5264,7 +5450,7 @@ async def test_configured_estimate_does_not_apply_to_embeddings(monkeypatch):
         local_cache,
         user_api_key_dict,
         {"model": "text-embedding-3-small", "input": "hello"},
-        call_type="embeddings",
+        call_type="embedding",
     )
 
     assert reserved == ONE_TOKEN_PROMPT_INPUT_ESTIMATE

tests/test_litellm/proxy/management_endpoints/test_access_group_management.py

@@ -13,6 +13,9 @@ sys.path.insert(
 )  # Adds the parent directory to the system path
 
 from litellm import Router
+from litellm.proxy.management_endpoints.model_management_endpoints import (
+    ReconcileOutcome,
+)
 
 
 @pytest.mark.asyncio
@@ -121,7 +124,7 @@ async def test_create_access_group_with_model_ids_tags_only_specific_deployments
         patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
         patch(
             "litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache",
-            new_callable=AsyncMock,
+            new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)),
         ),
     ):
         response = await create_model_group(
@@ -186,7 +189,7 @@ async def test_create_access_group_with_model_names_tags_all_deployments():
         patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
         patch(
             "litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache",
-            new_callable=AsyncMock,
+            new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)),
         ),
     ):
         response = await create_model_group(
@@ -236,7 +239,7 @@ async def test_create_access_group_model_ids_takes_priority_over_model_names():
         patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
         patch(
             "litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache",
-            new_callable=AsyncMock,
+            new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)),
         ),
     ):
         response = await create_model_group(
@@ -313,7 +316,7 @@ async def test_create_access_group_invalid_model_id_returns_400():
         patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
         patch(
             "litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache",
-            new_callable=AsyncMock,
+            new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)),
         ),
     ):
         with pytest.raises(HTTPException) as exc_info:
@@ -352,7 +355,7 @@ async def test_create_access_group_surfaces_dropped_models():
         patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
         patch(
             "litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache",
-            new=AsyncMock(return_value=None),
+            new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)),
         ),
     ):
         with pytest.raises(HTTPException) as exc_info:
@@ -365,6 +368,50 @@ async def test_create_access_group_surfaces_dropped_models():
     assert "deploy-A" in str(exc_info.value.detail)
 
 
+
+@pytest.mark.asyncio
+async def test_create_access_group_trusts_reload_snapshot_over_post_lock_fresh_read():
+    """A concurrent reconcile sampled after the lock is released must not make this
+    write's reload look like it dropped the tagged model: the verdict has to judge from
+    the ReconcileOutcome the reload captured under the lock, not a fresh router read."""
+    from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
+    from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
+        create_model_group,
+    )
+    from litellm.types.proxy.management_endpoints.model_management_endpoints import (
+        NewModelGroupRequest,
+    )
+
+    deploy_a = MagicMock(model_id="deploy-A", model_name="gpt-4o", model_info={})
+
+    mock_prisma = MagicMock()
+    mock_prisma.db.litellm_proxymodeltable.find_many = AsyncMock(return_value=[])
+    mock_prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=deploy_a)
+    mock_prisma.db.litellm_proxymodeltable.update = AsyncMock()
+
+    concurrently_wiped_router = MagicMock()
+    concurrently_wiped_router.get_model_ids.side_effect = [["deploy-A"], []]
+    with (
+        patch("litellm.proxy.proxy_server.llm_router", concurrently_wiped_router),
+        patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
+        patch(
+            "litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache",
+            new=AsyncMock(
+                return_value=ReconcileOutcome(
+                    still_desired=frozenset({"deploy-A"}), live_after=frozenset({"deploy-A"})
+                )
+            ),
+        ),
+    ):
+        response = await create_model_group(
+            data=NewModelGroupRequest(access_group="production-models", model_ids=["deploy-A"]),
+            user_api_key_dict=UserAPIKeyAuth(user_id="test_admin", user_role=LitellmUserRoles.PROXY_ADMIN),
+        )
+
+    assert response.models_updated == 1
+    assert concurrently_wiped_router.get_model_ids.call_count == 1
+
+
 @pytest.mark.asyncio
 async def test_tag_deployment_parses_string_model_info_and_refuses_corrupt():
     """The model_info column can arrive as its JSON string; tagging must parse it rather
@@ -420,7 +467,7 @@ async def test_delete_access_group_ignores_models_that_were_already_dead():
         patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
         patch(
             "litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache",
-            new=AsyncMock(return_value=None),
+            new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)),
         ),
     ):
         response = await delete_access_group(
@@ -430,3 +477,99 @@ async def test_delete_access_group_ignores_models_that_were_already_dead():
 
     assert response.models_updated == 1
     mock_prisma.db.litellm_proxymodeltable.update.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+async def test_create_access_group_read_through_recovers_model_created_on_sibling_replica():
+    """Regression: an access group referencing a model that another replica just wrote
+    to the DB must be created instead of 400ing until the periodic config reload."""
+    from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
+    from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
+        create_model_group,
+    )
+    from litellm.types.proxy.management_endpoints.model_management_endpoints import (
+        NewModelGroupRequest,
+    )
+
+    from types import SimpleNamespace
+
+    model_name = "e2e-ag-sibling-replica-model"
+    db_row = SimpleNamespace(
+        model_id=f"{model_name}-id",
+        model_name=model_name,
+        litellm_params={"model": "openai/gpt-4o", "api_key": "fake", "mock_response": "hi"},
+        model_info={},
+        blocked=False,
+    )
+
+    mock_router = Router(
+        model_list=[
+            {
+                "model_name": "some-other-model",
+                "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"},
+            }
+        ]
+    )
+
+    mock_prisma = MagicMock()
+    mock_prisma.db.litellm_proxymodeltable.find_many = AsyncMock(side_effect=[[db_row], [], [db_row]])
+    mock_prisma.db.litellm_proxymodeltable.update = AsyncMock()
+
+    with (
+        patch("litellm.proxy.proxy_server.llm_router", mock_router),
+        patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
+        patch("litellm.proxy.proxy_server.store_model_in_db", True),
+        patch(
+            "litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache",
+            new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)),
+        ),
+    ):
+        response = await create_model_group(
+            data=NewModelGroupRequest(access_group="replica-lag-group", model_names=[model_name]),
+            user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN),
+        )
+
+    assert response.models_updated == 1
+    assert response.model_names == [model_name]
+    assert mock_prisma.db.litellm_proxymodeltable.find_many.await_args_list[0].kwargs["where"] == {
+        "model_name": model_name
+    }
+
+
+@pytest.mark.asyncio
+async def test_create_access_group_model_missing_everywhere_still_400s():
+    from fastapi import HTTPException
+
+    from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
+    from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
+        create_model_group,
+    )
+    from litellm.types.proxy.management_endpoints.model_management_endpoints import (
+        NewModelGroupRequest,
+    )
+
+    model_name = "e2e-ag-model-nobody-created"
+    mock_router = Router(
+        model_list=[
+            {
+                "model_name": "some-other-model",
+                "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"},
+            }
+        ]
+    )
+    mock_prisma = MagicMock()
+    mock_prisma.db.litellm_proxymodeltable.find_many = AsyncMock(return_value=[])
+
+    with (
+        patch("litellm.proxy.proxy_server.llm_router", mock_router),
+        patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
+        patch("litellm.proxy.proxy_server.store_model_in_db", True),
+    ):
+        with pytest.raises(HTTPException) as exc_info:
+            await create_model_group(
+                data=NewModelGroupRequest(access_group="ghost-group", model_names=[model_name]),
+                user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN),
+            )
+
+    assert exc_info.value.status_code == 400
+    assert model_name in str(exc_info.value.detail)

tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py

@@ -14,6 +14,7 @@ sys.path.insert(0, os.path.abspath("../../../.."))  # Adds the parent directory
 from litellm.proxy.management_endpoints.common_daily_activity import (
     _adjust_dates_for_timezone,
     _build_aggregated_sql_query,
+    _build_entity_rollup_sql_query,
     _is_user_agent_tag,
     _record_to_spend_metrics,
     get_api_key_metadata,
@@ -982,6 +983,58 @@ class TestBuildAggregatedSqlQuery:
         assert "COALESCE(model_group, model)" not in normalized
 
 
+class TestAggregatedEmptyEntityFilter:
+    _BUILDERS: Final = (_build_aggregated_sql_query, _build_entity_rollup_sql_query)
+
+    @pytest.mark.parametrize("build", _BUILDERS)
+    def test_empty_entity_list_emits_no_degenerate_in_clause(self, build):
+        sql, params = build(
+            table_name="litellm_dailyteamspend",
+            entity_id_field="team_id",
+            entity_id=[],
+            start_date="2026-08-01",
+            end_date="2026-08-19",
+            model=None,
+            api_key=None,
+        )
+
+        normalized = " ".join(sql.split())
+        assert "IN ()" not in normalized
+        assert '"team_id" IN' not in normalized
+        assert params == ["2026-08-01", "2026-08-19"]
+
+    @pytest.mark.parametrize("build", _BUILDERS)
+    def test_empty_entity_list_matches_nothing_rather_than_everything(self, build):
+        sql, _ = build(
+            table_name="litellm_dailyteamspend",
+            entity_id_field="team_id",
+            entity_id=[],
+            start_date="2026-08-01",
+            end_date="2026-08-19",
+            model=None,
+            api_key=None,
+        )
+
+        assert "FALSE" in " ".join(sql.split())
+
+    @pytest.mark.parametrize("build", _BUILDERS)
+    def test_populated_entity_list_still_filters_on_its_ids(self, build):
+        sql, params = build(
+            table_name="litellm_dailyteamspend",
+            entity_id_field="team_id",
+            entity_id=["team-alpha", "team-beta"],
+            start_date="2026-08-01",
+            end_date="2026-08-19",
+            model=None,
+            api_key=None,
+        )
+
+        normalized = " ".join(sql.split())
+        assert '"team_id" IN ($3, $4)' in normalized
+        assert "FALSE" not in normalized
+        assert params == ["2026-08-01", "2026-08-19", "team-alpha", "team-beta"]
+
+
 @pytest.mark.asyncio
 async def test_get_daily_activity_aggregated_empty_result_set():
     """Regression test for the empty-range 500.

tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py

@@ -1286,6 +1286,75 @@ class TestOpenAIPassthroughIntegration:
         mock_chat_handler.assert_called_once()
         assert result == {"result": None, "kwargs": {}}
 
+    def test_openai_passthrough_handler_embeddings_unmapped_model_logs_zero_cost(self):
+        response_body = {
+            "object": "list",
+            "model": "lit5787-unmapped-embeddings-deployment",
+            "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}],
+            "usage": {"prompt_tokens": 9, "total_tokens": 9},
+        }
+        mock_logging_obj = self._create_mock_logging_obj()
+        result = OpenAIPassthroughLoggingHandler.openai_passthrough_handler(
+            httpx_response=self._create_mock_httpx_response(response_body),
+            response_body=response_body,
+            logging_obj=mock_logging_obj,
+            url_route="https://my-resource.openai.azure.com/openai/v1/embeddings",
+            result="",
+            start_time=self.start_time,
+            end_time=self.end_time,
+            cache_hit=False,
+            request_body={
+                "model": "lit5787-unmapped-embeddings-deployment",
+                "input": "spend probe",
+            },
+            passthrough_logging_payload=PassthroughStandardLoggingPayload(
+                url="https://my-resource.openai.azure.com/openai/v1/embeddings",
+                request_body={
+                    "model": "lit5787-unmapped-embeddings-deployment",
+                    "input": "spend probe",
+                },
+                request_method="POST",
+            ),
+            litellm_params={},
+        )
+
+        assert result["result"] is not None
+        assert result["result"].usage.prompt_tokens == 9
+        assert result["kwargs"]["response_cost"] == 0.0
+        assert result["kwargs"]["model"] == "lit5787-unmapped-embeddings-deployment"
+        assert result["result"]._hidden_params["response_cost"] == 0.0
+        assert mock_logging_obj.model_call_details["response_cost"] == 0.0
+
+    def test_openai_passthrough_handler_embeddings_error_skips_chat_fallback(self):
+        response_body = {
+            "object": "list",
+            "model": "text-embedding-3-small",
+            "usage": {"prompt_tokens": 9, "total_tokens": 9},
+        }
+        kwargs_in = {
+            "passthrough_logging_payload": PassthroughStandardLoggingPayload(
+                url="https://api.openai.com/v1/embeddings",
+                request_body={"model": "text-embedding-3-small", "input": "spend probe"},
+                request_method="POST",
+            ),
+            "litellm_params": {},
+        }
+        result = OpenAIPassthroughLoggingHandler.openai_passthrough_handler(
+            httpx_response=self._create_mock_httpx_response(response_body),
+            response_body=response_body,
+            logging_obj=self._create_mock_logging_obj(),
+            url_route="https://api.openai.com/v1/embeddings",
+            result="",
+            start_time=self.start_time,
+            end_time=self.end_time,
+            cache_hit=False,
+            request_body={"model": "text-embedding-3-small", "input": "spend probe"},
+            **kwargs_in,
+        )
+
+        assert result["result"] is None
+        assert result["kwargs"]["passthrough_logging_payload"] == kwargs_in["passthrough_logging_payload"]
+
     @patch(
         "litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler.OpenAIPassthroughLoggingHandler.openai_passthrough_handler"
     )

tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py

@@ -1,7 +1,10 @@
+import contextlib
 import json
 import os
 import sys
 import traceback
+from collections.abc import Mapping
+from types import MappingProxyType, SimpleNamespace
 from typing import Final
 from unittest import mock
 from unittest.mock import AsyncMock, MagicMock, Mock, patch
@@ -22,6 +25,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
     _join_url_paths,
     azure_proxy_route,
     bedrock_llm_proxy_route,
+    bedrock_proxy_route,
     create_pass_through_route,
     cursor_proxy_route,
     get_azure_ai_search_index_from_endpoint,
@@ -1730,6 +1734,116 @@ class TestBedrockLLMProxyRoute:
         assert "Blocked by guardrail" in str(exc_info.value.detail)
 
 
+class TestBedrockAgentRuntimePassthroughToggle:
+    AGENT_RUNTIME_ENDPOINT: Final = "knowledgebases/KB1234567/retrieve"
+    MODEL_ENDPOINT: Final = "model/us.anthropic.claude-sonnet-4-5-20250929-v1:0/converse"
+    DISABLED: Final = MappingProxyType({"disable_bedrock_agent_runtime_passthrough": True})
+
+    @staticmethod
+    def _mock_request() -> Mock:
+        request: Final = Mock()
+        request.method = "POST"
+        request.state = SimpleNamespace()
+        request.json = AsyncMock(return_value={"retrievalQuery": {"text": "hi"}})  # mutable-ok: must be json.dumps-able
+        return request
+
+    @contextlib.contextmanager
+    def _patched_dispatch(self, general_settings: Mapping[str, object]):
+        from botocore.credentials import Credentials
+
+        bedrock_llm: Final = Mock()
+        bedrock_llm.get_credentials = Mock(return_value=Credentials("ak", "sk"))
+        forwarder: Final = AsyncMock(return_value="forwarded")
+
+        with (
+            patch("litellm.proxy.proxy_server.general_settings", general_settings),
+            patch("litellm.utils.get_secret", return_value="us-east-1"),
+            patch("litellm.llms.bedrock.chat.BedrockConverseLLM", return_value=bedrock_llm),
+            patch(
+                "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_request_copy",
+                Mock(),
+            ),
+            patch(
+                "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route",
+                return_value=forwarder,
+            ) as create_route,
+        ):
+            yield create_route, forwarder
+
+    @pytest.mark.asyncio
+    async def test_agent_runtime_dispatch_allowed_by_default(self):
+        with self._patched_dispatch(MappingProxyType({})) as (create_route, forwarder):
+            result: Final = await bedrock_proxy_route(
+                endpoint=self.AGENT_RUNTIME_ENDPOINT,
+                request=self._mock_request(),
+                fastapi_response=Mock(),
+                user_api_key_dict=UserAPIKeyAuth(),
+            )
+
+        assert result == "forwarded"
+        forwarder.assert_awaited_once()
+        assert "bedrock-agent-runtime.us-east-1.amazonaws.com" in create_route.call_args.kwargs["target"]
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("value", (True, "true", "True"))
+    async def test_agent_runtime_dispatch_rejected_when_disabled(self, value: bool | str):
+        settings: Final = MappingProxyType({"disable_bedrock_agent_runtime_passthrough": value})
+
+        with self._patched_dispatch(settings) as (create_route, forwarder):
+            with pytest.raises(HTTPException) as exc_info:
+                await bedrock_proxy_route(
+                    endpoint=self.AGENT_RUNTIME_ENDPOINT,
+                    request=self._mock_request(),
+                    fastapi_response=Mock(),
+                    user_api_key_dict=UserAPIKeyAuth(),
+                )
+
+        assert exc_info.value.status_code == 403
+        assert "bedrock-agent-runtime pass-through is disabled" in str(exc_info.value.detail)
+        create_route.assert_not_called()
+        forwarder.assert_not_awaited()
+
+    @pytest.mark.asyncio
+    async def test_model_invoke_still_routed_when_agent_runtime_disabled(self):
+        with (
+            patch("litellm.proxy.proxy_server.general_settings", self.DISABLED),
+            patch("litellm.utils.get_secret", return_value="us-east-1"),
+            patch(
+                "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_request_copy",
+                Mock(),
+            ),
+            patch(
+                "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.bedrock_llm_proxy_route",
+                new=AsyncMock(return_value="llm-route"),
+            ) as llm_route,
+        ):
+            result: Final = await bedrock_proxy_route(
+                endpoint=self.MODEL_ENDPOINT,
+                request=self._mock_request(),
+                fastapi_response=Mock(),
+                user_api_key_dict=UserAPIKeyAuth(),
+            )
+
+        assert result == "llm-route"
+        llm_route.assert_awaited_once()
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("value", (False, "false", None, "", "yes"))
+    async def test_agent_runtime_dispatch_allowed_for_non_true_values(self, value: object):
+        settings: Final = MappingProxyType({"disable_bedrock_agent_runtime_passthrough": value})
+
+        with self._patched_dispatch(settings) as (create_route, forwarder):
+            result: Final = await bedrock_proxy_route(
+                endpoint=self.AGENT_RUNTIME_ENDPOINT,
+                request=self._mock_request(),
+                fastapi_response=Mock(),
+                user_api_key_dict=UserAPIKeyAuth(),
+            )
+
+        assert result == "forwarded"
+        create_route.assert_called_once()
+
+
 class TestLLMPassthroughFactoryProxyRoute:
     @pytest.mark.asyncio
     async def test_llm_passthrough_factory_proxy_route_success(self):

tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py

@@ -124,6 +124,30 @@ def test_get_logging_payload_maps_openai_cache_write_tokens_to_cache_creation_in
     assert additional_usage_values["prompt_tokens_details"]["cache_write_tokens"] == 800
 
 
+def test_get_logging_payload_maps_nested_cache_creation_input_tokens():
+    """
+    Regression (LIT-5757): DashScope nests cache_creation_input_tokens inside
+    prompt_tokens_details; SpendLogs must record it as cache_creation_input_tokens.
+    """
+    additional_usage_values: Final = _get_additional_usage_values_for_usage(
+        litellm.Usage(
+            prompt_tokens=2059,
+            completion_tokens=31,
+            total_tokens=2090,
+            prompt_tokens_details={
+                "cached_tokens": 0,
+                "text_tokens": 2059,
+                "cache_type": "ephemeral",
+                "cache_creation_input_tokens": 2048,
+                "cache_creation": {"ephemeral_5m_input_tokens": 2048},
+            },
+        )
+    )
+
+    assert additional_usage_values["cache_creation_input_tokens"] == 2048
+    assert additional_usage_values["prompt_tokens_details"]["cache_write_tokens"] == 2048
+
+
 def test_get_logging_payload_preserves_anthropic_cache_creation_input_tokens():
     additional_usage_values = _get_additional_usage_values_for_usage(
         litellm.Usage(

tests/test_litellm/proxy/test_proxy_server.py

@@ -11098,3 +11098,100 @@ async def test_moderations_reraises_proxy_exception_unwrapped():
     assert exc_info.value.code == "400"
     assert exc_info.value.param == "metadata"
     mock_logging.post_call_failure_hook.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+async def test_init_agents_in_db_rebuilds_registry_under_agent_reconcile_lock(monkeypatch):
+    from litellm.proxy.agent_endpoints.agent_registry import (
+        AGENT_RECONCILE_LOCK,
+        global_agent_registry,
+    )
+    from litellm.proxy.proxy_server import ProxyConfig
+
+    lock_states: list[bool] = []
+
+    async def fake_get_all_agents_from_db(prisma_client) -> list:
+        lock_states.append(AGENT_RECONCILE_LOCK.locked())
+        return []
+
+    def fake_load_agents_from_db_and_config(db_agents) -> None:
+        lock_states.append(AGENT_RECONCILE_LOCK.locked())
+
+    monkeypatch.setattr(global_agent_registry, "get_all_agents_from_db", fake_get_all_agents_from_db)
+    monkeypatch.setattr(global_agent_registry, "load_agents_from_db_and_config", fake_load_agents_from_db_and_config)
+
+    await ProxyConfig()._init_agents_in_db(prisma_client=MagicMock())
+
+    assert lock_states == [True, True]
+    assert not AGENT_RECONCILE_LOCK.locked()
+
+
+@pytest.mark.asyncio
+async def test_init_guardrails_in_db_snapshots_and_reconciles_under_guardrail_reconcile_lock(monkeypatch):
+    from litellm.proxy.guardrails.guardrail_registry import (
+        GUARDRAIL_RECONCILE_LOCK,
+        IN_MEMORY_GUARDRAIL_HANDLER,
+        GuardrailRegistry,
+    )
+    from litellm.proxy.proxy_server import ProxyConfig
+
+    lock_states: list[bool] = []
+
+    async def fake_get_all_guardrails_from_db(prisma_client) -> list:
+        lock_states.append(GUARDRAIL_RECONCILE_LOCK.locked())
+        return []
+
+    def fake_reconcile_db_guardrails(db_guardrail_ids) -> list:
+        lock_states.append(GUARDRAIL_RECONCILE_LOCK.locked())
+        return []
+
+    monkeypatch.setattr(GuardrailRegistry, "get_all_guardrails_from_db", fake_get_all_guardrails_from_db)
+    monkeypatch.setattr(IN_MEMORY_GUARDRAIL_HANDLER, "reconcile_db_guardrails", fake_reconcile_db_guardrails)
+
+    await ProxyConfig()._init_guardrails_in_db(prisma_client=MagicMock())
+
+    assert lock_states == [True, True]
+    assert not GUARDRAIL_RECONCILE_LOCK.locked()
+
+
+class TestEmbeddingsFailureHookRequestData:
+    @pytest.mark.asyncio
+    async def test_failure_hook_gets_post_setup_data_with_logging_obj(self):
+        """Request setup replaces the processor's data dict (adding the logging
+        object the failure hook needs to lift token usage from); the embeddings
+        exception handler must pass that replaced dict, not the raw request body
+        dict it was rebuilt from."""
+        from litellm.proxy._types import ProxyException
+
+        captured = {}
+        logging_obj_sentinel = MagicMock()
+
+        async def fake_process(self, **kwargs):
+            self.data = {**self.data, "litellm_logging_obj": logging_obj_sentinel}
+            captured["processor_data"] = self.data
+            raise RuntimeError("provider timeout")
+
+        with (
+            patch.object(
+                proxy_server_module,
+                "_read_request_body",
+                new=AsyncMock(return_value={"model": "my-embed", "input": "hello"}),
+            ),
+            patch.object(
+                proxy_server_module.ProxyBaseLLMRequestProcessing,
+                "base_process_llm_request",
+                new=fake_process,
+            ),
+            patch.object(proxy_server_module, "proxy_logging_obj") as mock_logging,
+        ):
+            mock_logging.post_call_failure_hook = AsyncMock(return_value=None)
+            with pytest.raises(ProxyException):
+                await proxy_server_module.embeddings(
+                    request=MagicMock(),
+                    fastapi_response=MagicMock(),
+                    user_api_key_dict=UserAPIKeyAuth(),
+                )
+
+        hook_request_data = mock_logging.post_call_failure_hook.await_args.kwargs["request_data"]
+        assert hook_request_data is captured["processor_data"]
+        assert hook_request_data["litellm_logging_obj"] is logging_obj_sentinel