diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json
index 5607a170e33..a7ec31f2ffd 100644
--- a/basedpyright-code-budget.json
+++ b/basedpyright-code-budget.json
@@ -57,7 +57,7 @@
"limit": 5681
},
"reportMissingTypeArgument": {
- "limit": 15608
+ "limit": 15605
},
"reportMissingTypeStubs": {
"limit": 40
@@ -108,7 +108,7 @@
"limit": 39154
},
"reportUnknownParameterType": {
- "limit": 19947
+ "limit": 19944
},
"reportUnknownVariableType": {
"limit": 30772
@@ -132,7 +132,7 @@
"limit": 27
},
"reportUnusedClass": {
- "limit": 23
+ "limit": 21
},
"reportUnusedFunction": {
"limit": 139
diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml
index 7a8031216e0..bb580c82760 100644
--- a/enterprise/pyproject.toml
+++ b/enterprise/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
-version = "0.1.56"
+version = "0.1.57"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
-version = "0.1.56"
+version = "0.1.57"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",
diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml
index e39f0dcf55a..e1d62b70c29 100644
--- a/litellm-proxy-extras/pyproject.toml
+++ b/litellm-proxy-extras/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
-version = "0.4.86"
+version = "0.4.87"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
-version = "0.4.86"
+version = "0.4.87"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",
diff --git a/litellm/_logging.py b/litellm/_logging.py
index 6add9d79a5b..7d3a30c6d1a 100644
--- a/litellm/_logging.py
+++ b/litellm/_logging.py
@@ -10,7 +10,7 @@ from typing import Any, Final
import litellm
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
-from litellm.litellm_core_utils.secret_redaction import redact_string
+from litellm.litellm_core_utils.secret_redaction import redact_string, redact_structured_value
set_verbose = False
@@ -59,6 +59,12 @@ def _redact_string(value: str) -> str:
return redact_string(value)
+def _redact_structured_value(key: str | None, value: str) -> str:
+ if not _ENABLE_SECRET_REDACTION:
+ return value
+ return redact_structured_value(key, value)
+
+
def redact_secrets(value: str) -> str:
"""Public API: redact known secret/credential patterns from an arbitrary string.
@@ -265,7 +271,7 @@ class JsonFormatter(Formatter):
if record.exc_info:
json_record["stacktrace"] = record.exc_text or self.formatException(record.exc_info)
- return safe_dumps(json_record)
+ return safe_dumps(json_record, value_transform=_redact_structured_value)
class CorrelationPlainFormatter(logging.Formatter):
@@ -276,7 +282,7 @@ class CorrelationPlainFormatter(logging.Formatter):
"""
def format(self, record: logging.LogRecord) -> str:
- formatted: Final = super().format(record)
+ formatted: Final = _redact_string(super().format(record))
trace_id: Final = getattr(record, "trace_id", None)
session_id: Final = getattr(record, "session_id", None)
if not trace_id and not session_id:
diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py
index 9363db12385..f2bd18cd046 100644
--- a/litellm/integrations/opentelemetry.py
+++ b/litellm/integrations/opentelemetry.py
@@ -20,6 +20,7 @@ from litellm.integrations.opentelemetry_utils.gen_ai_semconv import (
OTELSemconvCategory,
parse_semconv_opt_in,
)
+from litellm.integrations.otel.model.db_endpoint import db_span_attributes
from litellm.integrations.otel.model.semconv import Metric
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.secret_redaction import redact_string
@@ -718,6 +719,28 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
self._handle_failure(kwargs, response_obj, start_time, end_time)
+ def _start_service_span(self, payload: ServiceLoggerPayload, parent_otel_span: Span, start_time_ns: int) -> Span:
+ """Open a service span, named and classified by what the service is.
+
+ A datastore call is an outbound CLIENT span carrying ``db.*`` semconv.
+ Without those a Postgres span says only ``service=postgres``, so the
+ backend falls back to the transport peer, which for Prisma is the local
+ query engine on loopback. Everything else stays an INTERNAL span.
+ """
+ from opentelemetry import trace
+ from opentelemetry.trace import SpanKind
+
+ attributes: Final = db_span_attributes(payload.service.value, payload.call_type)
+ span: Final = self.tracer.start_span(
+ name=payload.service,
+ context=trace.set_span_in_context(parent_otel_span),
+ start_time=start_time_ns,
+ kind=SpanKind.CLIENT if attributes else SpanKind.INTERNAL,
+ )
+ for key, value in attributes.items():
+ self.safe_set_attribute(span=span, key=key, value=value)
+ return span
+
async def async_service_success_hook(
self,
payload: ServiceLoggerPayload,
@@ -726,7 +749,6 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
end_time: datetime | float | None = None,
event_metadata: dict | None = None,
):
- from opentelemetry import trace
from opentelemetry.trace import Status, StatusCode
_start_time_ns = 0
@@ -743,12 +765,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
_end_time_ns = self._to_ns(end_time)
if parent_otel_span is not None:
- _span_name: Final = payload.service
- service_logging_span: Final = self.tracer.start_span(
- name=_span_name,
- context=trace.set_span_in_context(parent_otel_span),
- start_time=_start_time_ns,
- )
+ service_logging_span: Final = self._start_service_span(payload, parent_otel_span, _start_time_ns)
self.safe_set_attribute(
span=service_logging_span,
key="call_type",
@@ -786,7 +803,6 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
end_time: float | datetime | None = None,
event_metadata: dict | None = None,
):
- from opentelemetry import trace
from opentelemetry.trace import Status, StatusCode
_start_time_ns = 0
@@ -803,12 +819,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
_end_time_ns = self._to_ns(end_time)
if parent_otel_span is not None:
- _span_name: Final = payload.service
- service_logging_span: Final = self.tracer.start_span(
- name=_span_name,
- context=trace.set_span_in_context(parent_otel_span),
- start_time=_start_time_ns,
- )
+ service_logging_span: Final = self._start_service_span(payload, parent_otel_span, _start_time_ns)
self.safe_set_attribute(
span=service_logging_span,
key="call_type",
diff --git a/litellm/integrations/otel/mappers/genai.py b/litellm/integrations/otel/mappers/genai.py
index 032441535e0..79487e69ac4 100644
--- a/litellm/integrations/otel/mappers/genai.py
+++ b/litellm/integrations/otel/mappers/genai.py
@@ -18,6 +18,7 @@ from litellm.integrations.otel.mappers.utils import (
serialize_messages,
tool_definition_attrs,
)
+from litellm.integrations.otel.model.db_endpoint import db_span_attributes
from litellm.integrations.otel.model.payloads import (
GuardrailSpanData,
LLMCallSpanData,
@@ -27,7 +28,6 @@ from litellm.integrations.otel.model.payloads import (
ToolDefinition,
)
from litellm.integrations.otel.model.semconv import (
- DB,
MCP,
Error,
GenAI,
@@ -36,7 +36,6 @@ from litellm.integrations.otel.model.semconv import (
RpcSystem,
Server,
)
-from litellm.integrations.otel.model.spans import db_system
class GenAIMapper:
@@ -182,12 +181,8 @@ class GenAIMapper:
def _service(cls, data: ServiceSpanData) -> AttributeMap:
attrs: Final = collect(cls._SERVICE_ATTRS, data)
# An outbound datastore call (DB_CALL / CLIENT span) also carries db.*
- # semconv. Internal services (router, budget jobs, …) have no db.system,
- # so they get only the litellm.service.* keys above.
- system: Final = db_system(data.service_name)
- if system is not None:
- attrs[DB.SYSTEM_NAME] = system
- if data.call_type:
- attrs[DB.OPERATION_NAME] = data.call_type
+ # semconv naming the server it reached. Internal services (router, budget
+ # jobs, …) have no db.system, so they get only the litellm.service.* keys.
+ attrs.update(db_span_attributes(data.service_name, data.call_type))
attrs.update({f"{LiteLLM.METADATA_PREFIX}{key}": value for key, value in data.event_metadata.items()})
return attrs
diff --git a/litellm/integrations/otel/model/db_endpoint.py b/litellm/integrations/otel/model/db_endpoint.py
new file mode 100644
index 00000000000..562162a8f31
--- /dev/null
+++ b/litellm/integrations/otel/model/db_endpoint.py
@@ -0,0 +1,164 @@
+"""OTel ``db.*`` / ``server.*`` attributes naming the database litellm talks to.
+
+Prisma reaches PostgreSQL through a query engine listening on loopback, so
+transport-level instrumentation attributes the work to ``localhost`` and an
+operator cannot tell it is a PostgreSQL call or correlate it with the database's
+own metrics. These attributes name the real server on litellm's DB spans.
+
+Only the host, port, database and schema of the DSN are read, so no credential
+can reach an exporter.
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import Final
+from urllib.parse import ParseResult, parse_qs, unquote, urlparse
+
+from litellm.integrations.otel.model.semconv import DB, Server
+from litellm.integrations.otel.model.spans import POSTGRESQL, db_system
+
+_DATABASE_URL_ENV: Final = "DATABASE_URL"
+_READ_REPLICA_ENV: Final = "DATABASE_URL_READ_REPLICA"
+_DEFAULT_POSTGRES_PORT: Final = 5432
+_DEFAULT_POSTGRES_SCHEMA: Final = "public"
+_POSTGRES_SCHEMES: Final = frozenset({"postgres", "postgresql"})
+_EMPTY_ATTRIBUTES: Final[Mapping[str, str | int]] = MappingProxyType({})
+
+
+@dataclass(frozen=True, slots=True)
+class DatabaseEndpoint:
+ """The non-sensitive identity of a PostgreSQL server, parsed from a DSN."""
+
+ address: str | None
+ port: int | None
+ namespace: str | None
+
+
+def parse_database_endpoint(url: str | None) -> DatabaseEndpoint | None:
+ """Parse a PostgreSQL DSN into its exportable endpoint identity.
+
+ Returns ``None`` for an absent, malformed or non-PostgreSQL URL rather than
+ raising: an unparseable DSN must degrade to a span without endpoint
+ attributes, never break the request that emitted it.
+ """
+ if not url:
+ return None
+ try:
+ parsed: Final = urlparse(url)
+ if parsed.scheme not in _POSTGRES_SCHEMES:
+ return None
+ query: Final = parse_qs(parsed.query)
+ raw_database: Final = (parsed.path or "").lstrip("/")
+ if _is_misparsed_authority(parsed, url, raw_database):
+ return None
+ # ``host=`` beats the netloc: it is how libpq names a Unix socket
+ # directory and how the Cloud SQL connector sits behind a localhost
+ # netloc, where the netloc is the very answer this module replaces.
+ address: Final = _first(query.get("host")) or parsed.hostname
+ # ``port=`` accompanies ``host=`` in a libpq URI, so honour it the same way.
+ port: Final = _port(_first(query.get("port")), parsed.port) if address else None
+ namespace: Final = _namespace(unquote(raw_database), _first(query.get("schema")))
+ except ValueError:
+ return None
+ if address is None and namespace is None:
+ return None
+ return DatabaseEndpoint(address=address, port=port, namespace=namespace)
+
+
+def _is_misparsed_authority(parsed: ParseResult, url: str, raw_database: str) -> bool:
+ """Whether the URL authority may have been truncated by an unencoded character.
+
+ ``/``, ``#`` or ``?`` in a password ends the netloc early, so urlparse hands
+ back the username as the host, the leading digits of the password as the
+ port, and the rest of the credential as the path, query or fragment. The
+ stranded userinfo ``@`` is the only surviving evidence.
+
+ A database name cannot hold an unencoded slash either, so a second path
+ segment is the same evidence.
+
+ A DSN that carries the at-sign in a query parameter instead, such as
+ ``?application_name=svc@prod``, is indistinguishable from a mis-split by any
+ property of the parse: both leave no userinfo, a host, a port and a path.
+ Since guessing wrong publishes a credential fragment to a tracing backend,
+ that ambiguity resolves to refusing the endpoint. Such a DSN loses
+ ``server.address`` and ``db.namespace`` and keeps the rest of the span,
+ which is the cheaper error of the two. Percent-encode the at-sign to keep
+ them.
+ """
+ if "/" in raw_database:
+ return True
+ return "@" in url and "@" not in parsed.netloc
+
+
+def _first(values: Sequence[str] | None) -> str:
+ return values[0] if values else ""
+
+
+def _port(from_query: str, from_netloc: int | None) -> int:
+ return int(from_query) if from_query.isdigit() else (from_netloc or _DEFAULT_POSTGRES_PORT)
+
+
+def _namespace(database: str, schema: str) -> str | None:
+ """``{database}|{schema}`` per the PostgreSQL semconv, dropping absent halves.
+
+ Only Prisma's literal default schema stays implicit. The match is
+ case-sensitive because Prisma quotes the name, so ``?schema=PUBLIC`` builds
+ a second schema alongside ``public`` and the two must not collapse to one
+ namespace.
+ """
+ qualifier: Final = "" if schema == _DEFAULT_POSTGRES_SCHEMA else schema
+ return "|".join(part for part in (database, qualifier) if part) or None
+
+
+def postgres_endpoint() -> DatabaseEndpoint | None:
+ """The PostgreSQL endpoint the process is currently connected to.
+
+ Read from ``os.environ`` on every span, deliberately, on both counts.
+
+ The environment is what Prisma itself connects with, so the span cannot
+ disagree with the connection; ``get_secret_str`` would consult a configured
+ secret manager first and could name a different server than the one serving
+ the query. And the value is not static: the RDS IAM refresh rebuilds the URL
+ from ``DATABASE_HOST``/``PORT``/``NAME``/``SCHEMA`` every rotation, the
+ reconnect path re-reads ``DATABASE_URL``, and the DB-backed
+ ``environment_variables`` config overlay can rewrite any of them after
+ startup, so a value cached for the process lifetime goes stale against a
+ connection that has genuinely moved. Nothing is memoized either: a cache
+ keyed on the URL would hold a rotated credential past its rotation, and the
+ parse is a single ``urlparse`` on a short string.
+
+ A configured read replica yields ``None``: ``RoutingPrismaWrapper`` picks
+ reader or writer per Prisma call, underneath the span, so naming the writer
+ would attribute replica reads to the primary.
+ """
+ if os.environ.get(_READ_REPLICA_ENV):
+ return None
+ return parse_database_endpoint(os.environ.get(_DATABASE_URL_ENV, ""))
+
+
+def db_span_attributes(service_name: str, call_type: str | None = None) -> Mapping[str, str | int]:
+ """The ``db.*``/``server.*`` attributes for a datastore service call.
+
+ Empty for services that are not outbound datastore calls. Endpoint
+ attributes are PostgreSQL-only: ``DATABASE_URL`` says nothing about where
+ the redis-backed services point. ``db.system`` rides alongside the current
+ ``db.system.name`` because Datadog's OTLP intake still types a database span
+ from the older key.
+ """
+ system: Final = db_system(service_name)
+ if system is None:
+ return _EMPTY_ATTRIBUTES
+ endpoint: Final = postgres_endpoint() if system == POSTGRESQL else None
+ pairs: Final[tuple[tuple[str, str | int | None], ...]] = (
+ (DB.SYSTEM_NAME, system),
+ (DB.SYSTEM_LEGACY, system),
+ (DB.OPERATION_NAME, call_type),
+ (Server.ADDRESS, endpoint.address if endpoint is not None else None),
+ (Server.PORT, endpoint.port if endpoint is not None else None),
+ (DB.NAMESPACE, endpoint.namespace if endpoint is not None else None),
+ )
+ return MappingProxyType({key: value for key, value in pairs if value})
diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py
index 3d585c36b67..ada2822ba66 100644
--- a/litellm/integrations/otel/model/semconv.py
+++ b/litellm/integrations/otel/model/semconv.py
@@ -238,7 +238,11 @@ class DB:
"""
SYSTEM_NAME: Final = "db.system.name"
+ # Superseded by SYSTEM_NAME, dual-emitted because Datadog's OTLP intake
+ # still infers a span's database type from this key.
+ SYSTEM_LEGACY: Final = "db.system"
OPERATION_NAME: Final = "db.operation.name"
+ NAMESPACE: Final = "db.namespace"
class HTTP:
diff --git a/litellm/integrations/otel/model/spans.py b/litellm/integrations/otel/model/spans.py
index 0f67f0e7a7c..08318f78b7c 100644
--- a/litellm/integrations/otel/model/spans.py
+++ b/litellm/integrations/otel/model/spans.py
@@ -115,10 +115,12 @@ SPAN_REGISTRY: Final[dict[SpanRole, SpanSpec]] = {
# redis-backed spend queues. Any service not mapped here is litellm-internal work
# and stays an INTERNAL ``SERVICE`` span. This table is the single source of
# datastore knowledge — both the role classifier and the mapper read it.
+POSTGRESQL: Final = "postgresql"
+
_DB_SYSTEM_BY_SERVICE: Final[dict[str, str]] = {
"redis": "redis",
- "postgres": "postgresql",
- "batch_write_to_db": "postgresql",
+ "postgres": POSTGRESQL,
+ "batch_write_to_db": POSTGRESQL,
}
diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py
index 58300d87a5a..b20c75bbc2a 100644
--- a/litellm/litellm_core_utils/litellm_logging.py
+++ b/litellm/litellm_core_utils/litellm_logging.py
@@ -1094,10 +1094,10 @@ class Logging(LiteLLMLoggingBaseClass):
data=additional_args.get("complete_input_dict", {}),
)
- _metadata["raw_request"] = str(curl_command)
+ _metadata["raw_request"] = _redact_string(str(curl_command))
# split up, so it's easier to parse in the UI
self.model_call_details["raw_request_typed_dict"] = RawRequestTypedDict(
- raw_request_api_base=str(additional_args.get("api_base") or ""),
+ raw_request_api_base=self._get_masked_api_base(str(additional_args.get("api_base") or "")),
raw_request_body=self._get_raw_request_body(additional_args.get("complete_input_dict", {})),
# NOTE: setting ignore_sensitive_headers to True will cause
# the Authorization header to be leaked when calls to the health
@@ -1111,8 +1111,10 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["raw_request_typed_dict"] = RawRequestTypedDict(
error=str(e),
)
- _metadata["raw_request"] = f"Unable to Log \
+ _metadata["raw_request"] = _redact_string(
+ f"Unable to Log \
raw request: {e}"
+ )
if getattr(self, "logger_fn", None) and callable(self.logger_fn):
try:
self.logger_fn(
@@ -1206,15 +1208,16 @@ class Logging(LiteLLMLoggingBaseClass):
if _is_debugging_on() or self.litellm_request_debug:
if json_logs:
masked_headers: Final = self._get_masked_headers(headers)
+ masked_api_base: Final = self._get_masked_api_base(str(api_base or ""))
if self.litellm_request_debug:
verbose_logger.warning( # .warning ensures this shows up in all environments
"POST Request Sent from LiteLLM",
- extra={"api_base": {api_base}, **masked_headers},
+ extra={"api_base": {masked_api_base}, **masked_headers},
)
else:
verbose_logger.debug(
"POST Request Sent from LiteLLM",
- extra={"api_base": {api_base}, **masked_headers},
+ extra={"api_base": {masked_api_base}, **masked_headers},
)
else:
headers = additional_args.get("headers", {})
@@ -1254,8 +1257,6 @@ class Logging(LiteLLMLoggingBaseClass):
curl_command = "\nRequest Sent from LiteLLM:\n"
request_str: Final = additional_args.get("request_str", "")
curl_command += request_str
- elif api_base == "":
- curl_command = str(self.model_call_details)
return curl_command
def _get_masked_headers(self, headers: dict, ignore_sensitive_headers: bool = False) -> dict:
diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py
index 07d5e6314dd..e4c1c9fc5cf 100644
--- a/litellm/litellm_core_utils/prompt_templates/common_utils.py
+++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py
@@ -1816,16 +1816,19 @@ def split_concatenated_json_objects(raw: str) -> list[dict[str, Any]]:
This helper uses ``json.JSONDecoder.raw_decode()`` to walk the string
and extract each JSON object individually.
+ The walk degrades gracefully: if the string is malformed or truncated
+ (e.g. a stream that ended mid-tool-call), whatever complete objects were
+ parsed before the bad tail are returned and the remainder is discarded
+ with a warning, rather than raising. The sole caller
+ (``_convert_to_bedrock_tool_call_invoke``) treats an empty result as
+ ``input={}`` so the conversation can continue instead of hard-failing.
+
Returns
-------
list[dict]
A list of parsed dicts – one per JSON object found. If *raw* is
- empty or whitespace-only, an empty list is returned.
-
- Raises
- ------
- json.JSONDecodeError
- If the string contains text that cannot be parsed as JSON at all.
+ empty, whitespace-only, or wholly unparseable, an empty list is
+ returned.
"""
import json
@@ -1845,7 +1848,17 @@ def split_concatenated_json_objects(raw: str) -> list[dict[str, Any]]:
if idx >= length:
break
- obj, end_idx = decoder.raw_decode(raw, idx)
+ try:
+ obj, end_idx = decoder.raw_decode(raw, idx)
+ except json.JSONDecodeError as e:
+ verbose_logger.warning(
+ "split_concatenated_json_objects: discarding unparseable tool-call "
+ "arguments tail after %d complete object(s); decode_start=%d error=%s",
+ len(results),
+ idx,
+ e,
+ )
+ break
if isinstance(obj, dict):
results.append(obj)
else:
diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py
index 2ffe015c727..0ed15c43ccf 100644
--- a/litellm/litellm_core_utils/prompt_templates/factory.py
+++ b/litellm/litellm_core_utils/prompt_templates/factory.py
@@ -3712,7 +3712,13 @@ def _convert_to_bedrock_tool_call_invoke(
_parts_list.append(cache_point_block)
return _parts_list
except Exception as e:
- raise Exception(f"Unable to convert openai tool calls={tool_calls} to bedrock tool calls. Received error={e}")
+ tool_call_ids: Final = tuple(tool.get("id") for tool in tool_calls if isinstance(tool, dict))
+ raise litellm.BadRequestError(
+ message=f"Unable to convert openai tool calls with ids={tool_call_ids} to bedrock tool calls. "
+ f"Received error={e}",
+ model=model or "",
+ llm_provider="bedrock",
+ ) from e
def _append_bedrock_tool_result_media_block(
diff --git a/litellm/litellm_core_utils/safe_json_dumps.py b/litellm/litellm_core_utils/safe_json_dumps.py
index ebf45ed747c..a1b71593dda 100644
--- a/litellm/litellm_core_utils/safe_json_dumps.py
+++ b/litellm/litellm_core_utils/safe_json_dumps.py
@@ -1,4 +1,5 @@
import json
+from collections.abc import Callable
from typing import Any, Final
from pydantic import BaseModel
@@ -11,20 +12,32 @@ def strip_null_bytes(value: str) -> str:
return value.replace("\x00", "")
-def safe_dumps(data: Any, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH) -> str:
+def safe_dumps(
+ data: Any,
+ max_depth: int = DEFAULT_MAX_RECURSE_DEPTH,
+ value_transform: Callable[[str | None, str], str] | None = None,
+) -> str:
"""
Recursively serialize data while detecting circular references.
If a circular reference is detected then a marker string is returned.
NUL bytes are stripped from strings to prevent PostgreSQL 22P05 errors.
+
+ value_transform, when given, is applied to every string leaf (and to the
+ str() fallback for non-serializable objects) with the mapping key the leaf
+ was reached under, so callers can rewrite values without touching structure.
"""
- def _serialize(obj: Any, seen: set, depth: int) -> Any:
+ def _transform(key: str | None, value: str) -> str:
+ return value if value_transform is None else value_transform(key, value)
+
+ def _serialize(obj: Any, seen: set, depth: int, key: str | None = None) -> Any:
# Check for maximum depth.
if depth > max_depth:
return "MaxDepthExceeded"
# Base-case: if it is a primitive, simply return it.
if isinstance(obj, str):
- return obj.replace("\x00", "") if "\x00" in obj else obj
+ cleaned = obj.replace("\x00", "") if "\x00" in obj else obj
+ return _transform(key, cleaned)
if isinstance(obj, (int, float, bool, type(None))):
return obj
# Check for circular reference.
@@ -37,30 +50,30 @@ def safe_dumps(data: Any, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH) -> str:
for k, v in obj.items():
if isinstance(k, (str)):
clean_k = k.replace("\x00", "") if "\x00" in k else k
- result[clean_k] = _serialize(v, seen, depth + 1)
+ result[clean_k] = _serialize(v, seen, depth + 1, clean_k)
seen.remove(id(obj))
return result
elif isinstance(obj, list):
- result = [_serialize(item, seen, depth + 1) for item in obj]
+ result = [_serialize(item, seen, depth + 1, key) for item in obj]
seen.remove(id(obj))
return result
elif isinstance(obj, tuple):
- result = tuple(_serialize(item, seen, depth + 1) for item in obj)
+ result = tuple(_serialize(item, seen, depth + 1, key) for item in obj)
seen.remove(id(obj))
return result
elif isinstance(obj, set):
- result = sorted([_serialize(item, seen, depth + 1) for item in obj])
+ result = sorted([_serialize(item, seen, depth + 1, key) for item in obj])
seen.remove(id(obj))
return result
elif isinstance(obj, BaseModel):
dumped: Final = obj.model_dump()
- result = _serialize(dumped, seen, depth + 1)
+ result = _serialize(dumped, seen, depth + 1, key)
seen.remove(id(obj))
return result
else:
# Fall back to string conversion for non-serializable objects.
try:
- return strip_null_bytes(str(obj))
+ return _transform(key, strip_null_bytes(str(obj)))
except Exception:
return "Unserializable Object"
diff --git a/litellm/litellm_core_utils/secret_redaction.py b/litellm/litellm_core_utils/secret_redaction.py
index c991a953530..5d5bd547d22 100644
--- a/litellm/litellm_core_utils/secret_redaction.py
+++ b/litellm/litellm_core_utils/secret_redaction.py
@@ -24,9 +24,6 @@ def _build_secret_patterns() -> "re.Pattern[str]":
r"(?:client_secret|azure_password|azure_username)\s+[^\s,'\"})\]{}>]+",
# AWS access key IDs
r"(?:AKIA|ASIA)[0-9A-Z]{16}",
- # AWS secrets / session tokens / access key IDs (key=value)
- r"(?:aws_secret_access_key|aws_session_token|aws_access_key_id)"
- r"\s*[:=]\s*[A-Za-z0-9/+=]{20,}",
# Bearer tokens (OAuth, JWT, etc.)
r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*",
# Basic auth headers
@@ -61,6 +58,7 @@ def _build_secret_patterns() -> "re.Pattern[str]":
# private_key with PEM-aware value capture
r"""private_key['\"]?\s*[:=]\s*['\"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'\"})\]{}>]+)""",
r"(?:master_key|xai_key|database_url|db_url|connection_string|"
+ r"aws_secret_access_key|aws_session_token|aws_access_key_id|"
r"signing_key|encryption_key|"
r"auth_token|access_token|refresh_token|"
r"slack_webhook_url|webhook_url|"
@@ -83,3 +81,19 @@ _SECRET_RE: Final = _build_secret_patterns()
def redact_string(value: str) -> str:
"""Scrub known secret/credential patterns from *value* and return the result."""
return _SECRET_RE.sub(_REDACTED, value)
+
+
+def redact_structured_value(key: str | None, value: str) -> str:
+ """Scrub *value* as it appeared under *key* inside a structured record.
+
+ redact_string() replaces a whole ``key: value`` span with REDACTED, which is
+ fine inside free text but destroys the surrounding syntax when the span is a
+ JSON member rather than message content. This renders the pair the way a dict
+ repr would, so the key-name patterns still fire, but collapses only the value
+ so the caller's structure survives.
+ """
+ scrubbed: Final = redact_string(value)
+ if scrubbed != value or key is None:
+ return scrubbed
+ rendered: Final = f"'{key}': '{value}'"
+ return _REDACTED if redact_string(rendered) != rendered else value
diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py
index 0d6d942e686..d147063df73 100644
--- a/litellm/llms/base_llm/chat/transformation.py
+++ b/litellm/llms/base_llm/chat/transformation.py
@@ -5,7 +5,7 @@ Common base config for all LLM providers
import types
from abc import ABC, abstractmethod
from collections.abc import AsyncIterator, Iterator
-from typing import TYPE_CHECKING, Any, Final, Union, cast
+from typing import TYPE_CHECKING, Any, Final, Union
import httpx
from pydantic import BaseModel
@@ -90,9 +90,9 @@ class BaseConfig(ABC):
return type_to_response_format_param(response_format=response_format)
def is_thinking_enabled(self, non_default_params: dict) -> bool:
- return (non_default_params.get("thinking") or {}).get("type") == "enabled" or non_default_params.get(
- "reasoning_effort"
- ) is not None
+ thinking: Final = non_default_params.get("thinking")
+ thinking_type: Final = thinking.get("type") if isinstance(thinking, dict) else None
+ return thinking is True or thinking_type == "enabled" or non_default_params.get("reasoning_effort") is not None
def is_max_tokens_in_request(self, non_default_params: dict) -> bool:
"""
@@ -112,7 +112,10 @@ class BaseConfig(ABC):
if is_thinking_enabled and (
"max_tokens" not in non_default_params and "max_completion_tokens" not in non_default_params
):
- thinking_token_budget: Final = cast(dict, optional_params["thinking"]).get("budget_tokens", None)
+ thinking_value: Final = optional_params.get("thinking")
+ thinking_token_budget: Final = (
+ thinking_value.get("budget_tokens") if isinstance(thinking_value, dict) else None
+ )
if thinking_token_budget is not None:
optional_params["max_tokens"] = thinking_token_budget + DEFAULT_MAX_TOKENS
diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py
index fd07999395b..cee89f42c2d 100644
--- a/litellm/llms/bedrock/chat/converse_transformation.py
+++ b/litellm/llms/bedrock/chat/converse_transformation.py
@@ -1090,7 +1090,10 @@ class AmazonConverseConfig(BaseConfig):
is_thinking_enabled: Final = self.is_thinking_enabled(optional_params)
is_max_tokens_in_request: Final = self.is_max_tokens_in_request(non_default_params)
if is_thinking_enabled and not is_max_tokens_in_request:
- thinking_token_budget: Final = cast(dict, optional_params["thinking"]).get("budget_tokens", None)
+ thinking_value: Final = optional_params.get("thinking")
+ thinking_token_budget: Final = (
+ thinking_value.get("budget_tokens") if isinstance(thinking_value, dict) else None
+ )
if thinking_token_budget is not None:
optional_params["maxTokens"] = thinking_token_budget + DEFAULT_MAX_TOKENS
diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py
index eccc783dd8b..3ee803646a9 100644
--- a/litellm/llms/custom_httpx/llm_http_handler.py
+++ b/litellm/llms/custom_httpx/llm_http_handler.py
@@ -2,7 +2,7 @@ import asyncio
import json
import os
import ssl
-from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping
+from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
from contextlib import asynccontextmanager
from functools import lru_cache
from types import ModuleType
@@ -73,6 +73,7 @@ from litellm.llms.custom_httpx.http_handler import (
from litellm.responses.streaming_iterator import (
BaseResponsesAPIStreamingIterator,
MockResponsesAPIStreamingIterator,
+ ProjectQuotaCallback,
ResponsesAPIStreamingIterator,
ResponsesWebSocketStreaming,
SyncResponsesAPIStreamingIterator,
@@ -256,6 +257,27 @@ def _has_pre_call_deployment_hook(logging_obj: LiteLLMLoggingObj) -> bool:
return False
+def _collect_ws_project_quota_callbacks() -> tuple[ProjectQuotaCallback, ...]:
+ """Duck-type discover proxy hooks exposing per-frame project ITPM/OTPM
+ enforcement, so the Responses WebSocket loop can charge every
+ ``response.create`` frame, not just the connection's first one.
+
+ Uses duck-typing on ``litellm.callbacks`` (rather than importing the
+ proxy hook directly) to avoid a layering violation (SDK importing from
+ the proxy layer).
+ """
+ import litellm as _litellm
+
+ callbacks: Final = cast( # cast-ok: callback registry is inspected before protocol use
+ Sequence[object], _litellm.callbacks
+ )
+ return tuple(
+ cast(ProjectQuotaCallback, callback) # cast-ok: required callback method is callable
+ for callback in callbacks
+ if callable(getattr(callback, "enforce_project_io_token_quota_for_frame", None))
+ )
+
+
class BaseLLMHTTPHandler:
async def _make_common_async_call(
self,
@@ -6179,6 +6201,8 @@ class BaseLLMHTTPHandler:
- Uses ManagedResponsesWebSocketHandler which makes HTTP streaming calls
- Forwards events over the websocket connection
"""
+ _ws_quota_callbacks: Final = _collect_ws_project_quota_callbacks()
+
if responses_api_provider_config is None or not responses_api_provider_config.supports_native_websocket():
from litellm.responses.streaming_iterator import (
ManagedResponsesWebSocketHandler,
@@ -6195,6 +6219,7 @@ class BaseLLMHTTPHandler:
timeout=timeout,
custom_llm_provider=custom_llm_provider,
first_message=first_message,
+ quota_callbacks=_ws_quota_callbacks,
**kwargs,
)
await handler.run()
@@ -6315,6 +6340,7 @@ class BaseLLMHTTPHandler:
first_message=first_message,
guardrail_callbacks=_ws_guardrail_callbacks,
output_guardrail_callbacks=_ws_output_guardrail_callbacks,
+ quota_callbacks=_ws_quota_callbacks,
authorized_model=model,
)
await streaming.bidirectional_forward()
@@ -9407,6 +9433,27 @@ class BaseLLMHTTPHandler:
)
###### VECTOR STORE HANDLER ######
+ @staticmethod
+ def _pre_call_direct_vector_store_search(
+ logging_obj: LiteLLMLoggingObj,
+ custom_llm_provider: str,
+ vector_store_id: str,
+ query: str | Sequence[str],
+ ) -> None:
+ """Direct providers have no HTTP request to echo, and an empty api_base makes the debug
+ logger fall back to dumping model_call_details, which holds stored provider credentials."""
+ endpoint: Final = f"{custom_llm_provider}://{vector_store_id}"
+ logging_obj.pre_call(
+ input="",
+ api_key="",
+ additional_args={ # mutable-ok: pre_call's additional_args contract is a dict
+ "query": query,
+ "vector_store_id": vector_store_id,
+ "api_base": endpoint,
+ "request_str": f"direct vector store search: {endpoint}",
+ },
+ )
+
async def async_vector_store_search_handler(
self,
vector_store_id: str,
@@ -9423,13 +9470,11 @@ class BaseLLMHTTPHandler:
_is_async: bool = False,
) -> VectorStoreSearchResponse:
if isinstance(vector_store_provider_config, BaseDirectVectorStoreConfig):
- logging_obj.pre_call(
- input="",
- api_key="",
- additional_args={ # mutable-ok: pre_call's additional_args contract is a dict
- "query": query,
- "vector_store_id": vector_store_id,
- },
+ self._pre_call_direct_vector_store_search(
+ logging_obj=logging_obj,
+ custom_llm_provider=custom_llm_provider,
+ vector_store_id=vector_store_id,
+ query=query,
)
return await vector_store_provider_config.aexecute_search_vector_store_request(
vector_store_id=vector_store_id,
@@ -9554,13 +9599,11 @@ class BaseLLMHTTPHandler:
)
if isinstance(vector_store_provider_config, BaseDirectVectorStoreConfig):
- logging_obj.pre_call(
- input="",
- api_key="",
- additional_args={ # mutable-ok: pre_call's additional_args contract is a dict
- "query": query,
- "vector_store_id": vector_store_id,
- },
+ self._pre_call_direct_vector_store_search(
+ logging_obj=logging_obj,
+ custom_llm_provider=custom_llm_provider,
+ vector_store_id=vector_store_id,
+ query=query,
)
return vector_store_provider_config.execute_search_vector_store_request(
vector_store_id=vector_store_id,
diff --git a/litellm/llms/deepseek/chat/transformation.py b/litellm/llms/deepseek/chat/transformation.py
index 24da5b79261..566c960333a 100644
--- a/litellm/llms/deepseek/chat/transformation.py
+++ b/litellm/llms/deepseek/chat/transformation.py
@@ -131,9 +131,11 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
- model supports reasoning (capability check)
- user explicitly passed thinking={"type": "enabled"} (opt-in check)
"""
+ thinking: Final = optional_params.get("thinking")
return (
supports_reasoning(model=model, custom_llm_provider="deepseek")
- and (optional_params.get("thinking") or {}).get("type") == "enabled"
+ and isinstance(thinking, dict)
+ and thinking.get("type") == "enabled"
)
@staticmethod
diff --git a/litellm/llms/tinyfish/search/transformation.py b/litellm/llms/tinyfish/search/transformation.py
index ba9ca2e1bde..b688dc2cd01 100644
--- a/litellm/llms/tinyfish/search/transformation.py
+++ b/litellm/llms/tinyfish/search/transformation.py
@@ -14,6 +14,7 @@ import httpx
from pydantic import TypeAdapter, ValidationError
from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.search.transformation import (
@@ -22,7 +23,7 @@ from litellm.llms.base_llm.search.transformation import (
)
from litellm.secret_managers.main import get_secret_str
-_UrlEncodableParams: Final = TypeAdapter(dict[str, str | int | bool])
+_UrlEncodableParams: Final = TypeAdapter(dict[str, str | int | float | bool])
_StrList: Final = TypeAdapter(list[str])
_StrFrozenSet: Final = TypeAdapter(frozenset[str])
@@ -94,16 +95,16 @@ class TinyfishSearchConfig(BaseSearchConfig):
TinyFish equivalents:
- ``query`` (str or list[str]) → ``query`` (list joined by spaces)
- ``country`` → ``location``
- - ``search_domain_filter`` (list[str]) → folded into the query as
- ``() (site:a OR site:b ...)`` (TinyFish has no first-class
- field today; see ML-2084 for the planned ``include_domains``)
+ - ``search_domain_filter`` (list[str]) → folded into the query using
+ search operators
- ``max_results`` → not sent on the wire; stashed on
``self._caller_max_results`` for client-side response truncation
(TinyFish doesn't honor it server-side)
- ``max_tokens_per_page`` → silently dropped (no TinyFish equivalent)
Any other ``optional_params`` keys are forwarded to TinyFish as-is.
- dict/list values are JSON-encoded so they survive ``urlencode``.
+ dict and list values are JSON-encoded so structured payloads survive
+ ``urlencode``.
Returns:
``{_TINYFISH_PARAMS_KEY: }``.
@@ -144,14 +145,12 @@ class TinyfishSearchConfig(BaseSearchConfig):
supported_perplexity: Final = _StrFrozenSet.validate_python(raw_supported)
for param, value in optional_params.items():
if param not in supported_perplexity and param not in request_data:
- # `fetch` expects a JSON-encoded object on the wire; accept the
- # natural Python dict form and serialize here so callers don't
- # have to pre-stringify.
- if isinstance(value, dict):
+ # Serialize dicts/lists as JSON so structured params survive urlencode.
+ if isinstance(value, (dict, list)):
value = json.dumps(value, separators=(",", ":"))
# `urlencode` would render Python bool as "True"/"False"
- # (capitalized). ux-labs validators require lowercase
- # "true"/"false" (e.g. `include_thumbnail`); normalize here.
+ # (capitalized). TinyFish Search's bool params require lowercase
+ # "true"/"false" strings on the wire; normalize here.
elif isinstance(value, bool):
value = "true" if value else "false"
request_data[param] = value
@@ -167,17 +166,35 @@ class TinyfishSearchConfig(BaseSearchConfig):
"""
Transform a TinyFish response to LiteLLM's unified ``SearchResponse``.
- Mappings (per-result):
- - ``title`` → ``SearchResult.title`` (defaults to ``""`` if missing/null)
- - ``url`` → ``SearchResult.url`` (defaults to ``""``)
- - ``snippet`` → ``SearchResult.snippet`` (defaults to ``""``)
- - all other per-result fields (``position``, ``site_name``,
- ``thumbnail_url``, ``fetch``, ``fetch_error``, ...) ride through as
- extras on ``SearchResult`` via its ``extra="allow"`` config.
+ Per-result field handling:
+ - ``title``, ``url``, ``snippet`` are declared on ``SearchResult`` and
+ populated by ``SearchResponse.model_validate`` when present. Missing
+ or ``None`` values are defaulted to ``""`` beforehand by
+ ``_default_missing_result_fields`` so a degraded result flows through
+ instead of failing the whole call.
+ - All undeclared per-result fields (``position``, ``site_name``, and
+ any others TinyFish returns) ride through as extras via
+ ``SearchResult``'s ``extra="allow"`` config — accessible as
+ attributes on the result object or enumerable via
+ ``result.model_extra``.
- Top-level ``parameter_warnings`` (see ML-2085) is read when present and
- each entry is re-fired via ``verbose_logger.warning``. Absent or
- malformed entries are silently skipped — never throws.
+ Top-level ``parameter_warnings`` is read when present and each entry
+ is re-fired via ``verbose_logger.warning``. Absent or malformed
+ entries are silently skipped — never throws.
+
+ Top-level extras (``query``, ``total_results``, ``page``, and any
+ future TinyFish additions) ride through via
+ ``SearchResponse.extra="allow"``. The validated response is returned
+ in place after truncating ``results`` to the caller's ``max_results``,
+ so every field pydantic populated survives regardless of which
+ storage bucket (declared attribute or ``__pydantic_extra__``) holds it.
+
+ TinyFish response headers (e.g. ``x-request-id``, ``retry-after``,
+ ``x-ratelimit-limit`` — httpx normalizes header names to lowercase)
+ are stashed on ``response._hidden_params["headers"]`` (raw) and
+ ``response._hidden_params["additional_headers"]`` (sanitized via
+ ``process_response_headers``) so callers can correlate a search with
+ server-side logs.
Error paths routed through ``self._wrap_error`` for uniform
``"TinyFish Search: . See for details."`` wrapping:
@@ -223,7 +240,12 @@ class TinyfishSearchConfig(BaseSearchConfig):
_emit_parameter_warnings(parsed)
max_results: Final = self._caller_max_results or _TINYFISH_RESULT_CAP
- return SearchResponse(results=list(parsed.results[:max_results]))
+ parsed.results = parsed.results[:max_results]
+ raw_headers: Final = dict(raw_response.headers)
+ hidden: Final = parsed._hidden_params # pyright: ignore[reportPrivateUsage] # sole hidden-params channel
+ hidden["headers"] = raw_headers
+ hidden["additional_headers"] = process_response_headers(raw_headers)
+ return parsed
def _wrap_error(
self,
@@ -243,9 +265,9 @@ class TinyfishSearchConfig(BaseSearchConfig):
carry the ``TinyFish Search:`` prefix — the bare error already names
the host in the URL, so attribution is implicit there.
"""
- # ux-labs frontend wraps every error body as {"error": {"code", "message", "details"?}}.
+ # TinyFish Search wraps every error body as {"error": {"code", "message", "details"?}}.
# Best-effort unwrap to surface the inner message; fall back to the raw body
- # for non-ux-labs responses (CDN HTML pages, other JSON envelopes, plain text).
+ # for other envelope shapes (CDN HTML pages, other JSON envelopes, plain text).
inner_message = error_message
try:
body: Final[object] = json.loads(error_message) # any-ok: json.loads -> Any
@@ -290,7 +312,7 @@ def _default_missing_result_fields(raw_json: object) -> None:
def _emit_parameter_warnings(parsed: SearchResponse) -> None:
- """Re-fire TinyFish-side ``parameter_warnings`` (see ML-2085) as warnings.
+ """Re-fire TinyFish-side ``parameter_warnings`` as warnings.
Defensive: skip silently on any shape we don't recognize so a malformed
entry (or an early/partial rollout of the field) never throws.
diff --git a/litellm/main.py b/litellm/main.py
index cc27da830d8..f0b20eba9b6 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -5007,7 +5007,6 @@ def completion(
tool_choice = validate_chat_completion_tool_choice(tool_choice=tool_choice)
# validate optional params
stop = validate_openai_optional_params(stop=stop)
- # normalize camelCase thinking keys (e.g. budgetTokens -> budget_tokens)
thinking = validate_and_fix_thinking_param(thinking=thinking)
######### unpacking kwargs #####################
diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 409022016b0..07f9027313b 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -14456,6 +14456,25 @@
"supports_tool_choice": true,
"supports_output_config": true
},
+ "databricks/databricks-claude-opus-4-6": {
+ "input_cost_per_token": 5.00003e-06,
+ "input_dbu_cost_per_token": 7.1429e-05,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 2.5000010000000002e-05,
+ "output_dbu_cost_per_token": 0.000357143,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+ "supports_assistant_prefill": true,
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true
+ },
"databricks/databricks-claude-sonnet-4": {
"input_cost_per_token": 2.9999900000000002e-06,
"input_dbu_cost_per_token": 4.2857e-05,
@@ -14513,6 +14532,25 @@
"supports_reasoning": true,
"supports_tool_choice": true
},
+ "databricks/databricks-claude-sonnet-4-6": {
+ "input_cost_per_token": 2.9999900000000002e-06,
+ "input_dbu_cost_per_token": 4.2857e-05,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 64000,
+ "max_tokens": 64000,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 1.5000020000000002e-05,
+ "output_dbu_cost_per_token": 0.000214286,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+ "supports_assistant_prefill": true,
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true
+ },
"databricks/databricks-gemini-2-5-flash": {
"input_cost_per_token": 3.0001999999999996e-07,
"input_dbu_cost_per_token": 4.285999999999999e-06,
@@ -14547,6 +14585,74 @@
"supports_function_calling": true,
"supports_tool_choice": true
},
+ "databricks/databricks-gemini-3-1-flash-lite": {
+ "input_cost_per_token": 3.1248e-07,
+ "input_dbu_cost_per_token": 4.464e-06,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 1.87502e-06,
+ "output_dbu_cost_per_token": 2.6786e-05,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+ "supports_function_calling": true,
+ "supports_tool_choice": true
+ },
+ "databricks/databricks-gemini-3-1-pro": {
+ "input_cost_per_token": 2.49998e-06,
+ "input_dbu_cost_per_token": 3.5714e-05,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 1.5000020000000002e-05,
+ "output_dbu_cost_per_token": 0.000214286,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+ "supports_function_calling": true,
+ "supports_tool_choice": true
+ },
+ "databricks/databricks-gemini-3-flash": {
+ "input_cost_per_token": 6.2503e-07,
+ "input_dbu_cost_per_token": 8.929e-06,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 3.74997e-06,
+ "output_dbu_cost_per_token": 5.3571e-05,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+ "supports_function_calling": true,
+ "supports_tool_choice": true
+ },
+ "databricks/databricks-gemini-3-pro": {
+ "input_cost_per_token": 2.49998e-06,
+ "input_dbu_cost_per_token": 3.5714e-05,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 1.5000020000000002e-05,
+ "output_dbu_cost_per_token": 0.000214286,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+ "supports_function_calling": true,
+ "supports_tool_choice": true
+ },
"databricks/databricks-gemma-3-12b": {
"input_cost_per_token": 1.5000999999999998e-07,
"input_dbu_cost_per_token": 2.1429999999999996e-06,
@@ -14592,6 +14698,126 @@
"output_dbu_cost_per_token": 0.000142857,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
},
+ "databricks/databricks-gpt-5-1-codex-max": {
+ "input_cost_per_token": 1.24999e-06,
+ "input_dbu_cost_per_token": 1.7857e-05,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 272000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 9.999990000000002e-06,
+ "output_dbu_cost_per_token": 0.000142857,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+ },
+ "databricks/databricks-gpt-5-1-codex-mini": {
+ "input_cost_per_token": 2.4997e-07,
+ "input_dbu_cost_per_token": 3.571e-06,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 272000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 1.99997e-06,
+ "output_dbu_cost_per_token": 2.8571e-05,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+ },
+ "databricks/databricks-gpt-5-2": {
+ "input_cost_per_token": 1.75e-06,
+ "input_dbu_cost_per_token": 2.5e-05,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 272000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 1.4e-05,
+ "output_dbu_cost_per_token": 0.0002,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+ },
+ "databricks/databricks-gpt-5-2-codex": {
+ "input_cost_per_token": 1.75e-06,
+ "input_dbu_cost_per_token": 2.5e-05,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 272000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 1.4e-05,
+ "output_dbu_cost_per_token": 0.0002,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+ },
+ "databricks/databricks-gpt-5-3-codex": {
+ "input_cost_per_token": 1.75e-06,
+ "input_dbu_cost_per_token": 2.5e-05,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 272000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 1.4e-05,
+ "output_dbu_cost_per_token": 0.0002,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+ },
+ "databricks/databricks-gpt-5-4": {
+ "input_cost_per_token": 2.49998e-06,
+ "input_dbu_cost_per_token": 3.5714e-05,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 272000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 1.5000020000000002e-05,
+ "output_dbu_cost_per_token": 0.000214286,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+ },
+ "databricks/databricks-gpt-5-4-mini": {
+ "input_cost_per_token": 7.4998e-07,
+ "input_dbu_cost_per_token": 1.0714e-05,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 272000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 4.50002e-06,
+ "output_dbu_cost_per_token": 6.4286e-05,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+ },
+ "databricks/databricks-gpt-5-4-nano": {
+ "input_cost_per_token": 1.9999e-07,
+ "input_dbu_cost_per_token": 2.857e-06,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 272000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 1.24999e-06,
+ "output_dbu_cost_per_token": 1.7857e-05,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+ },
"databricks/databricks-gpt-5-mini": {
"input_cost_per_token": 2.4997000000000006e-07,
"input_dbu_cost_per_token": 3.571e-06,
diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
index e46e6299277..1ca4c657706 100644
--- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
+++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
@@ -1341,7 +1341,7 @@ async def _persist_dcr_client_registration(
``update_mcp_server`` merges credential blobs: a re-registered public client must not
inherit the previous client's secret or auth method.
"""
- if mcp_server.is_true_passthrough or mcp_server.is_oauth_delegate:
+ if mcp_server.is_client_forwarded_token:
return "skipped"
try:
@@ -2187,7 +2187,7 @@ async def _build_oauth_protected_resource_response(
)
if upstream_metadata is not None:
- if mcp_server.is_true_passthrough or mcp_server.is_oauth_delegate:
+ if mcp_server.is_client_forwarded_token:
return upstream_metadata
return {**upstream_metadata, "resource": resource_url}
diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
index 7fff6c12fe0..65855df89f6 100644
--- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
+++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
@@ -123,6 +123,7 @@ from litellm.proxy._experimental.mcp_server.utils import (
iter_known_server_prefixes,
iter_known_tool_name_spellings,
logging_safe_mcp_headers,
+ lookup_mcp_server_auth_in_headers,
match_known_server_prefix,
match_known_tool_name,
merge_mcp_headers,
@@ -873,6 +874,53 @@ def _openapi_forwarded_extra_headers(
return forwarded or None
+def _resolve_openapi_tool_auth(
+ mcp_server: MCPServer,
+ mcp_auth_header: str | None,
+ mcp_server_auth_headers: Mapping[str, str | dict[str, str]] | None, # mutable-ok: sink shape
+ raw_headers: dict[str, str] | None, # mutable-ok: sink takes a concrete dict
+ user_api_key_auth: UserAPIKeyAuth | None,
+) -> tuple[str | None, dict[str, str] | None, str | dict[str, str] | None]: # mutable-ok: sink shapes
+ """The caller's upstream credential for one ``spec_path`` server, for both OpenAPI dispatch arms.
+
+ A per-server ``x-mcp-{alias}-authorization`` wins over the deprecated global / BYOK
+ ``mcp_auth_header``, the same precedence ``_call_regular_mcp_tool`` applies, so the OpenAPI and
+ managed paths cannot disagree about which credential is authoritative. The two kinds are not
+ interchangeable: a per-server value is already a complete header value and is forwarded verbatim,
+ while a BYOK credential is a raw secret that takes the server's auth-type prefix. Formatting the
+ former would ship ``Bearer Bearer ``.
+
+ Returns the ``Authorization`` value to inject, the extra headers to forward, and the credential to
+ hand ``resolve_openapi_upstream_auth``, whose passthrough arm reads it via
+ ``_passthrough_token_from_mcp_auth_header``. The per-server Authorization travels only in the
+ credential, never also in the forwarded headers, because the resolver pops Authorization out of
+ those and would otherwise have two sources to reconcile.
+ """
+ forwarded: Final = _openapi_forwarded_extra_headers(mcp_server, raw_headers, user_api_key_auth)
+ per_server: Final = (
+ lookup_mcp_server_auth_in_headers(
+ mcp_server_auth_headers,
+ alias=mcp_server.alias,
+ server_name=mcp_server.server_name,
+ )
+ if mcp_server_auth_headers
+ else None
+ )
+
+ if isinstance(per_server, dict):
+ authorization: Final = next((v for k, v in per_server.items() if k.lower() == "authorization"), None)
+ merged: Final = merge_mcp_headers(extra_headers=forwarded, static_headers=_without_authorization(per_server))
+ if authorization is None:
+ byok: Final = _format_byok_openapi_auth_header(mcp_server, mcp_auth_header) if mcp_auth_header else None
+ return byok, merged, mcp_auth_header
+ return authorization, merged, per_server
+ if isinstance(per_server, str) and per_server:
+ return per_server, forwarded, per_server
+ if mcp_auth_header:
+ return _format_byok_openapi_auth_header(mcp_server, mcp_auth_header), forwarded, mcp_auth_header
+ return None, forwarded, None
+
+
async def _resolve_byok_mcp_auth_header(
mcp_server: MCPServer,
user_api_key_auth: UserAPIKeyAuth | None,
@@ -1740,12 +1788,14 @@ class MCPServerManager:
server: The MCP server whose OAuth metadata must be resolved.
Returns:
- The resolved server, or the registered server when no discovery is
- pending.
+ The resolved server; the registered server when no discovery is
+ pending, or when discovery failed for a client-forwarded-token
+ server, whose session consumes no discovered endpoint.
Raises:
HTTPException: Status 503 when discovery times out or returns
- incomplete metadata.
+ incomplete metadata for a server whose OAuth flow the gateway
+ runs itself.
"""
acquisition: Final = self._get_or_start_oauth_discovery_task(server)
if acquisition is None:
@@ -1764,6 +1814,8 @@ class MCPServerManager:
return await self.ensure_oauth_metadata_discovered(server)
case _OAuthDiscoveryFailed(timed_out=timed_out):
current: Final = self._registered_server(server)
+ if current.is_client_forwarded_token:
+ return current
server_ref: Final = current.alias or current.server_name or current.name or current.server_id
reason: Final = "timed out" if timed_out else "returned incomplete metadata"
raise HTTPException(
@@ -2321,23 +2373,30 @@ class MCPServerManager:
openapi_key_prefix: Final = prefix_root + MCP_TOOL_PREFIX_SEPARATOR
global_mcp_tool_registry.unregister_tools_with_prefix(openapi_key_prefix)
- owned_raw: Final[set[str]] = set()
- for p in iter_known_server_prefixes(server):
- if p:
- owned_raw.add(p)
- if server.name:
- owned_raw.add(server.name)
+ owned_normalized: Final = self._owned_mapping_values(server)
- owned_normalized: Final = {normalize_server_name(x) for x in owned_raw}
-
- stale_mapping_keys: Final[list[str]] = []
- for tool_name, mapped_server in list(self.tool_name_to_mcp_server_name_mapping.items()):
- if mapped_server in owned_raw or normalize_server_name(str(mapped_server)) in owned_normalized:
- stale_mapping_keys.append(tool_name)
+ stale_mapping_keys: Final = tuple(
+ tool_name
+ for tool_name, mapped_server in self.tool_name_to_mcp_server_name_mapping.items()
+ if normalize_server_name(str(mapped_server)) in owned_normalized
+ )
for key in stale_mapping_keys:
del self.tool_name_to_mcp_server_name_mapping[key]
+ def _owned_mapping_values(self, server: MCPServer) -> frozenset[str]:
+ return frozenset(
+ normalize_server_name(value) for value in (*iter_known_server_prefixes(server), server.name) if value
+ )
+
+ def _server_exposes_tool(self, server: MCPServer, tool_name: str) -> bool:
+ owned: Final = self._owned_mapping_values(server)
+ mapped_owners: Final = (
+ self.tool_name_to_mcp_server_name_mapping.get(spelling)
+ for spelling in iter_known_tool_name_spellings(tool_name, server)
+ )
+ return any(owner is not None and normalize_server_name(owner) in owned for owner in mapped_owners)
+
def remove_server(self, mcp_server: LiteLLM_MCPServerTable):
"""
Remove a server from the registry
@@ -3182,10 +3241,6 @@ class MCPServerManager:
# Get server-specific auth header if available
server_auth_header: str | dict[str, str] | None = None
if mcp_server_auth_headers:
- from litellm.proxy._experimental.mcp_server.utils import (
- lookup_mcp_server_auth_in_headers,
- )
-
server_auth_header = lookup_mcp_server_auth_in_headers(
mcp_server_auth_headers,
alias=server.alias,
@@ -5210,11 +5265,6 @@ class MCPServerManager:
# the exact case of server alias/name (e.g., '1litellmagcgateway' vs '1LiteLLMAGCGateway')
server_auth_header: dict[str, str] | str | None = None
if mcp_server_auth_headers:
- # Normalize keys for case-insensitive lookup
- from litellm.proxy._experimental.mcp_server.utils import (
- lookup_mcp_server_auth_in_headers,
- )
-
server_auth_header = lookup_mcp_server_auth_in_headers(
mcp_server_auth_headers,
alias=mcp_server.alias,
@@ -5249,7 +5299,7 @@ class MCPServerManager:
user_api_key_auth=user_api_key_auth,
):
extra_headers = _without_authorization(extra_headers)
- elif mcp_server.is_true_passthrough or mcp_server.is_oauth_delegate:
+ elif mcp_server.is_client_forwarded_token:
extra_headers = _client_forwarded_authorization_headers(
mcp_server=mcp_server,
oauth2_headers=oauth2_headers,
@@ -5356,7 +5406,7 @@ class MCPServerManager:
# Scoped to the two client-forwarded token modes this stack introduced; legacy
# oauth2 + delegate_auth_to_upstream (is_oauth_passthrough) is being removed, so it is not
# added here even though the list path still relays for it.
- relays_upstream_auth: Final = mcp_server.is_true_passthrough or mcp_server.is_oauth_delegate
+ relays_upstream_auth: Final = mcp_server.is_client_forwarded_token
server_label: Final = mcp_server.name or mcp_server.server_name or mcp_server.alias or ""
async def _call_tool_via_client(client, params):
@@ -5463,13 +5513,8 @@ class MCPServerManager:
if mcp_server is None:
raise ValueError(f"Tool {name} not found")
- if resolved_by_server_name_only:
- tool_known: Final = (
- name in self.tool_name_to_mcp_server_name_mapping
- or prefixed_tool_name in self.tool_name_to_mcp_server_name_mapping
- )
- if not tool_known:
- raise ValueError(f"Tool {name} not found")
+ if resolved_by_server_name_only and not self._server_exposes_tool(mcp_server, name):
+ raise ValueError(f"Tool {name} not found")
return mcp_server
@@ -5712,16 +5757,20 @@ class MCPServerManager:
server_name,
)
- auth_header_value: Final = (
- _format_byok_openapi_auth_header(mcp_server, mcp_auth_header) if mcp_auth_header else None
+ auth_header_value, openapi_forwarded_headers, upstream_credential = _resolve_openapi_tool_auth(
+ mcp_server=mcp_server,
+ mcp_auth_header=mcp_auth_header,
+ mcp_server_auth_headers=mcp_server_auth_headers,
+ raw_headers=raw_headers,
+ user_api_key_auth=user_api_key_auth,
)
resolved_auth_headers, forwarded_headers = await self.resolve_openapi_upstream_auth(
mcp_server=mcp_server,
oauth2_headers=caller_oauth2_headers,
raw_headers=raw_headers,
- mcp_auth_header=mcp_auth_header,
+ mcp_auth_header=upstream_credential,
user_api_key_auth=user_api_key_auth,
- forwarded_headers=_openapi_forwarded_extra_headers(mcp_server, raw_headers, user_api_key_auth),
+ forwarded_headers=openapi_forwarded_headers,
)
async def _call_openapi_via_handler():
@@ -5847,10 +5896,7 @@ class MCPServerManager:
if matched is not None:
matched_prefix, original_tool_name = matched
matched_server: Final = prefix_to_server.get(matched_prefix)
- if matched_server is not None and (
- original_tool_name in self.tool_name_to_mcp_server_name_mapping
- or tool_name in self.tool_name_to_mcp_server_name_mapping
- ):
+ if matched_server is not None and self._server_exposes_tool(matched_server, original_tool_name):
return matched_server
return None
diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py
index 4184fad009c..8d69d84e492 100644
--- a/litellm/proxy/_experimental/mcp_server/server.py
+++ b/litellm/proxy/_experimental/mcp_server/server.py
@@ -430,6 +430,7 @@ if MCP_AVAILABLE:
MCPServerManager,
_caller_authorization_fans_out,
_client_forwarded_authorization_headers,
+ _resolve_openapi_tool_auth,
_should_strip_caller_authorization,
_without_authorization,
global_mcp_server_manager,
@@ -1704,7 +1705,7 @@ if MCP_AVAILABLE:
)
extra_headers: dict[str, str] | None = None
- is_client_forwarded_mode: Final = server.is_true_passthrough or server.is_oauth_delegate
+ is_client_forwarded_mode: Final = server.is_client_forwarded_token
# In a multi-server listing scope the request-wide Authorization can only carry one token,
# so it is withheld from a client-forwarded server when another server in scope also consumes
# it (RFC 9700 cross-resource replay); such scopes must bind per-server via
@@ -2013,6 +2014,9 @@ if MCP_AVAILABLE:
prefetched_creds=_prefetched_oauth_creds,
)
+ if server.is_byok and server.auth_type != MCPAuth.oauth2 and server_auth_header is None:
+ server_auth_header = await _get_byok_credential(server, user_api_key_auth)
+
try:
tools: Final = await global_mcp_server_manager._get_tools_from_server(
server=server,
@@ -2832,58 +2836,26 @@ if MCP_AVAILABLE:
arguments = hook_result["arguments"]
verbose_logger.debug("Executing local registry tool: %s", name)
- # For BYOK servers the credential must be injected via a ContextVar
- # because the tool function has headers baked into its closure.
- # Pre-format the full Authorization header value using the server's
- # configured auth_type so the generator doesn't need to know the prefix.
- auth_header_value: str | None = None
- if mcp_auth_header:
- server_auth_type: Final = getattr(mcp_server, "auth_type", None) if mcp_server else None
- if server_auth_type == MCPAuth.api_key:
- auth_header_value = f"ApiKey {mcp_auth_header}"
- elif server_auth_type == MCPAuth.basic:
- auth_header_value = f"Basic {mcp_auth_header}"
- else:
- auth_header_value = f"Bearer {mcp_auth_header}"
-
- # Forward named client headers to OpenAPI tool upstream requests.
- # MCPServer.extra_headers lists header names to copy from raw_headers.
- # The strip decision is centralized in _should_strip_caller_authorization so this
- # OpenAPI/local path agrees with the managed paths: M2M and the resolver-owned modes
- # (token_exchange's raw subject token, authorization_code's stored token) must never
- # have the caller's Authorization forwarded verbatim upstream.
- forwarded_headers: dict[str, str] | None = None
- if mcp_server and mcp_server.extra_headers and raw_headers:
- normalized_raw: Final = {str(k).lower(): v for k, v in raw_headers.items() if isinstance(k, str)}
- skip_caller_authorization: Final = _should_strip_caller_authorization(
- mcp_server=mcp_server,
- raw_headers=raw_headers,
- user_api_key_auth=user_api_key_auth,
- )
- for header_name in mcp_server.extra_headers:
- if not isinstance(header_name, str):
- continue
- if skip_caller_authorization and header_name.lower() == "authorization":
- continue
- value = normalized_raw.get(header_name.lower())
- if value is not None:
- if forwarded_headers is None:
- forwarded_headers = {}
- forwarded_headers[header_name] = value
-
- resolved_auth_headers: dict[str, str] | None = None
- if mcp_server:
- (
- resolved_auth_headers,
- forwarded_headers,
- ) = await global_mcp_server_manager.resolve_openapi_upstream_auth(
- mcp_server=mcp_server,
- oauth2_headers=oauth2_headers,
- raw_headers=raw_headers,
- mcp_auth_header=mcp_auth_header,
- user_api_key_auth=user_api_key_auth,
- forwarded_headers=forwarded_headers,
- )
+ # The credential rides ContextVars because the tool function has its
+ # headers baked into the closure at registration time.
+ auth_header_value, openapi_forwarded_headers, upstream_credential = _resolve_openapi_tool_auth(
+ mcp_server=mcp_server,
+ mcp_auth_header=mcp_auth_header,
+ mcp_server_auth_headers=mcp_server_auth_headers,
+ raw_headers=raw_headers,
+ user_api_key_auth=user_api_key_auth,
+ )
+ (
+ resolved_auth_headers,
+ forwarded_headers,
+ ) = await global_mcp_server_manager.resolve_openapi_upstream_auth(
+ mcp_server=mcp_server,
+ oauth2_headers=oauth2_headers,
+ raw_headers=raw_headers,
+ mcp_auth_header=upstream_credential,
+ user_api_key_auth=user_api_key_auth,
+ forwarded_headers=openapi_forwarded_headers,
+ )
_auth_token: Final = _request_auth_header.set(auth_header_value)
_extra_token: Final = _request_extra_headers.set(forwarded_headers)
diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index f128940c315..b0891b73fb8 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -3059,6 +3059,8 @@ class NewProjectRequest(LiteLLM_BudgetTable):
models: list[str] = []
model_rpm_limit: dict | None = None
model_tpm_limit: dict | None = None
+ model_itpm_limit: Mapping[str, int] | None = None
+ model_otpm_limit: Mapping[str, int] | None = None
blocked: bool = False
object_permission: LiteLLM_ObjectPermissionBase | None = None
@@ -3091,6 +3093,8 @@ class UpdateProjectRequest(LiteLLM_BudgetTable):
models: list[str] | None = None
model_rpm_limit: dict | None = None
model_tpm_limit: dict | None = None
+ model_itpm_limit: Mapping[str, int] | None = None
+ model_otpm_limit: Mapping[str, int] | None = None
blocked: bool | None = None
budget_id: str | None = None
object_permission: LiteLLM_ObjectPermissionBase | None = None
@@ -4244,6 +4248,8 @@ class PassThroughEndpointLoggingTypedDict(TypedDict):
LiteLLM_ManagementEndpoint_MetadataFields: Final = [
"model_rpm_limit",
"model_tpm_limit",
+ "model_itpm_limit",
+ "model_otpm_limit",
"default_estimated_output_tokens",
"default_estimated_output_tokens_per_model",
"mcp_rpm_limit",
diff --git a/litellm/proxy/agent_endpoints/a2a_endpoints.py b/litellm/proxy/agent_endpoints/a2a_endpoints.py
index 1cae87aed31..cea30ffad52 100644
--- a/litellm/proxy/agent_endpoints/a2a_endpoints.py
+++ b/litellm/proxy/agent_endpoints/a2a_endpoints.py
@@ -168,14 +168,13 @@ def _jsonrpc_error(
)
-def _get_agent(agent_id: str):
+async def _get_agent(agent_id: str) -> "AgentResponse | None":
"""Look up an agent by ID or name. Returns None if not found."""
- from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
+ from litellm.proxy.common_utils.registry_read_through import (
+ get_agent_with_read_through,
+ )
- agent = global_agent_registry.get_agent_by_id(agent_id=agent_id)
- if agent is None:
- agent = global_agent_registry.get_agent_by_name(agent_name=agent_id)
- return agent
+ return await get_agent_with_read_through(agent_id)
def _enforce_inbound_trace_id(agent: "AgentResponse", request: Request) -> None:
@@ -559,7 +558,7 @@ async def get_agent_card(
)
try:
- agent: Final = _get_agent(agent_id)
+ agent: Final = await _get_agent(agent_id)
if agent is None:
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
@@ -673,7 +672,7 @@ async def invoke_agent_a2a(
params.pop(key)
# Find the agent
- agent: Final = _get_agent(agent_id)
+ agent: Final = await _get_agent(agent_id)
if agent is None:
return _jsonrpc_error(request_id, -32000, f"Agent '{agent_id}' not found", 404)
diff --git a/litellm/proxy/agent_endpoints/a2a_routing.py b/litellm/proxy/agent_endpoints/a2a_routing.py
index 038b6b4a840..8a795214750 100644
--- a/litellm/proxy/agent_endpoints/a2a_routing.py
+++ b/litellm/proxy/agent_endpoints/a2a_routing.py
@@ -25,10 +25,12 @@ async def route_a2a_agent_request(
Returns None if not an A2A request (allows normal routing to continue).
"""
# Import here to avoid circular imports
- from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
from litellm.proxy.agent_endpoints.auth.agent_permission_handler import (
AgentRequestHandler,
)
+ from litellm.proxy.common_utils.registry_read_through import (
+ get_agent_with_read_through,
+ )
from litellm.proxy.route_llm_request import (
ROUTE_ENDPOINT_MAPPING,
ProxyModelNotFoundError,
@@ -44,11 +46,11 @@ async def route_a2a_agent_request(
agent_name: Final = model_name[4:]
# Look up agent in registry
- agent: Final = global_agent_registry.get_agent_by_name(agent_name)
+ agent: Final = await get_agent_with_read_through(agent_name)
if agent is None:
verbose_proxy_logger.error("[A2A] Agent '%s' not found in registry", agent_name)
route_name = ROUTE_ENDPOINT_MAPPING.get(route_type, route_type)
- raise ProxyModelNotFoundError(route=route_name, model_name=model_name)
+ raise ProxyModelNotFoundError(route=route_name, model_name=model_name, retryable_with_model_read_through=False)
# Verify the caller is permitted to use this agent (admins bypass the check)
is_admin: Final = user_api_key_dict is not None and (
@@ -70,7 +72,7 @@ async def route_a2a_agent_request(
if not agent.agent_card_params or "url" not in agent.agent_card_params:
verbose_proxy_logger.error("[A2A] Agent '%s' has no URL configured", agent_name)
route_name = ROUTE_ENDPOINT_MAPPING.get(route_type, route_type)
- raise ProxyModelNotFoundError(route=route_name, model_name=model_name)
+ raise ProxyModelNotFoundError(route=route_name, model_name=model_name, retryable_with_model_read_through=False)
# Inject API base and route to litellm
data["api_base"] = agent.agent_card_params["url"]
diff --git a/litellm/proxy/agent_endpoints/agent_registry.py b/litellm/proxy/agent_endpoints/agent_registry.py
index 742fdf35b1e..64de6827679 100644
--- a/litellm/proxy/agent_endpoints/agent_registry.py
+++ b/litellm/proxy/agent_endpoints/agent_registry.py
@@ -600,3 +600,4 @@ class AgentRegistry:
global_agent_registry: Final = AgentRegistry()
+AGENT_RECONCILE_LOCK: Final = asyncio.Lock()
diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py
index a48ef0f08bb..f742965ade2 100644
--- a/litellm/proxy/anthropic_endpoints/endpoints.py
+++ b/litellm/proxy/anthropic_endpoints/endpoints.py
@@ -179,7 +179,7 @@ async def anthropic_response(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=e,
- request_data=data,
+ request_data=base_llm_response_processor.data,
)
body: Final = AnthropicExceptionMapping.transform_to_anthropic_error(
status_code=e.status_code,
@@ -189,7 +189,7 @@ async def anthropic_response(
return JSONResponse(status_code=e.status_code, content=body)
except Exception as e:
await proxy_logging_obj.post_call_failure_hook(
- user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
+ user_api_key_dict=user_api_key_dict, original_exception=e, request_data=base_llm_response_processor.data
)
verbose_proxy_logger.exception("litellm.proxy.proxy_server.anthropic_response(): Exception occured - %s", e)
diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py
index c9f9c00f120..a105bf19458 100644
--- a/litellm/proxy/auth/auth_utils.py
+++ b/litellm/proxy/auth/auth_utils.py
@@ -1172,7 +1172,7 @@ def enforce_output_token_estimates_are_admin_only(
def get_model_rate_limit_from_metadata(
user_api_key_dict: UserAPIKeyAuth,
metadata_accessor_key: Literal["team_metadata", "organization_metadata", "project_metadata"],
- rate_limit_key: Literal["model_rpm_limit", "model_tpm_limit"],
+ rate_limit_key: Literal["model_rpm_limit", "model_tpm_limit", "model_itpm_limit", "model_otpm_limit"],
) -> dict[str, int] | None:
if getattr(user_api_key_dict, metadata_accessor_key):
return getattr(user_api_key_dict, metadata_accessor_key).get(rate_limit_key)
diff --git a/litellm/proxy/common_utils/registry_read_through.py b/litellm/proxy/common_utils/registry_read_through.py
new file mode 100644
index 00000000000..460b348e188
--- /dev/null
+++ b/litellm/proxy/common_utils/registry_read_through.py
@@ -0,0 +1,224 @@
+"""Read-through recovery for in-memory registries in multi-replica deployments.
+
+A management write (POST /model/new, /guardrails, /v1/agents) lands on one
+replica and reaches Postgres, but sibling replicas only refresh their in-memory
+registries on the periodic config reload, so a request using the new object
+immediately can land on a sibling that has never heard of it and fail 400/404.
+On a registry miss, callers here fetch the missing row from the DB and load it
+into the local registry before giving up. A short negative-result TTL per key
+plus a global resync budget per window bound the DB load from lookups of
+genuinely unknown names.
+"""
+
+import asyncio
+import time
+from collections.abc import Awaitable, Callable
+from typing import TYPE_CHECKING, Final
+
+from litellm._logging import verbose_proxy_logger
+from litellm.caching.in_memory_cache import InMemoryCache
+
+if TYPE_CHECKING:
+ from prisma.types import (
+ LiteLLM_AgentsTableInclude,
+ LiteLLM_AgentsTableWhereUniqueInput,
+ LiteLLM_GuardrailsTableWhereInput,
+ LiteLLM_ProxyModelTableWhereInput,
+ )
+
+ from litellm.integrations.custom_guardrail import CustomGuardrail
+ from litellm.types.agents import AgentResponse
+
+READ_THROUGH_MISS_TTL_SECONDS: Final = 2.0
+READ_THROUGH_RESYNC_WINDOW_SECONDS: Final = 5.0
+READ_THROUGH_MAX_RESYNCS_PER_WINDOW: Final = 20
+
+
+class RegistryReadThrough:
+ __slots__ = (
+ "_lock",
+ "_max_resyncs_per_window",
+ "_miss_ttl_seconds",
+ "_recent_misses",
+ "_resync",
+ "_resync_window_seconds",
+ "_window_resyncs",
+ "_window_started_at",
+ )
+
+ def __init__(
+ self,
+ resync: Callable[[str], Awaitable[bool]],
+ miss_ttl_seconds: float = READ_THROUGH_MISS_TTL_SECONDS,
+ max_resyncs_per_window: int = READ_THROUGH_MAX_RESYNCS_PER_WINDOW,
+ resync_window_seconds: float = READ_THROUGH_RESYNC_WINDOW_SECONDS,
+ ) -> None:
+ self._resync = resync
+ self._miss_ttl_seconds = miss_ttl_seconds
+ self._max_resyncs_per_window = max_resyncs_per_window
+ self._resync_window_seconds = resync_window_seconds
+ self._lock = asyncio.Lock()
+ self._recent_misses = InMemoryCache(max_size_in_memory=1000)
+ self._window_started_at = float("-inf")
+ self._window_resyncs = 0
+
+ def _consume_resync_budget(self) -> bool:
+ now: Final = time.monotonic()
+ if now - self._window_started_at >= self._resync_window_seconds:
+ self._window_started_at = now
+ self._window_resyncs = 0
+ if self._window_resyncs >= self._max_resyncs_per_window:
+ return False
+ self._window_resyncs += 1
+ return True
+
+ async def attempt(self, key: str) -> bool:
+ if self._recent_misses.get_cache(key) is not None:
+ return False
+ async with self._lock:
+ if self._recent_misses.get_cache(key) is not None:
+ return False
+ if not self._consume_resync_budget():
+ verbose_proxy_logger.warning(
+ "registry read-through for %r skipped: resync budget of %s per %ss exhausted",
+ key,
+ self._max_resyncs_per_window,
+ self._resync_window_seconds,
+ )
+ return False
+ try:
+ found: Final = await self._resync(key)
+ except Exception as e: # noqa: BLE001 # a failed read-through must surface the original miss error, not a 500
+ verbose_proxy_logger.warning("registry read-through for %r failed: %s", key, e)
+ return False
+ if not found:
+ self._recent_misses.set_cache(key, True, ttl=self._miss_ttl_seconds)
+ return found
+
+
+def _db_backed_registries_enabled(object_type: str) -> bool:
+ from litellm.proxy import proxy_server
+
+ if proxy_server.prisma_client is None or proxy_server.store_model_in_db is not True:
+ return False
+ return proxy_server.should_load_db_object(object_type=object_type)
+
+
+async def _resync_model_deployments(model_name: str) -> bool:
+ from litellm.proxy import proxy_server
+ from litellm.repositories.model_repository import ModelRepository
+
+ if not _db_backed_registries_enabled("models"):
+ return False
+ prisma_client: Final = proxy_server.prisma_client
+ assert prisma_client is not None
+ table: Final = ModelRepository(prisma_client).table
+ name_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_name": model_name}
+ id_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_id": model_name}
+ rows: Final = await table.find_many(where=name_filter) or await table.find_many(where=id_filter)
+ if not rows:
+ return False
+ router: Final = proxy_server.llm_router
+ if router is None:
+ await proxy_server.proxy_config.add_deployment(
+ prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj
+ )
+ return proxy_server.llm_router is not None
+ async with proxy_server.MODEL_RECONCILE_LOCK:
+ proxy_server.proxy_config._add_deployment(db_models=rows)
+ proxy_server.llm_model_list = router.get_model_list()
+ return True
+
+
+async def _resync_guardrails(guardrail_name: str) -> bool:
+ from litellm.proxy import proxy_server
+ from litellm.proxy.guardrails.guardrail_registry import (
+ GUARDRAIL_RECONCILE_LOCK,
+ IN_MEMORY_GUARDRAIL_HANDLER,
+ )
+ from litellm.repositories.table_repositories import GuardrailsRepository
+ from litellm.types.guardrails import Guardrail
+
+ if not _db_backed_registries_enabled("guardrails"):
+ return False
+ prisma_client: Final = proxy_server.prisma_client
+ assert prisma_client is not None
+ active_row_filter: Final[LiteLLM_GuardrailsTableWhereInput] = {
+ "guardrail_name": guardrail_name,
+ "status": "active",
+ }
+ row: Final = await GuardrailsRepository(prisma_client).table.find_first(where=active_row_filter)
+ if row is None:
+ return False
+ async with GUARDRAIL_RECONCILE_LOCK:
+ IN_MEMORY_GUARDRAIL_HANDLER.sync_guardrail_from_db(guardrail=Guardrail(**dict(row)))
+ return _initialized_guardrail(guardrail_name) is not None
+
+
+async def _resync_agents(agent_id_or_name: str) -> bool:
+ from litellm.proxy import proxy_server
+ from litellm.proxy.agent_endpoints.agent_registry import (
+ AGENT_RECONCILE_LOCK,
+ agents_table,
+ global_agent_registry,
+ )
+ from litellm.types.agents import AgentResponse
+
+ if not _db_backed_registries_enabled("agents"):
+ return False
+ if _agent_from_registry(agent_id_or_name) is not None:
+ return True
+ prisma_client: Final = proxy_server.prisma_client
+ assert prisma_client is not None
+ table: Final = agents_table(prisma_client)
+ id_filter: Final[LiteLLM_AgentsTableWhereUniqueInput] = {"agent_id": agent_id_or_name}
+ name_filter: Final[LiteLLM_AgentsTableWhereUniqueInput] = {"agent_name": agent_id_or_name}
+ include_permission: Final[LiteLLM_AgentsTableInclude] = {"object_permission": True}
+ async with AGENT_RECONCILE_LOCK:
+ if _agent_from_registry(agent_id_or_name) is not None:
+ return True
+ row: Final = await table.find_unique(where=id_filter, include=include_permission) or await table.find_unique(
+ where=name_filter, include=include_permission
+ )
+ if row is None:
+ return False
+ global_agent_registry.register_agent(agent_config=AgentResponse.model_validate(row.model_dump()))
+ return True
+
+
+model_registry_read_through: Final = RegistryReadThrough(resync=_resync_model_deployments)
+guardrail_registry_read_through: Final = RegistryReadThrough(resync=_resync_guardrails)
+agent_registry_read_through: Final = RegistryReadThrough(resync=_resync_agents)
+
+
+def _agent_from_registry(agent_id_or_name: str) -> "AgentResponse | None":
+ from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
+
+ by_id: Final = global_agent_registry.get_agent_by_id(agent_id=agent_id_or_name)
+ if by_id is not None:
+ return by_id
+ return global_agent_registry.get_agent_by_name(agent_name=agent_id_or_name)
+
+
+async def get_agent_with_read_through(agent_id_or_name: str) -> "AgentResponse | None":
+ agent: Final = _agent_from_registry(agent_id_or_name)
+ if agent is not None:
+ return agent
+ if not await agent_registry_read_through.attempt(agent_id_or_name):
+ return None
+ return _agent_from_registry(agent_id_or_name)
+
+
+def _initialized_guardrail(guardrail_name: str) -> "CustomGuardrail | None":
+ from litellm.proxy.guardrails import guardrail_endpoints
+
+ return guardrail_endpoints.GUARDRAIL_REGISTRY.get_initialized_guardrail_callback(guardrail_name=guardrail_name)
+
+
+async def get_initialized_guardrail_with_read_through(guardrail_name: str) -> "CustomGuardrail | None":
+ active: Final = _initialized_guardrail(guardrail_name)
+ if active is not None:
+ return active
+ if not await guardrail_registry_read_through.attempt(guardrail_name):
+ return None
+ return _initialized_guardrail(guardrail_name)
diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py
index b68d4a68b79..e50a3a5a1e7 100644
--- a/litellm/proxy/guardrails/guardrail_endpoints.py
+++ b/litellm/proxy/guardrails/guardrail_endpoints.py
@@ -2305,8 +2305,12 @@ async def apply_guardrail(
litellm_logging_obj = None
start_time: Final = datetime.now(timezone.utc)
+ from litellm.proxy.common_utils.registry_read_through import (
+ get_initialized_guardrail_with_read_through,
+ )
+
try:
- active_guardrail: Final[CustomGuardrail | None] = GUARDRAIL_REGISTRY.get_initialized_guardrail_callback(
+ active_guardrail: Final[CustomGuardrail | None] = await get_initialized_guardrail_with_read_through(
guardrail_name=request.guardrail_name
)
if active_guardrail is None:
diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py
index 5f7374581a2..f6d348c1045 100644
--- a/litellm/proxy/guardrails/guardrail_registry.py
+++ b/litellm/proxy/guardrails/guardrail_registry.py
@@ -1,5 +1,6 @@
# litellm/proxy/guardrails/guardrail_registry.py
+import asyncio
import importlib
import os
from collections.abc import Callable, Iterator, Mapping
@@ -813,4 +814,6 @@ class InMemoryGuardrailHandler:
# In Memory Guardrail Handler for LiteLLM Proxy
########################################################
IN_MEMORY_GUARDRAIL_HANDLER: Final = InMemoryGuardrailHandler()
+
+GUARDRAIL_RECONCILE_LOCK: Final = asyncio.Lock()
########################################################
diff --git a/litellm/proxy/hooks/batch_rate_limiter.py b/litellm/proxy/hooks/batch_rate_limiter.py
index 7e33583fc9d..d6229fb80a6 100644
--- a/litellm/proxy/hooks/batch_rate_limiter.py
+++ b/litellm/proxy/hooks/batch_rate_limiter.py
@@ -18,11 +18,12 @@ Quick summary:
"""
import json
-from collections.abc import Iterable
-from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn
+from collections.abc import Iterable, Mapping, Sequence
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, TypeAlias
from fastapi import HTTPException
-from pydantic import BaseModel
+from pydantic import BaseModel, Field, TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
@@ -40,10 +41,15 @@ from litellm.proxy._types import (
SpecialModelNames,
UserAPIKeyAuth,
)
+from litellm.proxy.auth.auth_utils import get_model_rate_limit_from_metadata
from litellm.proxy.common_utils.proxy_rate_limit_error import (
ProxyRateLimitError,
map_v3_rate_limit_type,
)
+from litellm.proxy.hooks.parallel_request_limiter_v3 import (
+ PROJECT_ITPM_DESCRIPTOR_KEY,
+ PROJECT_OTPM_DESCRIPTOR_KEY,
+)
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
if TYPE_CHECKING:
@@ -76,6 +82,11 @@ else:
RateLimitDescriptor = dict[str, Any]
+_BATCH_BODY_ADAPTER: Final = TypeAdapter(dict[str, object])
+
+IncrementAmounts: TypeAlias = dict[Literal["requests", "tokens"], int]
+
+
class BatchFileUsage(BaseModel):
"""
Internal model for batch file usage tracking, used for batch rate limiting
@@ -83,6 +94,16 @@ class BatchFileUsage(BaseModel):
total_tokens: int
request_count: int
+ output_tokens: int = 0
+ # Keyed by each row's own `body.model`, distinct from `total_tokens`/
+ # `output_tokens` (the whole-file totals charged to the file-bound/
+ # top-level routing model's key/team/model limits). A batch's rows can
+ # each target a different model, so the project's per-model ITPM/OTPM
+ # quota for a row's actual model must be charged with that row's own
+ # tokens -- see `_create_project_io_descriptors_for_models`.
+ per_model_usage: dict[str, dict[str, int]] = Field(
+ default_factory=dict
+ ) # mutable-ok: accumulated incrementally per row while parsing the batch file
class _PROXY_BatchRateLimiter(CustomLogger):
@@ -198,6 +219,15 @@ class _PROXY_BatchRateLimiter(CustomLogger):
user_api_key_dict: UserAPIKeyAuth,
data: dict,
) -> list["RateLimitDescriptor"]:
+ """Build the standard key/user/team/model descriptor list a batch is charged against.
+
+ Deliberately excludes the project-scoped ITPM/OTPM descriptors: those
+ are charged per the JSONL row's own `body.model` once the file is
+ parsed (`_create_project_io_descriptors_for_models`), not the
+ file-bound/top-level routing model this function resolves. Charging
+ project quotas here would let a caller bind the file to a model
+ without a quota while rows execute against a quota-limited model.
+ """
return self.parallel_request_limiter._create_rate_limit_descriptors(
user_api_key_dict=user_api_key_dict,
data=data,
@@ -206,6 +236,57 @@ class _PROXY_BatchRateLimiter(CustomLogger):
model_has_failures=False,
)
+ @staticmethod
+ def _project_has_any_io_token_limits(user_api_key_dict: UserAPIKeyAuth) -> bool:
+ """True when the project has any per-model ITPM/OTPM quota configured.
+
+ Used to stop the "skip batch input file processing" fast path from
+ bypassing a project quota configured for a model other than the
+ batch's file-bound/top-level routing model: the row models that
+ actually drive execution and billing aren't known until the JSONL
+ is parsed, so the file must be read whenever *any* model could be
+ quota-limited, not only when the routing model itself is.
+ """
+ if user_api_key_dict.project_id is None:
+ return False
+ return bool(
+ get_model_rate_limit_from_metadata(user_api_key_dict, "project_metadata", "model_itpm_limit")
+ ) or bool(get_model_rate_limit_from_metadata(user_api_key_dict, "project_metadata", "model_otpm_limit"))
+
+ def _create_project_io_descriptors_for_models(
+ self,
+ user_api_key_dict: UserAPIKeyAuth,
+ per_model_usage: Mapping[str, Mapping[str, int]],
+ ) -> tuple[list["RateLimitDescriptor"], list[IncrementAmounts]]: # mutable-ok: see below
+ """Build project ITPM/OTPM descriptors charged against each row's own model.
+
+ One descriptor pair per distinct `body.model` found in the JSONL,
+ each incremented only by that model's own counted usage -- never the
+ whole-batch total -- so a quota-limited model can't hide behind an
+ unlimited routing model, and an unrelated model's rows can't inflate
+ a different model's counter.
+ """
+ extra_descriptors: Final[list[RateLimitDescriptor]] = [] # mutable-ok: see above
+ extra_increments: Final[list[IncrementAmounts]] = [] # mutable-ok: see above
+ for model, usage in per_model_usage.items():
+ model_descriptors: list[RateLimitDescriptor] = [] # mutable-ok: reset per loop iteration, not module state
+ self.parallel_request_limiter.add_project_io_token_rate_limit_descriptors_from_metadata(
+ user_api_key_dict=user_api_key_dict,
+ requested_model=model,
+ descriptors=model_descriptors,
+ )
+ for descriptor in model_descriptors:
+ extra_descriptors.append(descriptor)
+ extra_increments.append(
+ { # mutable-ok: atomic limiter API requires mutable increment records
+ "requests": 0,
+ "tokens": usage.get("output_tokens", 0)
+ if descriptor["key"] == PROJECT_OTPM_DESCRIPTOR_KEY
+ else usage.get("total_tokens", 0),
+ }
+ )
+ return extra_descriptors, extra_increments
+
def _should_skip_batch_input_file_processing(
self,
data: dict,
@@ -232,6 +313,11 @@ class _PROXY_BatchRateLimiter(CustomLogger):
routing deployment's trusted credentials and the batch is constrained
to run on that provider.
+ The no-limits check also treats any project-configured ITPM/OTPM
+ quota as an applicable limit, even when it isn't scoped to the
+ routing model: a row can target a different, quota-limited model,
+ and that isn't knowable without parsing the JSONL.
+
Returns ``(should_skip, descriptors)`` where ``descriptors`` is the
rate-limit descriptor list computed for the no-limits check, so the
caller can reuse it for counter enforcement without recomputing.
@@ -257,7 +343,9 @@ class _PROXY_BatchRateLimiter(CustomLogger):
user_api_key_dict=user_api_key_dict,
data=data,
)
- if not self._has_applicable_batch_rate_limits(descriptors):
+ if not self._has_applicable_batch_rate_limits(descriptors) and not self._project_has_any_io_token_limits(
+ user_api_key_dict
+ ):
verbose_proxy_logger.debug("Skipping batch input file processing: no rate limits configured")
return True, None
@@ -297,6 +385,58 @@ class _PROXY_BatchRateLimiter(CustomLogger):
return False
return True
+ def _estimate_entry_output_tokens(
+ self,
+ entry: Mapping[str, object],
+ min_configured_otpm_limit: int | None,
+ ) -> int:
+ """Conservative per-row output-token estimate for the project OTPM reservation.
+
+ Batch completion never reconciles actual usage back into the rate
+ limiter, so this pre-call estimate is the only OTPM enforcement a
+ batch gets. Mirrors the real-time no-``max_tokens`` floor so a row
+ that omits an output cap can't be used to bypass OTPM the way an
+ unbounded streaming request could.
+
+ Embeddings rows are identified by the row's own ``url`` (the OpenAI
+ batch schema puts the target route there, e.g. ``/v1/embeddings``),
+ never by body shape: a `/v1/responses` row also carries `body.input`
+ with no `messages`/`prompt`, so guessing from body shape alone would
+ misclassify a token-generating Responses row as a zero-output
+ embeddings row and let it skip the OTPM reservation entirely.
+ """
+ url: Final = entry.get("url")
+ if isinstance(url, str) and "embeddings" in url:
+ return 0 # embeddings: no output tokens
+ raw_body: Final = entry.get("body")
+ body: Final[Mapping[str, object]] = (
+ MappingProxyType(_BATCH_BODY_ADAPTER.validate_python(raw_body))
+ if isinstance(raw_body, Mapping)
+ else MappingProxyType({}) # mutable-ok: immediately frozen empty fallback
+ )
+ # `max_tokens`/`max_completion_tokens` cap chat completions; `/v1/responses`
+ # rows cap output with `max_output_tokens` instead -- omitting it here
+ # would fall through to the floor estimate for every capped Responses row.
+ explicit_cap: Final = next(
+ (
+ v
+ for v in (
+ body.get("max_tokens"),
+ body.get("max_completion_tokens"),
+ body.get("max_output_tokens"),
+ )
+ if v is not None
+ ),
+ None,
+ )
+ candidate_count: Final = self.parallel_request_limiter.get_output_candidate_count(body)
+ if explicit_cap is not None:
+ try:
+ return max(0, int(explicit_cap)) * candidate_count
+ except (TypeError, ValueError, OverflowError):
+ pass
+ return self.parallel_request_limiter.no_max_tokens_output_floor(min_configured_otpm_limit) * candidate_count
+
@staticmethod
def _has_applicable_batch_rate_limits(
descriptors: list["RateLimitDescriptor"],
@@ -382,9 +522,22 @@ class _PROXY_BatchRateLimiter(CustomLogger):
"""Raise :class:`ProxyRateLimitError` (a 429) for batch rate limit exceeded."""
from datetime import datetime
- # Find the descriptor for this status
+ # Find the descriptor for this status. Matching on (key, value) is
+ # required, not key alone: a batch can carry several project ITPM/OTPM
+ # descriptors sharing one key (e.g. `model_per_project_otpm`) but
+ # scoped to different models via `value`
+ # ("{project_id}:{model}") -- key-only matching would always resolve
+ # to the first same-keyed descriptor regardless of which one was
+ # actually over its limit. Falls back to key-only matching for
+ # statuses that predate `descriptor_value` (e.g. from should_rate_limit).
+ status_descriptor_value: Final = status.get("descriptor_value")
descriptor_index: Final = next(
- (i for i, d in enumerate(descriptors) if d.get("key") == status.get("descriptor_key")),
+ (
+ i
+ for i, d in enumerate(descriptors)
+ if d.get("key") == status.get("descriptor_key")
+ and (status_descriptor_value is None or d.get("value") == status_descriptor_value)
+ ),
0,
)
descriptor: Final[RateLimitDescriptor] = (
@@ -407,9 +560,27 @@ class _PROXY_BatchRateLimiter(CustomLogger):
f"Limit resets at: {reset_time_formatted}"
)
else: # tokens
+ # Project ITPM/OTPM descriptors are keyed "{project_id}:{model}" and
+ # charged with that model's own rows (see
+ # `_create_project_io_descriptors_for_models`), not the whole
+ # batch's totals -- report the matching per-model figure when one
+ # is available so the error reflects what was actually charged.
+ descriptor_model: Final = (
+ descriptor.get("value", "").split(":", 1)[-1]
+ if descriptor.get("key") in (PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY)
+ else None
+ )
+ model_usage: Final = batch_usage.per_model_usage.get(descriptor_model) if descriptor_model else None
+ batch_token_count: Final = (
+ (model_usage or {}).get("output_tokens", batch_usage.output_tokens)
+ if descriptor.get("key") == PROJECT_OTPM_DESCRIPTOR_KEY
+ else (model_usage or {}).get("total_tokens", batch_usage.total_tokens)
+ if descriptor.get("key") == PROJECT_ITPM_DESCRIPTOR_KEY
+ else batch_usage.total_tokens
+ )
detail = (
f"Batch rate limit exceeded for {descriptor.get('key', 'unknown')}: {descriptor.get('value', 'unknown')}. "
- f"Batch contains {batch_usage.total_tokens} tokens but only {remaining_display} tokens remaining "
+ f"Batch contains {batch_token_count} tokens but only {remaining_display} tokens remaining "
f"out of {current_limit} TPM limit. "
f"Limit resets at: {reset_time_formatted}"
)
@@ -444,7 +615,10 @@ class _PROXY_BatchRateLimiter(CustomLogger):
falls back to a per-process asyncio.Lock + in-memory operation.
``descriptors`` may be passed in by the pre-call hook to reuse the list
- already computed when deciding whether to skip file processing.
+ already computed when deciding whether to skip file processing. It
+ never contains project ITPM/OTPM descriptors (those are model-specific
+ and only knowable once ``batch_usage.per_model_usage`` is populated by
+ parsing the JSONL), so this always builds and appends them here.
"""
if descriptors is None:
descriptors = self._create_batch_rate_limit_descriptors(
@@ -452,11 +626,20 @@ class _PROXY_BatchRateLimiter(CustomLogger):
data=data,
)
- increment: Final[dict[Literal["requests", "tokens"], int]] = {
- "requests": batch_usage.request_count,
- "tokens": batch_usage.total_tokens,
- }
- increments: Final[list[dict[Literal["requests", "tokens"], int]]] = [increment for _ in descriptors]
+ increments: list[IncrementAmounts] = [ # mutable-ok: reassigned below to append project IO increments
+ { # mutable-ok: atomic limiter API requires mutable increment records
+ "requests": batch_usage.request_count,
+ "tokens": batch_usage.total_tokens,
+ }
+ for _d in descriptors
+ ]
+
+ project_io_descriptors, project_io_increments = self._create_project_io_descriptors_for_models(
+ user_api_key_dict=user_api_key_dict,
+ per_model_usage=batch_usage.per_model_usage,
+ )
+ descriptors = [*descriptors, *project_io_descriptors]
+ increments = [*increments, *project_io_increments]
rate_limit_response: Final = await self.parallel_request_limiter.atomic_check_and_increment_by_n(
descriptors=descriptors,
@@ -482,6 +665,7 @@ class _PROXY_BatchRateLimiter(CustomLogger):
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
user_api_key_dict: UserAPIKeyAuth | None = None,
data: dict | None = None,
+ descriptors: Sequence["RateLimitDescriptor"] | None = None,
) -> BatchFileUsage:
"""
Count number of requests and tokens in a batch input file.
@@ -490,10 +674,37 @@ class _PROXY_BatchRateLimiter(CustomLogger):
file_id: The file ID to read
custom_llm_provider: The custom LLM provider to use for token encoding
user_api_key_dict: User authentication information for file access (required for managed files)
+ descriptors: Rate limit descriptors already computed for this batch, so the
+ configured project OTPM limit can scale the no-``max_tokens`` output floor
Returns:
- BatchFileUsage with total_tokens and request_count
+ BatchFileUsage with total_tokens, output_tokens, request_count, and
+ per_model_usage (each row's own totals, keyed by its `body.model`)
"""
+ descriptor_otpm_limits: Final = tuple(
+ int(v)
+ for d in (descriptors or ())
+ if d.get("key") == PROJECT_OTPM_DESCRIPTOR_KEY
+ for rate_limit in (d.get("rate_limit"),)
+ for v in (rate_limit.get("tokens_per_unit") if rate_limit is not None else None,)
+ if v is not None
+ )
+ # `descriptors` only ever carries the routing model's own OTPM limit
+ # (see `_create_batch_rate_limit_descriptors`), but a row can target
+ # any project-configured model. Folding in every configured model's
+ # OTPM limit keeps the no-`max_tokens` floor from drifting wide just
+ # because a row's specific model isn't known until parsed below.
+ project_otpm_limits: Final = (
+ tuple(int(v) for v in project_otpm_limit_map.values())
+ if user_api_key_dict is not None
+ and (
+ project_otpm_limit_map := get_model_rate_limit_from_metadata(
+ user_api_key_dict, "project_metadata", "model_otpm_limit"
+ )
+ )
+ else ()
+ )
+ min_configured_otpm_limit: Final = min((*descriptor_otpm_limits, *project_otpm_limits), default=None)
try:
# Check if this is a managed file (base64 encoded unified file ID)
from litellm.proxy.openai_files_endpoints.common_utils import (
@@ -545,23 +756,51 @@ class _PROXY_BatchRateLimiter(CustomLogger):
# Counting stays best-effort, so a legitimate (e.g. multimodal) row
# the counter can't measure is estimated, not hard-rejected.
models: Final[set] = set()
+ # Keyed by each row's own `body.model`, so the project ITPM/OTPM
+ # quota for that model is charged with only its own rows' tokens,
+ # never the whole batch's -- see `_create_project_io_descriptors_for_models`.
+ per_model_usage: Final[dict[str, dict[str, int]]] = {}
total_tokens = 0
+ output_tokens = 0 # rebind-ok: accumulated per JSONL row in the loop below
request_count = 0
for raw_line in _iter_batch_input_lines(file_content_bytes):
request_count += 1
try:
entry = json.loads(raw_line)
except Exception:
- total_tokens += _estimate_batch_entry_tokens(raw_line)
+ entry_total_tokens = _estimate_batch_entry_tokens(raw_line)
+ entry_output_tokens = self.parallel_request_limiter.no_max_tokens_output_floor(
+ min_configured_otpm_limit
+ )
+ total_tokens += entry_total_tokens
+ output_tokens += entry_output_tokens
continue
+
+ model: str | None = (entry.get("body") or {}).get("model") if isinstance(entry, dict) else None
+ if model:
+ models.add(model)
+
if isinstance(entry, dict):
- model = (entry.get("body") or {}).get("model")
- if model:
- models.add(model)
+ entry_output_tokens = self._estimate_entry_output_tokens(entry, min_configured_otpm_limit)
+ else:
+ entry_output_tokens = self.parallel_request_limiter.no_max_tokens_output_floor(
+ min_configured_otpm_limit
+ )
+ output_tokens += entry_output_tokens
+
try:
- total_tokens += _count_entry_tokens(entry)
+ entry_total_tokens = _count_entry_tokens(entry)
except Exception:
- total_tokens += _estimate_batch_entry_tokens(raw_line)
+ entry_total_tokens = _estimate_batch_entry_tokens(raw_line)
+ total_tokens += entry_total_tokens
+
+ if model:
+ model_usage = per_model_usage.setdefault(
+ model, {"total_tokens": 0, "output_tokens": 0, "request_count": 0}
+ )
+ model_usage["total_tokens"] += entry_total_tokens
+ model_usage["output_tokens"] += entry_output_tokens
+ model_usage["request_count"] += 1
# Validate every model named in the batch JSONL against the
# caller's per-key model allowlist. Without this, a caller
@@ -578,6 +817,8 @@ class _PROXY_BatchRateLimiter(CustomLogger):
return BatchFileUsage(
total_tokens=total_tokens,
request_count=request_count,
+ output_tokens=output_tokens,
+ per_model_usage=per_model_usage,
)
except HTTPException as e:
@@ -814,6 +1055,7 @@ class _PROXY_BatchRateLimiter(CustomLogger):
custom_llm_provider=custom_llm_provider,
user_api_key_dict=user_api_key_dict,
data=data,
+ descriptors=batch_rate_limit_descriptors,
)
verbose_proxy_logger.debug(
diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py
index 4492f42782c..de8834449de 100644
--- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py
+++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py
@@ -454,7 +454,9 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger):
parent_otel_span=user_api_key_dict.parent_otel_span,
)
- verbose_proxy_logger.debug("Atomic check+increment response: %s", json.dumps(atomic_response, indent=2))
+ verbose_proxy_logger.debug(
+ "Atomic check+increment response: %s", json.dumps(atomic_response, indent=2, default=list)
+ )
if atomic_response["overall_code"] == "OVER_LIMIT":
resolved_model, llm_provider = resolve_llm_provider_for_rate_limit(model)
diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py
index 94ef08782d9..2d799ded752 100644
--- a/litellm/proxy/hooks/parallel_request_limiter_v3.py
+++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py
@@ -8,7 +8,7 @@ import asyncio
import binascii
import os
import uuid
-from collections.abc import Callable, Sequence
+from collections.abc import Callable, Mapping, Sequence, Set
from contextvars import ContextVar
from dataclasses import dataclass, field
from datetime import datetime
@@ -22,6 +22,8 @@ from typing import (
TypedDict,
)
+from typing_extensions import NotRequired, ReadOnly
+
from litellm import DualCache
from litellm._logging import verbose_proxy_logger
from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, INTERNAL_CALL_ORIGIN_METADATA_KEY
@@ -44,11 +46,12 @@ from litellm.proxy.common_utils.proxy_rate_limit_error import (
)
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
from litellm.types.caching import RedisPipelineIncrementOperation
-from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject
+from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject, ResponseAPIUsage
from litellm.types.utils import (
CallTypes,
EmbeddingResponse,
ModelResponse,
+ RerankResponse,
TextCompletionResponse,
Usage,
)
@@ -66,6 +69,7 @@ else:
Span = Any
InternalUsageCache = Any
+
BATCH_RATE_LIMITER_SCRIPT: Final = """
local results = {}
local now = tonumber(ARGV[1])
@@ -120,7 +124,8 @@ CHECK_AND_INCREMENT_BY_N_SCRIPT: Final = """
-- ARGV[(i-1)*4 + 3] = ttl_seconds (counter TTL when window resets)
-- ARGV[(i-1)*4 + 4] = window_size_seconds (sliding-window length)
--
--- Return on success: { 0, new_counter_1, new_counter_2, ... }
+-- Return on success:
+-- { 0, new_counter_1, window_start_1, new_counter_2, window_start_2, ... }
-- Return on over-limit: { 1, descriptor_index, current_counter, limit }
local time_reply = redis.call('TIME')
local now = tonumber(time_reply[1])
@@ -157,7 +162,7 @@ for i = 1, descriptor_count do
return { 1, i, current_counter, limit }
end
- descriptor_state[i] = { window_expired, current_counter }
+ descriptor_state[i] = { window_expired, current_counter, window_start }
end
-- Pass 2: all checks passed. Apply increments.
@@ -171,8 +176,10 @@ for i = 1, descriptor_count do
local window_size = tonumber(ARGV[arg_base + 3])
local window_expired = descriptor_state[i][1]
+ local active_window_start
if window_expired then
+ active_window_start = now
redis.call('SET', window_key, tostring(now))
redis.call('SET', counter_key, increment)
redis.call('EXPIRE', window_key, window_size)
@@ -181,6 +188,7 @@ for i = 1, descriptor_count do
end
table.insert(results, increment)
else
+ active_window_start = tonumber(descriptor_state[i][3])
local new_counter = redis.call('INCRBY', counter_key, increment)
local current_ttl = redis.call('TTL', counter_key)
if current_ttl == -1 and ttl > 0 then
@@ -188,11 +196,39 @@ for i = 1, descriptor_count do
end
table.insert(results, new_counter)
end
+ table.insert(results, active_window_start)
end
return results
"""
+WINDOW_GUARDED_TOKEN_INCREMENT_SCRIPT: Final = """
+local results = {}
+for i = 1, #KEYS, 2 do
+ local window_key = KEYS[i]
+ local counter_key = KEYS[i + 1]
+ local arg_base = ((i - 1) / 2) * 3 + 1
+ local expected_window_start = ARGV[arg_base]
+ local increment = tonumber(ARGV[arg_base + 1])
+ local ttl = tonumber(ARGV[arg_base + 2])
+ local active_window_start = redis.call('GET', window_key)
+
+ if active_window_start and active_window_start == expected_window_start then
+ local new_counter = redis.call('INCRBY', counter_key, increment)
+ local current_ttl = redis.call('TTL', counter_key)
+ if current_ttl == -1 and ttl > 0 then
+ redis.call('EXPIRE', counter_key, ttl)
+ end
+ table.insert(results, 1)
+ table.insert(results, new_counter)
+ else
+ table.insert(results, 0)
+ table.insert(results, tonumber(redis.call('GET', counter_key) or 0))
+ end
+end
+return results
+"""
+
PARALLEL_ACQUIRE_SCRIPT: Final = """
-- Atomic check-and-acquire for the max_parallel_requests concurrency gauge.
-- Each gauge key is a sorted set of per-request slot ids scored by acquire
@@ -297,6 +333,38 @@ DEFAULT_CHARS_PER_TOKEN: Final = 4
# (baseline floor) and to the smallest configured TPM limit (capped floor for
# small per-tenant TPM caps).
_TPM_FLOOR_FRACTION: Final = 4
+# Both embeddings and the Responses API put their prompt in data["input"],
+# but only embeddings have no output tokens. Every "is this an embedding"
+# check on data["input"] must exclude these call types, or a Responses call
+# gets misclassified as an embedding and skips output-token reservation/caps.
+RESPONSES_API_CALL_TYPES: Final = ("aresponses", "responses")
+EMBEDDING_API_CALL_TYPES: Final = ("aembedding", "embedding")
+TEXT_COMPLETION_API_CALL_TYPES: Final = ("atext_completion", "text_completion")
+RERANK_API_CALL_TYPES: Final = (CallTypes.rerank.value, CallTypes.arerank.value)
+GOOGLE_GENAI_NATIVE_CALL_TYPES: Final = (
+ CallTypes.generate_content.value,
+ CallTypes.agenerate_content.value,
+ CallTypes.generate_content_stream.value,
+ CallTypes.agenerate_content_stream.value,
+)
+RESPONSES_API_MIN_OUTPUT_TOKENS: Final = 16
+# litellm.token_counter has no per-type handling for "input_audio" content
+# blocks (unlike images, which use use_default_image_token_count) -- it
+# silently contributes 0 tokens for them. When the block carries a base64
+# payload, the estimate is derived from the decoded byte count; when the
+# block is a reference without a payload (or the payload is missing), this
+# flat per-block floor is used instead.
+DEFAULT_AUDIO_TOKEN_ESTIMATE: Final = 300
+# Conservative bytes-per-token assumption for size-based audio estimation:
+# equivalent to 8 kHz mono PCM-16 (16 000 bytes/s) at 10 tokens/s. Choosing
+# the lowest reasonable bitrate means we never under-reserve for higher-
+# quality audio recorded at the same wall-clock duration.
+_AUDIO_BYTES_PER_TOKEN: Final = 1600
+# Descriptor "key" values for project-scoped ITPM/OTPM. Distinct from
+# "model_per_project" (the combined-TPM descriptor) so both can be enforced
+# on the same project+model simultaneously without colliding on cache keys.
+PROJECT_ITPM_DESCRIPTOR_KEY: Final = "model_per_project_itpm"
+PROJECT_OTPM_DESCRIPTOR_KEY: Final = "model_per_project_otpm"
# How long an acquired slot counts toward the in-flight total before it is
# considered leaked (worker crashed without any release callback firing) and
# pruned. Also the longest request duration the gauge can track: a request
@@ -341,11 +409,24 @@ class RateLimitStatus(TypedDict):
limit_remaining: int
rate_limit_type: Literal["requests", "tokens", "max_parallel_requests"]
descriptor_key: str
+ # Only populated by the atomic_check_and_increment_by_n path. A caller
+ # matching a status back to its descriptor must key on (descriptor_key,
+ # descriptor_value) when this is present, not descriptor_key alone --
+ # e.g. a batch charging several models' project ITPM/OTPM in one call
+ # produces multiple statuses sharing the same descriptor_key.
+ descriptor_value: NotRequired[ReadOnly[str]]
class RateLimitResponse(TypedDict):
overall_code: str
statuses: list[RateLimitStatus]
+ reservation_windows: NotRequired[ReadOnly[frozenset[tuple[str, str, Literal["redis", "local"]]]]]
+
+
+class ReservationAwareIncrementOperation(RedisPipelineIncrementOperation):
+ window_key: NotRequired[str]
+ expected_window_start: NotRequired[str]
+ reservation_backend: NotRequired[Literal["redis", "local"]]
class RateLimitResponseWithDescriptors(TypedDict):
@@ -353,6 +434,10 @@ class RateLimitResponseWithDescriptors(TypedDict):
response: RateLimitResponse
+class _RateLimitDescriptorSink(Protocol):
+ def append(self, descriptor: RateLimitDescriptor, /) -> None: ...
+
+
class WindowKeyMetadata(TypedDict):
requests_limit: int | None
tokens_limit: int | None
@@ -362,6 +447,7 @@ class WindowKeyMetadata(TypedDict):
class AtomicCounterMeta(TypedDict):
descriptor_key: str
+ descriptor_value: ReadOnly[str]
current_limit: int
rate_limit_type: Literal["requests", "tokens"]
window_key: str
@@ -374,6 +460,7 @@ class AtomicCounterMeta(TypedDict):
class AtomicCounterState(TypedDict):
window_expired: bool
current: int
+ window_start: ReadOnly[str]
DescriptorAtomicGroup: TypeAlias = tuple[list[str], list[int], list[AtomicCounterMeta]]
@@ -418,6 +505,16 @@ class RequestRateLimiterStash:
reserved_tokens: int = 0
reserved_model: str | None = None
reserved_scopes: frozenset[tuple[str, str]] = field(default_factory=frozenset)
+ itpm_reserved_tokens: int = 0
+ itpm_reserved_scopes: frozenset[tuple[str, str]] = field(default_factory=frozenset)
+ itpm_reserved_window_identities: frozenset[tuple[str, str, Literal["redis", "local"]]] = field(
+ default_factory=frozenset
+ )
+ otpm_reserved_tokens: int = 0
+ otpm_reserved_scopes: frozenset[tuple[str, str]] = field(default_factory=frozenset)
+ otpm_reserved_window_identities: frozenset[tuple[str, str, Literal["redis", "local"]]] = field(
+ default_factory=frozenset
+ )
reservation_released: bool = False
@@ -462,21 +559,13 @@ def _call_id_from_callback_kwargs(kwargs: object) -> str | None:
return call_id if isinstance(call_id, str) else None
-def _declared_output_budget(value: object) -> int | None:
- """Coerce a declared output budget to tokens, or None when it names no budget.
-
- Accepts every shape the pre-existing ``int(...)`` coercion did, floats and numeric
- strings included, because a budget this cannot read is a budget this cannot reserve
- against, which is the bypass the caller-declared limits are checked for.
- """
- if isinstance(value, (int, float)):
- return int(value)
- if isinstance(value, str):
- try:
- return int(float(value))
- except ValueError:
- return None
- return None
+def _parse_output_cap_value(raw_value: object) -> int | None:
+ if isinstance(raw_value, bool) or not isinstance(raw_value, (int, float, str)):
+ return None
+ try:
+ return int(float(raw_value))
+ except (ValueError, OverflowError):
+ return None
class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
@@ -497,6 +586,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
self.check_and_increment_by_n_script = (
self.internal_usage_cache.dual_cache.redis_cache.async_register_script(CHECK_AND_INCREMENT_BY_N_SCRIPT)
)
+ self.window_guarded_token_increment_script = (
+ self.internal_usage_cache.dual_cache.redis_cache.async_register_script(
+ WINDOW_GUARDED_TOKEN_INCREMENT_SCRIPT
+ )
+ )
self.parallel_acquire_script = self.internal_usage_cache.dual_cache.redis_cache.async_register_script(
PARALLEL_ACQUIRE_SCRIPT
)
@@ -510,6 +604,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
self.batch_rate_limiter_script = None
self.token_increment_script = None
self.check_and_increment_by_n_script = None
+ self.window_guarded_token_increment_script = None
self.parallel_acquire_script = None
self.parallel_release_script = None
self.parallel_count_script = None
@@ -562,7 +657,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
return self._time_provider()
@staticmethod
- def _no_max_tokens_output_floor(
+ def no_max_tokens_output_floor(
min_configured_tpm_limit: int | None,
) -> int:
"""Output-budget floor used when the request omits max_tokens.
@@ -576,11 +671,164 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
return baseline
return min(baseline, max(1, min_configured_tpm_limit // _TPM_FLOOR_FRACTION))
+ @staticmethod
+ def _is_embedding_request(data: object, call_type: str | None) -> bool:
+ if call_type in EMBEDDING_API_CALL_TYPES:
+ return True
+ if call_type in RESPONSES_API_CALL_TYPES:
+ return False
+ if call_type:
+ return False
+ if not isinstance(data, dict):
+ return False
+ return data.get("input") is not None
+
+ @staticmethod
+ def _translate_google_genai_native_request(
+ data: object,
+ call_type: str | None,
+ ) -> Mapping[str, object] | None:
+ contents: Final = data.get("contents") if isinstance(data, dict) else None
+ if (
+ not isinstance(data, dict)
+ or call_type not in GOOGLE_GENAI_NATIVE_CALL_TYPES
+ or not isinstance(contents, (dict, list))
+ ):
+ return None
+ from litellm.google_genai.adapters.transformation import GoogleGenAIAdapter
+
+ config: Final = data.get("config") if "config" in data else data.get("generationConfig")
+ return GoogleGenAIAdapter().translate_generate_content_to_completion(
+ model=data.get("model") if isinstance(data.get("model"), str) else "",
+ contents=contents,
+ config=config if isinstance(config, dict) else None,
+ systemInstruction=data.get("systemInstruction"),
+ system_instruction=data.get("system_instruction"),
+ tools=data.get("tools"),
+ toolConfig=data.get("toolConfig"),
+ tool_config=data.get("tool_config"),
+ )
+
+ @staticmethod
+ def _get_explicit_output_cap(data: object, call_type: str | None) -> int | None:
+ if not isinstance(data, dict):
+ return None
+ if call_type in GOOGLE_GENAI_NATIVE_CALL_TYPES:
+ config: Final = data.get("config") if "config" in data else data.get("generationConfig")
+ google_cap_values: Final = tuple(
+ parsed
+ for field in ("maxOutputTokens", "max_output_tokens")
+ if isinstance(config, dict)
+ for parsed in (_parse_output_cap_value(config.get(field)),)
+ if parsed is not None
+ )
+ return max(google_cap_values, default=None)
+ if call_type in RESPONSES_API_CALL_TYPES:
+ responses_cap: Final = _parse_output_cap_value(data.get("max_output_tokens"))
+ if responses_cap is None:
+ return None
+ return max(RESPONSES_API_MIN_OUTPUT_TOKENS, responses_cap)
+ if call_type in EMBEDDING_API_CALL_TYPES:
+ return None
+ fields: Final = (
+ ("max_tokens", "max_completion_tokens")
+ if call_type
+ else ("max_tokens", "max_completion_tokens", "max_output_tokens")
+ )
+ output_cap_values: Final = tuple(
+ parsed for field in fields for parsed in (_parse_output_cap_value(data.get(field)),) if parsed is not None
+ )
+ return max(output_cap_values, default=None)
+
+ @classmethod
+ def _has_explicit_output_cap(cls, data: object, call_type: str | None) -> bool:
+ """Whether the caller explicitly set an output-token cap.
+
+ Checked via ``is not None`` (not truthiness) so an explicit 0 --
+ a legitimate zero-output request -- counts as explicit.
+ """
+ return cls._get_explicit_output_cap(data, call_type) is not None
+
+ @staticmethod
+ def get_output_candidate_count(data: object, call_type: str | None = None) -> int:
+ if not isinstance(data, Mapping):
+ return 1
+ config: Final = (
+ (data.get("config") if "config" in data else data.get("generationConfig"))
+ if call_type in GOOGLE_GENAI_NATIVE_CALL_TYPES
+ else None
+ )
+ candidate_values: Final = (
+ data.get("n"),
+ data.get("best_of"),
+ config.get("candidateCount") if isinstance(config, dict) else None,
+ config.get("candidate_count") if isinstance(config, dict) else None,
+ )
+ candidate_count = 1 # rebind-ok: running maximum across candidate-count aliases
+ for value in candidate_values:
+ try:
+ candidate_count = max(candidate_count, int(value or 1))
+ except (TypeError, ValueError, OverflowError):
+ continue
+ return candidate_count
+
+ @staticmethod
+ def _apply_implicit_output_cap(
+ data: object,
+ min_configured_limit: int | None,
+ call_type: str | None,
+ configured_output_tokens: int | None = None,
+ ) -> None:
+ """Hard-cap generation length when the request has no explicit cap.
+
+ Guards against an unbounded response overshooting a small TPM/OTPM
+ budget before post-call reconciliation runs. Skips requests that
+ already set an explicit cap and embeddings, which have no generation
+ budget. The Responses API only honors ``max_output_tokens`` (its
+ underlying chat-completion transformation ignores ``max_tokens``), so
+ the cap must be written to that field for Responses call types.
+
+ ``configured_output_tokens`` is the operator-declared per-tenant
+ estimate; when it exceeds the safety floor, the cap is raised to that
+ value instead of clamping every tenant to the same floor.
+ """
+ if not isinstance(data, dict):
+ return
+ base_capped_floor: Final = _PROXY_MaxParallelRequestsHandler_v3.no_max_tokens_output_floor(min_configured_limit)
+ capped_floor: Final = (
+ max(base_capped_floor, RESPONSES_API_MIN_OUTPUT_TOKENS)
+ if call_type in RESPONSES_API_CALL_TYPES
+ else base_capped_floor
+ )
+ baseline_floor: Final = DEFAULT_MAX_TOKENS_ESTIMATE // _TPM_FLOOR_FRACTION
+ is_embedding: Final = _PROXY_MaxParallelRequestsHandler_v3._is_embedding_request(data, call_type)
+ if (
+ capped_floor >= baseline_floor
+ or _PROXY_MaxParallelRequestsHandler_v3._has_explicit_output_cap(data, call_type)
+ or is_embedding
+ ):
+ return
+ effective_cap: Final = max(capped_floor, configured_output_tokens or 0)
+ if call_type in GOOGLE_GENAI_NATIVE_CALL_TYPES:
+ config_field: Final = "config" if "config" in data or "generationConfig" not in data else "generationConfig"
+ config: Final = data.get(config_field)
+ if config is None or isinstance(config, dict):
+ data[config_field] = { # rebind-ok: routed request needs cap # mutable-ok: downstream needs dict
+ **(config or {}), # mutable-ok: downstream native routing requires a mutable request config
+ "maxOutputTokens": effective_cap,
+ }
+ return
+ cap_field: Final = "max_output_tokens" if call_type in RESPONSES_API_CALL_TYPES else "max_tokens"
+ existing_cap: Final = data.get(cap_field)
+ if existing_cap is None or effective_cap < existing_cap:
+ data[cap_field] = effective_cap # rebind-ok: downstream routing requires the bounded output cap
+
def _estimate_tokens_for_request(
self,
data: dict,
model: str | None = None,
min_configured_tpm_limit: int | None = None,
+ call_type: str | None = None,
configured_output_tokens: int | None = None,
) -> int:
"""
@@ -588,7 +836,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
upfront (input + output budget):
estimated = input_tokens + max_tokens.
- Supports chat (messages), completions (prompt), and embeddings (input).
+ Supports chat (messages), completions (prompt), embeddings (input),
+ and the Responses API (also `input`, disambiguated from embeddings
+ via ``call_type``).
``min_configured_tpm_limit`` is the smallest ``tokens_per_unit`` among
the TPM-bearing descriptors this request will be charged against. When
@@ -601,78 +851,108 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
floor entirely, so the reservation reflects what this tenant's model
actually emits rather than one constant shared by every tenant.
"""
- messages = data.get("messages")
- prompt: Final = data.get("prompt")
- input_text: Final = data.get("input") # embeddings
-
- match (messages, prompt, input_text):
- case (messages, _, _) if messages:
- total_chars = len(get_str_from_messages(messages))
- case (_, str() as p, _):
- total_chars = len(p)
- case (_, list() as p, _):
- total_chars = sum(len(str(item)) for item in p)
- case (_, _, str() as t):
- total_chars = len(t)
- case (_, _, list() as t):
- total_chars = sum(len(str(item)) for item in t)
- case _:
- total_chars = 0
-
- estimated_input_tokens: Final = max(1, total_chars // DEFAULT_CHARS_PER_TOKEN) if total_chars > 0 else 0
-
- # Both spellings can arrive together, e.g. a deployment-level max_tokens default under a
- # client-supplied max_completion_tokens. Reserving against the larger keeps the estimate an
- # upper bound on what the provider can emit, whichever one it ends up honouring.
- declared_output_budgets: Final = tuple(
- budget
- for budget in (
- _declared_output_budget(data.get("max_tokens")),
- _declared_output_budget(data.get("max_completion_tokens")),
- )
- if budget is not None
+ estimated_input_tokens, max_tokens_estimate = self._estimate_input_and_output_tokens(
+ data=data,
+ min_configured_tpm_limit=min_configured_tpm_limit,
+ call_type=call_type,
+ configured_output_tokens=configured_output_tokens,
)
- explicit_max_tokens: Final = max(declared_output_budgets) if declared_output_budgets else None
-
- match (explicit_max_tokens, input_text):
- case (mt, _) if mt is not None:
- max_tokens_estimate = int(mt)
- case (_, embeddings_input) if embeddings_input:
- # Embeddings have no output tokens
- max_tokens_estimate = 0
- case _ if total_chars == 0 and configured_output_tokens is None:
- # Fully contentless request (no messages, prompt, or input).
- # Don't apply the conservative output-budget floor here — it
- # would over-reserve and could push small TPM limits into a
- # false 429. The caller floors at 1 so backpressure still
- # applies once the counter is at limit.
- max_tokens_estimate = 0
- case _:
- # No max_tokens specified — reserve at least the input size with a
- # conservative floor so a stream of small concurrent requests can't
- # collectively bypass the limit. Cap the floor by a fraction of
- # the smallest TPM limit this request will be charged against,
- # so a small per-tenant TPM cap can't be tripped by the floor
- # alone.
- output_floor: Final = self._no_max_tokens_output_floor(min_configured_tpm_limit)
- max_tokens_estimate = (
- configured_output_tokens
- if configured_output_tokens is not None
- else max(estimated_input_tokens, output_floor)
- )
-
total_estimated: Final = estimated_input_tokens + max_tokens_estimate
verbose_proxy_logger.debug(
- "TPM reservation estimate: input=%s, max_tokens=%s (explicit=%s), total=%s",
+ "TPM reservation estimate: input=%s, max_tokens=%s, total=%s",
estimated_input_tokens,
max_tokens_estimate,
- explicit_max_tokens is not None,
total_estimated,
)
return total_estimated
+ def _estimate_input_and_output_tokens(
+ self,
+ data: object,
+ min_configured_tpm_limit: int | None = None,
+ call_type: str | None = None,
+ configured_output_tokens: int | None = None,
+ ) -> tuple[int, int]:
+ """
+ Estimate input tokens and output (max_tokens) budget separately, so
+ callers needing independent ITPM/OTPM reservations (rather than one
+ combined TPM reservation) can use each half on its own.
+
+ ``min_configured_tpm_limit`` is the smallest ``tokens_per_unit`` among
+ the TPM-bearing descriptors this request will be charged against. When
+ provided, the no-``max_tokens`` output-budget floor is capped at a
+ fraction of that limit so small TPM caps remain usable. Omit to
+ preserve the unconstrained floor.
+
+ ``call_type`` disambiguates embeddings from the Responses API: both
+ put their prompt in ``data["input"]``, but only embeddings have no
+ output tokens. Unset (the default) preserves the historical
+ "any `input` means zero output" behavior for callers that don't have
+ a call type to pass.
+
+ ``configured_output_tokens`` is the operator-declared estimate resolved
+ from key or team metadata. When provided it replaces the heuristic
+ floor entirely, so the reservation reflects what this tenant's model
+ actually emits rather than one constant shared by every tenant.
+ """
+ if not isinstance(data, dict):
+ return 0, 0
+ translated_data: Final = self._translate_google_genai_native_request(data, call_type)
+ estimable_data: Final = translated_data if translated_data is not None else data
+ selected_fields: Final[tuple[object | None, object | None, object | None]] = (
+ (None, None, estimable_data.get("input"))
+ if call_type in RESPONSES_API_CALL_TYPES or call_type in EMBEDDING_API_CALL_TYPES
+ else (None, estimable_data.get("prompt"), None)
+ if call_type in TEXT_COMPLETION_API_CALL_TYPES
+ else (estimable_data.get("messages"), None, None)
+ if call_type
+ else (
+ estimable_data.get("messages"),
+ estimable_data.get("prompt"),
+ estimable_data.get("input"),
+ )
+ )
+ messages, prompt, input_text = selected_fields
+
+ total_chars: Final = (
+ len(get_str_from_messages(messages))
+ if isinstance(messages, list) and messages
+ else len(prompt)
+ if isinstance(prompt, str)
+ else sum(len(str(item)) for item in prompt)
+ if isinstance(prompt, list)
+ else len(input_text)
+ if isinstance(input_text, str)
+ else sum(len(str(item)) for item in input_text)
+ if isinstance(input_text, list)
+ else 0
+ )
+
+ estimated_input_tokens: Final = max(1, total_chars // DEFAULT_CHARS_PER_TOKEN) if total_chars > 0 else 0
+
+ explicit_max_tokens: Final = self._get_explicit_output_cap(data, call_type)
+ is_embedding: Final = self._is_embedding_request(data, call_type)
+
+ base_output_floor: Final = self.no_max_tokens_output_floor(min_configured_tpm_limit)
+ output_floor: Final = (
+ max(base_output_floor, RESPONSES_API_MIN_OUTPUT_TOKENS)
+ if call_type in RESPONSES_API_CALL_TYPES
+ else base_output_floor
+ )
+ max_tokens_estimate: Final = (
+ 0
+ if is_embedding or (explicit_max_tokens is None and total_chars == 0 and configured_output_tokens is None)
+ else explicit_max_tokens
+ if explicit_max_tokens is not None
+ else configured_output_tokens
+ if configured_output_tokens is not None
+ else max(estimated_input_tokens, output_floor)
+ )
+
+ return estimated_input_tokens, max_tokens_estimate * self.get_output_candidate_count(data, call_type)
+
def _is_redis_cluster(self) -> bool:
"""
Check if the dual cache is using Redis cluster.
@@ -933,7 +1213,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
async def should_rate_limit(
self,
- descriptors: list[RateLimitDescriptor],
+ descriptors: Sequence[RateLimitDescriptor],
parent_otel_span: Span | None = None,
read_only: bool = False,
skip_tpm_check: bool = False,
@@ -1059,7 +1339,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
def _collect_windowed_keys_and_gauges(
self,
- descriptors: list[RateLimitDescriptor],
+ descriptors: Sequence[RateLimitDescriptor],
skip_tpm_check: bool,
) -> tuple[list[str], dict[str, WindowKeyMetadata], list[ParallelRequestGauge]]:
"""
@@ -1463,6 +1743,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
meta.append(
{
"descriptor_key": descriptor_key,
+ "descriptor_value": descriptor_value,
"current_limit": int(limit_value),
"rate_limit_type": rlt,
"window_key": window_key,
@@ -1485,6 +1766,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
descriptor i, refund descriptors 0..i-1's increments. On Lua failure
mid-loop, refund applied increments and fall back to in-memory.
"""
+ if not descriptor_groups:
+ return RateLimitResponse(
+ overall_code="OK",
+ statuses=[], # mutable-ok: response contract requires a status list
+ )
applied: Final[list[list[AtomicCounterMeta]]] = []
statuses: Final[list[RateLimitStatus]] = []
raw: list[CacheCounterValue]
@@ -1519,10 +1805,16 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
if response["overall_code"] == "OVER_LIMIT":
await self._refund_applied_descriptor_groups(applied)
return response
+ if len(descriptor_groups) == 1:
+ return response
applied.append(meta)
statuses.extend(response["statuses"])
- return RateLimitResponse(overall_code="OK", statuses=statuses)
+ return RateLimitResponse(
+ overall_code="OK",
+ statuses=statuses,
+ reservation_windows=frozenset(),
+ )
async def _refund_applied_descriptor_groups(
self,
@@ -1585,12 +1877,14 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
limit_remaining=max(0, limit - current_counter),
rate_limit_type=meta["rate_limit_type"],
descriptor_key=meta["descriptor_key"],
+ descriptor_value=meta["descriptor_value"],
)
],
)
statuses: Final[list[RateLimitStatus]] = []
- for meta, new_counter in zip(per_counter_meta, raw[1:]):
+ for index, meta in enumerate(per_counter_meta):
+ new_counter = raw[1 + index * 2]
statuses.append(
RateLimitStatus(
code="OK",
@@ -1598,9 +1892,21 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
limit_remaining=max(0, meta["current_limit"] - int(new_counter)),
rate_limit_type=meta["rate_limit_type"],
descriptor_key=meta["descriptor_key"],
+ descriptor_value=meta["descriptor_value"],
)
)
- return RateLimitResponse(overall_code="OK", statuses=statuses)
+ return RateLimitResponse(
+ overall_code="OK",
+ statuses=statuses,
+ reservation_windows=frozenset(
+ (
+ meta["counter_key"],
+ str(int(raw[2 + index * 2])),
+ "redis",
+ )
+ for index, meta in enumerate(per_counter_meta)
+ ),
+ )
async def _atomic_check_and_increment_in_memory(
self,
@@ -1653,10 +1959,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
limit_remaining=max(0, meta["current_limit"] - current_counter),
rate_limit_type=meta["rate_limit_type"],
descriptor_key=meta["descriptor_key"],
+ descriptor_value=meta["descriptor_value"],
)
],
)
- descriptor_state.append({"window_expired": window_expired, "current": current_counter})
+ descriptor_state.append(
+ { # mutable-ok: local atomic-counter state is updated during pass two
+ "window_expired": window_expired,
+ "current": current_counter,
+ "window_start": str(now_int if window_expired else int(window_start)),
+ }
+ )
# Pass 2: apply increments.
statuses: Final[list[RateLimitStatus]] = []
@@ -1684,9 +1997,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
limit_remaining=max(0, meta["current_limit"] - new_counter),
rate_limit_type=meta["rate_limit_type"],
descriptor_key=meta["descriptor_key"],
+ descriptor_value=meta["descriptor_value"],
)
)
- return RateLimitResponse(overall_code="OK", statuses=statuses)
+ return RateLimitResponse(
+ overall_code="OK",
+ statuses=statuses,
+ reservation_windows=frozenset(
+ (meta["counter_key"], state["window_start"], "local")
+ for meta, state in zip(per_counter_meta, descriptor_state)
+ ),
+ )
async def reserve_tpm_tokens(
self,
@@ -1703,6 +2024,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
TPM-only descriptor/increment list and delegates the all-or-nothing
atomicity (Lua on Redis, asyncio-locked DualCache otherwise) to the
shared primitive.
+
+ Excludes project ITPM/OTPM descriptors -- those are reserved
+ separately (different estimate per bucket) via ``reserve_io_tokens``.
"""
tpm_descriptors: Final[list[RateLimitDescriptor]] = [
RateLimitDescriptor(
@@ -1714,7 +2038,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
),
)
for d in descriptors
- if (d.get("rate_limit") or {}).get("tokens_per_unit") is not None
+ if d["key"] not in (PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY)
+ and (d.get("rate_limit") or {}).get("tokens_per_unit") is not None # mutable-ok: optional descriptor
]
if not tpm_descriptors:
return RateLimitResponse(overall_code="OK", statuses=[])
@@ -1728,6 +2053,179 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
parent_otel_span=parent_otel_span,
)
+ async def _refund_reserved_tokens(
+ self,
+ scopes: Sequence[tuple[str, str]],
+ amount: int,
+ reservation_windows: frozenset[tuple[str, str, Literal["redis", "local"]]] = frozenset(),
+ parent_otel_span: Span | None = None,
+ ) -> None:
+ """
+ Directly decrement previously-reserved token counters for ``scopes``
+ by ``amount``. Used to roll back a reservation that already
+ succeeded once a *different* bucket in the same request turns out to
+ be over its limit (e.g. ITPM reserved fine, OTPM then hits its
+ limit -- the ITPM reservation must not be left inflated).
+ """
+ if amount <= 0 or not scopes:
+ return
+ if not reservation_windows:
+ await self.async_increment_tokens_with_ttl_preservation(
+ pipeline_operations=self._build_reservation_aware_tpm_ops(
+ targets=scopes,
+ reserved_scopes=frozenset(scopes),
+ actual_tokens=0,
+ reserved_tokens=amount,
+ ),
+ parent_otel_span=parent_otel_span,
+ )
+ return
+ pipeline_operations: Final = self._build_project_reservation_ops(
+ targets=scopes,
+ reserved_scopes=frozenset(scopes),
+ actual_tokens=0,
+ reserved_tokens=amount,
+ reservation_window_identities=reservation_windows,
+ )
+ await self.async_increment_reservation_aware_tokens(
+ pipeline_operations=pipeline_operations,
+ parent_otel_span=parent_otel_span,
+ )
+
+ async def reserve_io_tokens(
+ self,
+ descriptors: Sequence[RateLimitDescriptor],
+ estimated_input_tokens: int,
+ estimated_output_tokens: int,
+ parent_otel_span: Span | None = None,
+ ) -> tuple[RateLimitResponse, int, int]:
+ """
+ Reserve ``estimated_input_tokens`` against project ITPM descriptors
+ and ``estimated_output_tokens`` against project OTPM descriptors.
+
+ ITPM and OTPM are reserved from different-sized estimates, so unlike
+ same-size TPM descriptors they can't share a single
+ ``atomic_check_and_increment_by_n`` call -- each bucket gets its own
+ all-or-nothing atomic call. If the OTPM reservation is over limit
+ after ITPM already succeeded, the ITPM reservation this call made is
+ rolled back before returning, so a partial reservation never leaks.
+
+ Returns ``(response, itpm_reserved, otpm_reserved)`` -- the latter two
+ are the amounts actually reserved (0 if that bucket wasn't
+ configured, or if the reservation failed), for the caller to stash
+ for post-call reconciliation.
+ """
+ itpm_descriptors: Final = [ # mutable-ok: atomic limiter API requires lists
+ d for d in descriptors if d["key"] == PROJECT_ITPM_DESCRIPTOR_KEY
+ ]
+ otpm_descriptors: Final = [ # mutable-ok: atomic limiter API requires lists
+ d for d in descriptors if d["key"] == PROJECT_OTPM_DESCRIPTOR_KEY
+ ]
+
+ if not itpm_descriptors and not otpm_descriptors:
+ return RateLimitResponse(overall_code="OK", statuses=[]), 0, 0 # mutable-ok: response contract uses a list
+
+ itpm_response: Final = (
+ await self.atomic_check_and_increment_by_n(
+ descriptors=itpm_descriptors,
+ increments=[ # mutable-ok: atomic limiter API requires mutable increment records
+ {"tokens": estimated_input_tokens} # mutable-ok: atomic limiter increment record
+ for _ in itpm_descriptors
+ ],
+ parent_otel_span=parent_otel_span,
+ )
+ if itpm_descriptors
+ else None
+ )
+ if itpm_response is not None and itpm_response["overall_code"] == "OVER_LIMIT":
+ return itpm_response, 0, 0
+ itpm_reserved: Final = estimated_input_tokens if itpm_response is not None else 0
+
+ if otpm_descriptors:
+ otpm_response: Final = await self.atomic_check_and_increment_by_n(
+ descriptors=otpm_descriptors,
+ increments=[ # mutable-ok: atomic limiter API requires mutable increment records
+ {"tokens": estimated_output_tokens} # mutable-ok: atomic limiter increment record
+ for _ in otpm_descriptors
+ ],
+ parent_otel_span=parent_otel_span,
+ )
+ if otpm_response["overall_code"] == "OVER_LIMIT":
+ if itpm_reserved > 0:
+ await self._refund_reserved_tokens(
+ scopes=[ # mutable-ok: reservation rollback accepts collected scopes
+ (d["key"], d["value"]) for d in itpm_descriptors
+ ],
+ amount=itpm_reserved,
+ reservation_windows=itpm_response.get("reservation_windows", frozenset()),
+ parent_otel_span=parent_otel_span,
+ )
+ return otpm_response, 0, 0
+ statuses: Final = (
+ [ # mutable-ok: response contract uses a list
+ *itpm_response["statuses"],
+ *otpm_response["statuses"],
+ ]
+ if itpm_response is not None
+ else otpm_response["statuses"]
+ )
+ return (
+ RateLimitResponse(
+ overall_code="OK",
+ statuses=statuses,
+ reservation_windows=(
+ (
+ itpm_response.get("reservation_windows", frozenset())
+ if itpm_response is not None
+ else frozenset()
+ )
+ | otpm_response.get("reservation_windows", frozenset())
+ ),
+ ),
+ itpm_reserved,
+ estimated_output_tokens,
+ )
+
+ assert itpm_response is not None
+ return itpm_response, itpm_reserved, 0
+
+ async def enforce_project_io_token_quota_for_frame(
+ self,
+ user_api_key_dict: UserAPIKeyAuth | None,
+ requested_model: str | None,
+ estimated_input_tokens: int,
+ estimated_output_tokens: int,
+ ) -> None:
+ """Reserve one WebSocket ``response.create`` frame's tokens against
+ the caller's project ITPM/OTPM quota.
+
+ The Responses WebSocket connection-level pre-call hook only runs once
+ per connection, but a connection accepts many ``response.create``
+ frames over its lifetime. Without this, a project caller could send
+ unlimited high-token generations after a single minimal reservation.
+ There is no per-frame post-call hook to reconcile against, so --
+ like the batch rate limiter -- this charges the estimate immediately
+ and never refunds it.
+ """
+ if user_api_key_dict is None:
+ return
+ descriptors: Final[list[RateLimitDescriptor]] = [] # mutable-ok: descriptor helper appends in place
+ self.add_project_io_token_rate_limit_descriptors_from_metadata(
+ user_api_key_dict=user_api_key_dict,
+ requested_model=requested_model,
+ descriptors=descriptors,
+ )
+ if not descriptors:
+ return
+ response, _itpm_reserved, _otpm_reserved = await self.reserve_io_tokens(
+ descriptors=descriptors,
+ estimated_input_tokens=estimated_input_tokens,
+ estimated_output_tokens=estimated_output_tokens,
+ parent_otel_span=user_api_key_dict.parent_otel_span,
+ )
+ if response["overall_code"] == "OVER_LIMIT":
+ self._handle_rate_limit_error(response, descriptors, requested_model)
+
def create_organization_rate_limit_descriptor(
self, user_api_key_dict: UserAPIKeyAuth, requested_model: str | None = None
) -> list[RateLimitDescriptor]:
@@ -2434,6 +2932,62 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
)
+ def add_project_io_token_rate_limit_descriptors_from_metadata(
+ self,
+ user_api_key_dict: UserAPIKeyAuth,
+ requested_model: str | None,
+ descriptors: _RateLimitDescriptorSink,
+ ) -> None:
+ """Add project-scoped ITPM/OTPM descriptors from project_metadata.
+
+ Enforced independently of, and alongside, the combined ``model_per_project``
+ TPM descriptor above -- these give Bedrock Mantle-style separate input/output
+ token quotas at the project level.
+ """
+ if requested_model is None or user_api_key_dict.project_id is None:
+ return
+
+ itpm_limit_for_project_model: Final = (
+ get_model_rate_limit_from_metadata(user_api_key_dict, "project_metadata", "model_itpm_limit")
+ or {} # mutable-ok: metadata helper returns an optional mapping
+ )
+ otpm_limit_for_project_model: Final = (
+ get_model_rate_limit_from_metadata(user_api_key_dict, "project_metadata", "model_otpm_limit")
+ or {} # mutable-ok: metadata helper returns an optional mapping
+ )
+
+ model_itpm_limit: Final = itpm_limit_for_project_model.get(requested_model)
+ model_otpm_limit: Final = otpm_limit_for_project_model.get(requested_model)
+
+ if model_itpm_limit is None and model_otpm_limit is None:
+ return
+
+ descriptor_value: Final = f"{user_api_key_dict.project_id}:{requested_model}"
+ if model_itpm_limit is not None:
+ descriptors.append(
+ RateLimitDescriptor(
+ key=PROJECT_ITPM_DESCRIPTOR_KEY,
+ value=descriptor_value,
+ rate_limit={ # mutable-ok: descriptor TypedDict requires a runtime dict
+ "requests_per_unit": None,
+ "tokens_per_unit": model_itpm_limit,
+ "window_size": self.window_size,
+ },
+ )
+ )
+ if model_otpm_limit is not None:
+ descriptors.append(
+ RateLimitDescriptor(
+ key=PROJECT_OTPM_DESCRIPTOR_KEY,
+ value=descriptor_value,
+ rate_limit={ # mutable-ok: descriptor TypedDict requires a runtime dict
+ "requests_per_unit": None,
+ "tokens_per_unit": model_otpm_limit,
+ "window_size": self.window_size,
+ },
+ )
+ )
+
def _handle_rate_limit_error(
self,
response: RateLimitResponse,
@@ -2478,6 +3032,342 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
llm_provider=llm_provider,
)
+ @staticmethod
+ def _estimate_audio_block_tokens(block: object) -> int:
+ """
+ Token estimate for one ``input_audio`` content block.
+
+ When the block carries a base64 ``data`` payload, the estimate comes
+ from the decoded byte count (``len(b64) * 3 // 4 // _AUDIO_BYTES_PER_TOKEN``),
+ assuming the lowest reasonable audio bitrate so we never under-reserve
+ for higher-quality recordings of the same duration.
+
+ When no payload is present (reference-only block or missing ``data``),
+ falls back to ``DEFAULT_AUDIO_TOKEN_ESTIMATE``.
+ """
+ if not isinstance(block, dict):
+ return DEFAULT_AUDIO_TOKEN_ESTIMATE
+ input_audio: Final = block.get("input_audio")
+ b64_data: Final = input_audio.get("data") if isinstance(input_audio, dict) else None
+ if b64_data and isinstance(b64_data, str):
+ decoded_bytes: Final = len(b64_data) * 3 // 4
+ return max(decoded_bytes // _AUDIO_BYTES_PER_TOKEN, DEFAULT_AUDIO_TOKEN_ESTIMATE)
+ return DEFAULT_AUDIO_TOKEN_ESTIMATE
+
+ @classmethod
+ def _estimate_audio_content_tokens(cls, messages: object) -> int:
+ """
+ Sum of per-block audio token estimates across all ``messages``.
+ Returns 0 when there are no ``input_audio`` blocks, which the caller
+ uses to skip the (relatively expensive) strip pass.
+ """
+ if not isinstance(messages, list):
+ return 0
+ return sum(
+ cls._estimate_audio_block_tokens(block)
+ for message in messages
+ if isinstance(message, dict)
+ for content in (message.get("content"),)
+ if isinstance(content, list)
+ for block in content
+ if isinstance(block, dict) and block.get("type") == "input_audio"
+ )
+
+ @staticmethod
+ def _strip_audio_content_blocks(messages: object) -> object:
+ """
+ Drop ``input_audio`` content blocks before passing ``messages`` to
+ ``token_counter``, which raises ``ValueError`` on them (no per-type
+ handling, unlike images). The audio contribution is added back
+ separately via ``DEFAULT_AUDIO_TOKEN_ESTIMATE`` so the rest of the
+ message (text/images/tools) still gets counted accurately instead of
+ the whole call falling back to the cheap char-count estimate.
+ """
+ if not isinstance(messages, list):
+ return messages
+ sanitized: Final[list[object]] = [] # mutable-ok: token_counter requires a list of message dicts
+ for message in messages:
+ if not isinstance(message, dict):
+ sanitized.append(message)
+ continue
+ content = message.get("content")
+ if not isinstance(content, list):
+ sanitized.append(message)
+ continue
+ filtered_content = [ # mutable-ok: token_counter requires list content blocks
+ block for block in content if not (isinstance(block, dict) and block.get("type") == "input_audio")
+ ]
+ sanitized.append( # mutable-ok: token_counter requires mutable message dicts
+ {**message, "content": filtered_content} # mutable-ok: token_counter requires message dicts
+ )
+ return sanitized
+
+ @staticmethod
+ def _responses_input_to_chat_messages(data: object) -> Sequence[object]:
+ """
+ Convert a Responses API ``input`` (string or list of input items) into
+ chat-completion-style messages via the standard LiteLLM transformation
+ (the same one guardrails use, e.g. ``purview_dlp.py``), so multimodal
+ ``input_image``/``input_text`` content blocks get counted by
+ ``token_counter``'s ``messages`` path instead of silently contributing
+ zero tokens via its ``text`` path, which only joins plain strings.
+ """
+ from litellm.responses.litellm_completion_transformation.transformation import (
+ LiteLLMCompletionResponsesConfig,
+ )
+
+ if not isinstance(data, dict):
+ return ()
+ return LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
+ input=data.get("input") or "",
+ responses_api_request=data,
+ )
+
+ @staticmethod
+ def _count_pretokenized_embedding_input(value: object) -> int | None:
+ if not isinstance(value, list):
+ return None
+ if all(isinstance(token, int) for token in value):
+ return len(value)
+ if all(
+ isinstance(token_ids, list) and all(isinstance(token, int) for token in token_ids) for token_ids in value
+ ):
+ return sum(len(token_ids) for token_ids in value)
+ return None
+
+ @staticmethod
+ def _rerank_input_to_text(data: Mapping[str, object]) -> str:
+ documents: Final = data.get("documents")
+ document_items: Final[Sequence[object]] = documents if isinstance(documents, list) else () # pyright: ignore[reportUnknownVariableType] # rerank documents are validated runtime JSON
+ input_parts: Final[tuple[object, ...]] = ( # pyright: ignore[reportUnknownVariableType] # list narrowing preserves unknown JSON element types
+ data.get("query"),
+ *document_items,
+ )
+ return "\n".join(
+ str(part) # pyright: ignore[reportUnknownArgumentType] # accepted document dicts have provider-defined fields
+ for part in input_parts # pyright: ignore[reportUnknownVariableType] # runtime JSON list elements remain unknown after list narrowing
+ if isinstance(part, (str, dict))
+ )
+
+ def _estimate_precise_input_tokens(self, data: object, model: str | None, call_type: str | None = None) -> int:
+ """
+ Model-aware input token estimate for the project ITPM reservation,
+ using ``litellm.token_counter`` -- the same approach the
+ deployment-level itpm/otpm check uses in
+ ``io_token_rate_limit_check.py``. Unlike the cheap char-count
+ estimate the combined-TPM path uses, this accounts for image/tool
+ content and derives per-``input_audio``-block estimates from the
+ base64 payload size (assuming the lowest reasonable bitrate so
+ longer recordings always reserve proportionally more), so a burst
+ of multimodal, tool-heavy, or audio-heavy requests can't each
+ reserve only the one-token floor and blow past ITPM before
+ post-call reconciliation catches up.
+
+ For the Responses API, ``input`` is converted to chat messages first
+ (via ``_responses_input_to_chat_messages``) so its own multimodal
+ content blocks are counted the same way; ``token_counter``'s ``text``
+ argument can only see plain strings in a list, not content blocks.
+
+ Falls back to the cheap char-count estimate if ``token_counter``
+ can't resolve a tokenizer for this model (e.g. an unrecognized
+ custom model name) or otherwise raises -- the audio add-on still
+ applies on top of the fallback.
+ """
+ from litellm import token_counter
+
+ if not isinstance(data, dict):
+ return 0
+ is_responses_request: Final = call_type in RESPONSES_API_CALL_TYPES
+ translated_request: Final = (
+ None if is_responses_request else self._translate_google_genai_native_request(data, call_type)
+ )
+ is_embedding_request: Final = self._is_embedding_request(data, call_type)
+ embedding_text: Final = data.get("input") if is_embedding_request else None
+ pretokenized_input_tokens: Final = (
+ self._count_pretokenized_embedding_input(embedding_text) if is_embedding_request else None
+ )
+ if pretokenized_input_tokens is not None:
+ return pretokenized_input_tokens
+
+ prompt: Final = data.get("prompt")
+ fallback_text: Final = prompt if prompt is not None else data.get("input")
+ selected_inputs: Final[tuple[object | None, object | None, object | None, object | None]] = (
+ (self._responses_input_to_chat_messages(data), None, data.get("tools"), data.get("tool_choice"))
+ if is_responses_request
+ else (
+ translated_request.get("messages"),
+ None,
+ translated_request.get("tools"),
+ translated_request.get("tool_choice"),
+ )
+ if translated_request is not None
+ else (None, embedding_text, data.get("tools"), data.get("tool_choice"))
+ if is_embedding_request
+ else (None, self._rerank_input_to_text(data), data.get("tools"), data.get("tool_choice"))
+ if call_type in RERANK_API_CALL_TYPES
+ else (None, prompt, data.get("tools"), data.get("tool_choice"))
+ if call_type in TEXT_COMPLETION_API_CALL_TYPES
+ else (data.get("messages"), fallback_text, data.get("tools"), data.get("tool_choice"))
+ )
+ messages, selected_text, countable_tools, countable_tool_choice = selected_inputs
+
+ audio_token_estimate: Final = self._estimate_audio_content_tokens(messages)
+ countable_messages: Final = self._strip_audio_content_blocks(messages) if audio_token_estimate > 0 else messages
+
+ try:
+ estimate: Final = max(
+ 0,
+ int(
+ token_counter(
+ model=model or "",
+ messages=countable_messages,
+ text=selected_text,
+ tools=countable_tools,
+ tool_choice=countable_tool_choice,
+ use_default_image_token_count=True,
+ )
+ ),
+ )
+ return estimate + audio_token_estimate
+ except Exception: # noqa: BLE001 # tokenizer failures degrade to the cheap estimate
+ if call_type in RERANK_API_CALL_TYPES and isinstance(selected_text, str):
+ return max(0, len(selected_text) // DEFAULT_CHARS_PER_TOKEN)
+ estimated_input_tokens, _ = self._estimate_input_and_output_tokens(data=data, call_type=call_type)
+ return estimated_input_tokens + audio_token_estimate
+
+ async def _reserve_project_io_tokens_or_raise(
+ self,
+ descriptors: Sequence[RateLimitDescriptor],
+ data: object,
+ requested_model: str | None,
+ user_api_key_dict: UserAPIKeyAuth,
+ tpm_reservation_scopes: Sequence[tuple[str, str]],
+ tpm_reservation_amount: int,
+ call_type: str | None = None,
+ ) -> None:
+ """
+ Reserve project-scoped ITPM/OTPM tokens (Bedrock Mantle-style
+ separate input/output token buckets), independently of -- and, when
+ both are configured, in addition to -- the combined-TPM reservation
+ the caller already made. Raises (via ``_handle_rate_limit_error``) on
+ an over-limit reservation, first rolling back the combined-TPM
+ reservation named by ``tpm_reservation_scopes``/``tpm_reservation_amount``
+ if one was made, so a partial reservation never leaks.
+ """
+ if not isinstance(data, dict):
+ return
+ stash: Final = claim_request_stash_for_data(data)
+ io_token_descriptors: Final = [ # mutable-ok: reservation API requires descriptor lists
+ d for d in descriptors if d["key"] in (PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY)
+ ]
+ if not io_token_descriptors:
+ return
+
+ configured_otpm_limits: Final = [ # mutable-ok: min calculation materializes validated limits
+ int(v)
+ for d in io_token_descriptors
+ if d["key"] == PROJECT_OTPM_DESCRIPTOR_KEY
+ for v in [ # mutable-ok: comprehension binds the optional descriptor value
+ (d.get("rate_limit") or {}).get( # mutable-ok: optional descriptor fallback
+ "tokens_per_unit"
+ )
+ ]
+ if v is not None
+ ]
+ min_configured_otpm_limit: Final = min(configured_otpm_limits) if configured_otpm_limits else None
+ _, raw_estimated_output_tokens = self._estimate_input_and_output_tokens(
+ data=data,
+ min_configured_tpm_limit=min_configured_otpm_limit,
+ call_type=call_type,
+ )
+ raw_estimated_input_tokens: Final = self._estimate_precise_input_tokens(
+ data=data, model=requested_model, call_type=call_type
+ )
+ estimated_input_tokens: Final = max(raw_estimated_input_tokens, 1)
+ estimated_output_tokens: Final = (
+ raw_estimated_output_tokens
+ if self._has_explicit_output_cap(data, call_type)
+ else max(raw_estimated_output_tokens, 1)
+ )
+
+ # Hard-cap generation length so an unbounded response can't overshoot
+ # the OTPM budget before post-call reconciliation runs, mirroring the
+ # combined-TPM floor cap in the caller.
+ self._apply_implicit_output_cap(
+ data=data,
+ min_configured_limit=min_configured_otpm_limit,
+ call_type=call_type,
+ )
+
+ io_response, itpm_reserved, otpm_reserved = await self.reserve_io_tokens(
+ descriptors=io_token_descriptors,
+ estimated_input_tokens=estimated_input_tokens,
+ estimated_output_tokens=estimated_output_tokens,
+ parent_otel_span=user_api_key_dict.parent_otel_span,
+ )
+
+ if io_response["overall_code"] == "OVER_LIMIT":
+ # A combined-TPM reservation may have already succeeded above for
+ # this same request; refund it too, or its counter stays inflated
+ # until the window's TTL expires. Mark it released so the
+ # ProxyRateLimitError we're about to raise doesn't get refunded
+ # a second time when async_post_call_failure_hook sees the same
+ # (still-stashed) reservation and refunds it again.
+ if tpm_reservation_amount > 0:
+ await self._refund_reserved_tokens(
+ scopes=tpm_reservation_scopes,
+ amount=tpm_reservation_amount,
+ parent_otel_span=user_api_key_dict.parent_otel_span,
+ )
+ stash.reservation_released = True
+ acquisition: Final = stash.parallel_slot
+ if acquisition is not None:
+ await self._release_parallel_request_slots(
+ acquisition=acquisition,
+ parent_otel_span=user_api_key_dict.parent_otel_span,
+ )
+ stash.parallel_slot = None
+ self._handle_rate_limit_error(
+ response=io_response,
+ descriptors=descriptors,
+ requested_model=requested_model,
+ )
+
+ if itpm_reserved > 0:
+ itpm_scopes: Final = tuple(
+ (d["key"], d["value"]) for d in io_token_descriptors if d["key"] == PROJECT_ITPM_DESCRIPTOR_KEY
+ )
+ stash.itpm_reserved_tokens = itpm_reserved
+ stash.itpm_reserved_scopes = frozenset(itpm_scopes)
+ stash.itpm_reserved_window_identities = frozenset(
+ (counter_key, window_start, backend)
+ for counter_key, window_start, backend in io_response.get("reservation_windows", frozenset())
+ if "model_per_project_itpm" in counter_key
+ )
+ if otpm_reserved > 0:
+ otpm_scopes: Final = tuple(
+ (d["key"], d["value"]) for d in io_token_descriptors if d["key"] == PROJECT_OTPM_DESCRIPTOR_KEY
+ )
+ stash.otpm_reserved_tokens = otpm_reserved
+ stash.otpm_reserved_scopes = frozenset(otpm_scopes)
+ stash.otpm_reserved_window_identities = frozenset(
+ (counter_key, window_start, backend)
+ for counter_key, window_start, backend in io_response.get("reservation_windows", frozenset())
+ if "model_per_project_otpm" in counter_key
+ )
+
+ if stash.rate_limit_response is not None:
+ stash.rate_limit_response["statuses"].extend(io_response["statuses"])
+ elif io_response["statuses"]:
+ stash.rate_limit_response = io_response
+
+ verbose_proxy_logger.debug(
+ "ITPM/OTPM tokens reserved: itpm=%s, otpm=%s for model %s",
+ itpm_reserved,
+ otpm_reserved,
+ requested_model,
+ )
+
async def async_pre_call_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
@@ -2550,6 +3440,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
requested_model=requested_model,
descriptors=descriptors,
)
+ self.add_project_io_token_rate_limit_descriptors_from_metadata(
+ user_api_key_dict=user_api_key_dict,
+ requested_model=requested_model,
+ descriptors=descriptors,
+ )
# Org Level Rate Limits
descriptors.extend(self.create_organization_rate_limit_descriptor(user_api_key_dict, requested_model))
@@ -2565,7 +3460,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
# in-flight request would pre-inflate the :tokens counter by 1,
# shrinking the effective TPM budget by N and causing
# false-positive 429s under bursts. When reservation is disabled,
- # this pass enforces TPM directly from the post-call counters.
+ # this pass enforces TPM directly from the post-call counters --
+ # except for project ITPM/OTPM descriptors, which are excluded
+ # then because _reserve_project_io_tokens_or_raise below charges
+ # them unconditionally and counting them here too would
+ # double-charge every request.
parallel_counter_keys: Final = [
self.create_rate_limit_keys(d["key"], d["value"], "max_parallel_requests")
for d in descriptors
@@ -2573,8 +3472,15 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
]
parallel_slot_id: Final = uuid.uuid4().hex if parallel_counter_keys else None
+ first_pass_descriptors: Final = (
+ descriptors
+ if self.tpm_reservation_enabled
+ else tuple(
+ d for d in descriptors if d["key"] not in (PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY)
+ )
+ )
response: Final = await self.should_rate_limit(
- descriptors=descriptors,
+ descriptors=first_pass_descriptors,
parent_otel_span=user_api_key_dict.parent_otel_span,
skip_tpm_check=self.tpm_reservation_enabled,
parallel_slot_id=parallel_slot_id,
@@ -2606,32 +3512,39 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
configured_tpm_limits: Final = [
int(v)
for d in descriptors
+ if d["key"] not in (PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY)
for v in [(d.get("rate_limit") or {}).get("tokens_per_unit")]
if v is not None
]
has_tpm_limits: Final = bool(configured_tpm_limits)
+ # Populated on a successful combined-TPM reservation below, so the
+ # project ITPM/OTPM block further down can roll it back if a
+ # different bucket in the same request subsequently hits its
+ # limit. Stays empty/0 whenever no combined-TPM reservation was
+ # made (or it was over limit, in which case execution never
+ # reaches the ITPM/OTPM block -- `_handle_rate_limit_error` raises).
+ tpm_reservation_scopes: Sequence[tuple[str, str]] = () # rebind-ok: set after successful reservation
+ tpm_reservation_amount = 0 # rebind-ok: set after successful reservation
+
if has_tpm_limits and self.tpm_reservation_enabled:
min_configured_tpm_limit: Final = min(configured_tpm_limits)
- # When the configured TPM cap is small enough to constrain the
- # no-max_tokens floor, also hard-cap the model output via
- # data["max_tokens"] so concurrent unbounded generations can't
- # spend past the limit before post-call reconciliation runs.
- # Skip when the request already sets max_tokens or has no
- # generation budget at all (embeddings).
- capped_floor: Final = self._no_max_tokens_output_floor(min_configured_tpm_limit)
- baseline_floor: Final = DEFAULT_MAX_TOKENS_ESTIMATE // _TPM_FLOOR_FRACTION
- has_explicit_max_tokens: Final = (
- data.get("max_tokens") is not None or data.get("max_completion_tokens") is not None
- )
- is_embedding: Final = data.get("input") is not None
configured_output_tokens: Final = get_estimated_output_tokens(
user_api_key_dict=user_api_key_dict,
model_name=requested_model,
)
- if capped_floor < baseline_floor and not has_explicit_max_tokens and not is_embedding:
- data["max_tokens"] = max(capped_floor, configured_output_tokens or 0)
+
+ # When the configured TPM cap is small enough to constrain the
+ # no-max_tokens floor, also hard-cap the model output so
+ # concurrent unbounded generations can't spend past the limit
+ # before post-call reconciliation runs.
+ self._apply_implicit_output_cap(
+ data=data,
+ min_configured_limit=min_configured_tpm_limit,
+ call_type=call_type,
+ configured_output_tokens=configured_output_tokens,
+ )
# Floor at 1 token so contentless requests (/responses,
# tool-call continuations, empty messages) still flow
@@ -2645,6 +3558,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
data=data,
model=requested_model,
min_configured_tpm_limit=min_configured_tpm_limit,
+ call_type=call_type,
configured_output_tokens=configured_output_tokens,
),
1,
@@ -2691,8 +3605,16 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
stash.reserved_scopes = frozenset(
(d["key"], d["value"])
for d in descriptors
- if (d.get("rate_limit") or {}).get("tokens_per_unit") is not None
+ if d["key"] not in (PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY)
+ and (d.get("rate_limit") or {}).get( # mutable-ok: optional descriptor fallback
+ "tokens_per_unit"
+ )
+ is not None
)
+ tpm_reservation_scopes = tuple( # rebind-ok: record successful reservation scopes
+ stash.reserved_scopes
+ )
+ tpm_reservation_amount = estimated_tokens # rebind-ok: record successful reservation amount
# Merge TPM statuses into the stored rate-limit response
# so x-ratelimit-{key}-remaining-tokens / -limit-tokens
@@ -2706,6 +3628,15 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
verbose_proxy_logger.debug(
"TPM tokens reserved: %s for model %s", estimated_tokens, requested_model
)
+ await self._reserve_project_io_tokens_or_raise(
+ descriptors=descriptors,
+ data=data,
+ requested_model=requested_model,
+ user_api_key_dict=user_api_key_dict,
+ tpm_reservation_scopes=tpm_reservation_scopes,
+ tpm_reservation_amount=tpm_reservation_amount,
+ call_type=call_type,
+ )
def _create_pipeline_operations(
self,
@@ -2782,7 +3713,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
return total_tokens
@staticmethod
- def _aggregate_only_total_tokens(usage: Usage | dict | None) -> int:
+ def _aggregate_only_total_tokens(usage: Usage | ResponseAPIUsage | Mapping[str, object] | None) -> int:
"""Total for usage that carries no input/output split, else 0.
A source that can only report one number for the whole request (a
@@ -2792,24 +3723,43 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
uncharged, which is how pass-through traffic slips past a TPM limit
it is supposed to share.
"""
- if isinstance(usage, Usage):
- prompt_tokens, completion_tokens, total_tokens = (
- usage.prompt_tokens or 0,
- usage.completion_tokens or 0,
- usage.total_tokens or 0,
- )
- elif isinstance(usage, dict):
- prompt_tokens, completion_tokens, total_tokens = (
- usage.get("prompt_tokens") or 0,
- usage.get("completion_tokens") or 0,
+ if usage is None:
+ return 0
+ token_counts: Final = (
+ (usage.prompt_tokens or 0, usage.completion_tokens or 0, usage.total_tokens or 0)
+ if isinstance(usage, Usage)
+ else (usage.input_tokens or 0, usage.output_tokens or 0, usage.total_tokens or 0)
+ if isinstance(usage, ResponseAPIUsage)
+ else (
+ usage.get("prompt_tokens") or usage.get("input_tokens") or 0,
+ usage.get("completion_tokens") or usage.get("output_tokens") or 0,
usage.get("total_tokens") or 0,
)
- else:
- return 0
- if prompt_tokens or completion_tokens:
+ )
+ prompt_tokens, completion_tokens, total_tokens = token_counts
+ if prompt_tokens or completion_tokens or not isinstance(total_tokens, int):
return 0
return total_tokens
+ @staticmethod
+ def _response_usage(
+ response_obj: object,
+ ) -> Usage | ResponseAPIUsage | Mapping[str, object] | None:
+ if isinstance(response_obj, (Usage, ResponseAPIUsage)):
+ return response_obj
+ if isinstance(
+ response_obj,
+ (ModelResponse, EmbeddingResponse, TextCompletionResponse, BaseLiteLLMOpenAIResponseObject),
+ ):
+ usage: Final = getattr(response_obj, "usage", None)
+ return usage if isinstance(usage, (Usage, ResponseAPIUsage, dict)) else None
+ if isinstance(response_obj, dict):
+ nested_usage: Final = response_obj.get("usage")
+ if isinstance(nested_usage, (Usage, ResponseAPIUsage, dict)):
+ return nested_usage
+ return response_obj
+ return None
+
async def _execute_token_increment_script(
self,
pipeline_operations: list["RedisPipelineIncrementOperation"],
@@ -2885,6 +3835,116 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
litellm_parent_otel_span=parent_otel_span,
)
+ async def _apply_local_window_guarded_token_increments(
+ self,
+ operations: Sequence[ReservationAwareIncrementOperation],
+ parent_otel_span: Span | None = None,
+ ) -> None:
+ async with self._check_and_increment_lock:
+ for operation in operations:
+ window_key = operation.get("window_key")
+ expected_window_start = operation.get("expected_window_start")
+ if window_key is None or expected_window_start is None:
+ continue
+ active_window_start = await self.internal_usage_cache.async_get_cache(
+ key=window_key,
+ litellm_parent_otel_span=parent_otel_span,
+ local_only=True,
+ )
+ if active_window_start is None or str(active_window_start) != expected_window_start:
+ continue
+ current_counter = (
+ await self.internal_usage_cache.async_get_cache(
+ key=operation["key"],
+ litellm_parent_otel_span=parent_otel_span,
+ local_only=True,
+ )
+ or 0
+ )
+ await self.internal_usage_cache.async_set_cache(
+ key=operation["key"],
+ value=float(current_counter) + operation["increment_value"],
+ ttl=operation["ttl"],
+ litellm_parent_otel_span=parent_otel_span,
+ local_only=True,
+ )
+
+ async def _apply_redis_window_guarded_token_increments(
+ self,
+ operations: Sequence[ReservationAwareIncrementOperation],
+ parent_otel_span: Span | None = None,
+ ) -> None:
+ for operation in operations:
+ window_key = operation.get("window_key")
+ expected_window_start = operation.get("expected_window_start")
+ if window_key is None or expected_window_start is None:
+ continue
+ if self.window_guarded_token_increment_script is not None:
+ try:
+ await self.window_guarded_token_increment_script(
+ keys=[ # mutable-ok: Redis script interface requires a key list
+ window_key,
+ operation["key"],
+ ],
+ args=[ # mutable-ok: Redis script interface requires an argument list
+ expected_window_start,
+ operation["increment_value"],
+ operation["ttl"] or 0,
+ ],
+ )
+ continue
+ except Exception as e: # noqa: BLE001 # Redis failures use the plain increment fallback
+ verbose_proxy_logger.warning(
+ "Window-guarded token adjustment failed for %s: %s",
+ operation["key"],
+ e,
+ )
+ if operation["increment_value"] > 0:
+ await self.internal_usage_cache.async_increment_cache(
+ key=operation["key"],
+ value=operation["increment_value"],
+ litellm_parent_otel_span=parent_otel_span,
+ ttl=operation["ttl"],
+ )
+
+ async def async_increment_reservation_aware_tokens(
+ self,
+ pipeline_operations: Sequence[ReservationAwareIncrementOperation],
+ parent_otel_span: Span | None = None,
+ ) -> None:
+ for operation in pipeline_operations:
+ if operation.get("window_key") is None or operation.get("expected_window_start") is None:
+ await self.internal_usage_cache.async_increment_cache(
+ key=operation["key"],
+ value=operation["increment_value"],
+ litellm_parent_otel_span=parent_otel_span,
+ ttl=operation["ttl"],
+ )
+ local_guarded_operations: Final = tuple(
+ operation
+ for operation in pipeline_operations
+ if operation.get("window_key") is not None
+ and operation.get("expected_window_start") is not None
+ and operation.get("reservation_backend") == "local"
+ )
+ redis_guarded_operations: Final = tuple(
+ operation
+ for operation in pipeline_operations
+ if operation.get("window_key") is not None
+ and operation.get("expected_window_start") is not None
+ and operation.get("reservation_backend") != "local"
+ )
+ if local_guarded_operations:
+ await self._apply_local_window_guarded_token_increments(
+ operations=local_guarded_operations,
+ parent_otel_span=parent_otel_span,
+ )
+ if redis_guarded_operations:
+ await self._apply_redis_window_guarded_token_increments(
+ operations=redis_guarded_operations,
+ parent_otel_span=parent_otel_span,
+ )
+
def get_rate_limit_type(self) -> Literal["output", "input", "total"]:
from litellm.proxy.proxy_server import general_settings
@@ -2914,6 +3974,164 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
merged[f"{prefix}-limit-{status['rate_limit_type']}"] = status["current_limit"]
return merged
+ @staticmethod
+ def _resolve_rerank_token_usage(response_obj: object) -> tuple[int, int, bool] | None:
+ if not isinstance(response_obj, RerankResponse) or response_obj.meta is None:
+ return None
+
+ rerank_tokens: Final = response_obj.meta.get("tokens") # pyright: ignore[reportUnknownMemberType] # TypedDict's optional generic metadata widens get overloads
+ if rerank_tokens is not None:
+ input_tokens: Final = rerank_tokens.get("input_tokens") or 0 # pyright: ignore[reportUnknownMemberType] # token fields are typed integers despite the generic get overload
+ output_tokens: Final = rerank_tokens.get("output_tokens") or 0 # pyright: ignore[reportUnknownMemberType] # token fields are typed integers despite the generic get overload
+ if input_tokens or output_tokens:
+ return max(0, input_tokens), max(0, output_tokens), True
+
+ billed_units: Final = response_obj.meta.get("billed_units") # pyright: ignore[reportUnknownMemberType] # TypedDict's optional generic metadata widens get overloads
+ if billed_units is not None:
+ total_tokens: Final = billed_units.get("total_tokens") or 0 # pyright: ignore[reportUnknownMemberType] # billed total is a typed integer despite the generic get overload
+ if total_tokens:
+ return max(0, total_tokens), 0, True
+ return None
+
+ def _resolve_io_token_reconcile_usage(
+ self,
+ response_obj: object,
+ ) -> tuple[int, int, bool]:
+ """
+ Resolve ``(billable_input_tokens, completion_tokens, usage_resolved)``
+ for ITPM/OTPM reconciliation. Cache-read tokens are excluded from
+ billable input -- Bedrock Mantle doesn't count them toward ITPM --
+ but they're untouched everywhere else (cost/usage logging still sees
+ the full prompt token count).
+ """
+ rerank_usage: Final = self._resolve_rerank_token_usage(response_obj)
+ if rerank_usage is not None:
+ return rerank_usage
+
+ usage: Final = self._response_usage(response_obj)
+
+ if isinstance(usage, Usage):
+ prompt_tokens: Final = usage.prompt_tokens or 0
+ completion_tokens: Final = usage.completion_tokens or 0
+ cached_tokens: Final = (
+ getattr(usage.prompt_tokens_details, "cached_tokens", 0) or 0
+ if usage.prompt_tokens_details is not None
+ else 0
+ )
+ if prompt_tokens == 0 and completion_tokens == 0:
+ return 0, 0, False
+ return max(0, prompt_tokens - cached_tokens), completion_tokens, True
+
+ if isinstance(usage, ResponseAPIUsage):
+ response_input_tokens: Final = usage.input_tokens or 0
+ response_output_tokens: Final = usage.output_tokens or 0
+ response_cached_tokens: Final = (
+ usage.input_tokens_details.cached_tokens or 0 if usage.input_tokens_details is not None else 0
+ )
+ if response_input_tokens == 0 and response_output_tokens == 0:
+ return 0, 0, False
+ return max(0, response_input_tokens - response_cached_tokens), response_output_tokens, True
+
+ if isinstance(usage, Mapping):
+ raw_prompt_tokens: Final = usage.get("prompt_tokens") or usage.get("input_tokens") or 0
+ raw_completion_tokens: Final = usage.get("completion_tokens") or usage.get("output_tokens") or 0
+ mapped_prompt_tokens: Final = raw_prompt_tokens if isinstance(raw_prompt_tokens, int) else 0
+ mapped_completion_tokens: Final = raw_completion_tokens if isinstance(raw_completion_tokens, int) else 0
+ prompt_details: Final = usage.get("prompt_tokens_details") or usage.get("input_tokens_details")
+ raw_cached_tokens: Final = (
+ (prompt_details.get("cached_tokens", 0) if isinstance(prompt_details, dict) else 0)
+ or usage.get("cache_read_input_tokens")
+ or 0
+ )
+ mapped_cached_tokens: Final = raw_cached_tokens if isinstance(raw_cached_tokens, int) else 0
+ if mapped_prompt_tokens == 0 and mapped_completion_tokens == 0:
+ return 0, 0, False
+ return max(0, mapped_prompt_tokens - mapped_cached_tokens), mapped_completion_tokens, True
+
+ return 0, 0, False
+
+ def _build_io_token_reservation_ops(
+ self,
+ kwargs: object,
+ response_obj: object,
+ ) -> Sequence[RedisPipelineIncrementOperation]:
+ """
+ Reconcile project ITPM/OTPM reservations to actual usage on success:
+ ITPM to billable input tokens, OTPM to actual completion tokens.
+ Reuses ``_build_reservation_aware_tpm_ops``'s delta pattern -- ITPM/OTPM
+ are stored in the same ":tokens" cache bucket as combined TPM, just
+ under distinct scope keys, so the reservation-aware increment math is
+ identical; only the usage fields being reconciled against differ.
+ """
+ if not isinstance(kwargs, dict):
+ return ()
+ stash: Final = get_request_stash_for_call(_call_id_from_callback_kwargs(kwargs))
+ if stash is None:
+ return ()
+
+ itpm_reserved: Final = stash.itpm_reserved_tokens
+ otpm_reserved: Final = stash.otpm_reserved_tokens
+ if itpm_reserved <= 0 and otpm_reserved <= 0:
+ return ()
+
+ response_usage: Final = self._resolve_io_token_reconcile_usage(response_obj)
+ combined_usage: Final = self._resolve_io_token_reconcile_usage(kwargs.get("combined_usage_object"))
+ aggregate_total: Final = self._aggregate_only_total_tokens(
+ self._response_usage(response_obj)
+ ) or self._aggregate_only_total_tokens(self._response_usage(kwargs.get("combined_usage_object")))
+
+ if not response_usage[2] and not combined_usage[2] and aggregate_total <= 0 and not stash.reservation_released:
+ return ()
+ resolved_usage: Final = (
+ response_usage
+ if response_usage[2]
+ else combined_usage
+ if combined_usage[2]
+ else (aggregate_total, aggregate_total, True)
+ if aggregate_total > 0
+ else (itpm_reserved, otpm_reserved, False)
+ )
+ billable_input, completion_tokens, _ = resolved_usage
+
+ if stash.reservation_released or (
+ not stash.itpm_reserved_window_identities and not stash.otpm_reserved_window_identities
+ ):
+ return self._build_reservation_aware_tpm_ops(
+ targets=tuple(stash.itpm_reserved_scopes),
+ reserved_scopes=frozenset() if stash.reservation_released else stash.itpm_reserved_scopes,
+ actual_tokens=billable_input,
+ reserved_tokens=0 if stash.reservation_released else itpm_reserved,
+ ) + self._build_reservation_aware_tpm_ops(
+ targets=tuple(stash.otpm_reserved_scopes),
+ reserved_scopes=frozenset() if stash.reservation_released else stash.otpm_reserved_scopes,
+ actual_tokens=completion_tokens,
+ reserved_tokens=0 if stash.reservation_released else otpm_reserved,
+ )
+
+ itpm_ops: Final[Sequence[ReservationAwareIncrementOperation]] = (
+ self._build_project_reservation_ops(
+ targets=tuple(stash.itpm_reserved_scopes),
+ reserved_scopes=frozenset() if stash.reservation_released else stash.itpm_reserved_scopes,
+ actual_tokens=billable_input,
+ reserved_tokens=itpm_reserved,
+ reservation_window_identities=stash.itpm_reserved_window_identities,
+ )
+ if itpm_reserved > 0
+ else ()
+ )
+ otpm_ops: Final[Sequence[ReservationAwareIncrementOperation]] = (
+ self._build_project_reservation_ops(
+ targets=tuple(stash.otpm_reserved_scopes),
+ reserved_scopes=frozenset() if stash.reservation_released else stash.otpm_reserved_scopes,
+ actual_tokens=completion_tokens,
+ reserved_tokens=otpm_reserved,
+ reservation_window_identities=stash.otpm_reserved_window_identities,
+ )
+ if otpm_reserved > 0
+ else ()
+ )
+ return tuple((*itpm_ops, *otpm_ops))
+
def _collect_tpm_scope_targets(
self,
standard_logging_metadata: dict[str, Any],
@@ -2978,8 +4196,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
def _build_reservation_aware_tpm_ops(
self,
- targets: list[tuple[str, str]],
- reserved_scopes: frozenset[tuple[str, str]],
+ targets: Sequence[tuple[str, str]],
+ reserved_scopes: Set[tuple[str, str]],
actual_tokens: int,
reserved_tokens: int,
) -> list[RedisPipelineIncrementOperation]:
@@ -3012,6 +4230,66 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
return ops
+ def _build_project_reservation_op(
+ self,
+ scope: tuple[str, str],
+ reserved_scopes: Set[tuple[str, str]],
+ actual_tokens: int,
+ reserved_tokens: int,
+ reservation_window_identities: frozenset[tuple[str, str, Literal["redis", "local"]]],
+ ) -> ReservationAwareIncrementOperation | None:
+ scope_key, scope_value = scope
+ is_reserved_scope: Final = scope in reserved_scopes
+ increment: Final = actual_tokens - reserved_tokens if is_reserved_scope else actual_tokens
+ if increment == 0:
+ return None
+ counter_key: Final = self.create_rate_limit_keys(scope_key, scope_value, "tokens")
+ window_identity: Final = next(
+ (
+ (window_start, backend)
+ for identity_counter_key, window_start, backend in reservation_window_identities
+ if identity_counter_key == counter_key
+ ),
+ None,
+ )
+ if not is_reserved_scope or window_identity is None:
+ return ReservationAwareIncrementOperation(
+ key=counter_key,
+ increment_value=increment,
+ ttl=self.window_size,
+ )
+ return ReservationAwareIncrementOperation(
+ key=counter_key,
+ increment_value=increment,
+ ttl=self.window_size,
+ window_key=f"{{{scope_key}:{scope_value}}}:window",
+ expected_window_start=window_identity[0],
+ reservation_backend=window_identity[1],
+ )
+
+ def _build_project_reservation_ops(
+ self,
+ targets: Sequence[tuple[str, str]],
+ reserved_scopes: Set[tuple[str, str]],
+ actual_tokens: int,
+ reserved_tokens: int,
+ reservation_window_identities: frozenset[tuple[str, str, Literal["redis", "local"]]],
+ ) -> tuple[ReservationAwareIncrementOperation, ...]:
+ return tuple(
+ operation
+ for scope in targets
+ if (
+ operation := self._build_project_reservation_op(
+ scope=scope,
+ reserved_scopes=reserved_scopes,
+ actual_tokens=actual_tokens,
+ reserved_tokens=reserved_tokens,
+ reservation_window_identities=reservation_window_identities,
+ )
+ )
+ is not None
+ )
+
def _build_success_event_pipeline_operations(
self,
kwargs: Any,
@@ -3134,12 +4412,26 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
response_obj=response_obj,
rate_limit_type=rate_limit_type,
)
-
if pipeline_operations:
await self.async_increment_tokens_with_ttl_preservation(
pipeline_operations=pipeline_operations,
parent_otel_span=litellm_parent_otel_span,
)
+ io_token_operations: Final = self._build_io_token_reservation_ops(
+ kwargs=kwargs,
+ response_obj=response_obj,
+ )
+ if io_token_operations:
+ if isinstance(io_token_operations, list):
+ await self.async_increment_tokens_with_ttl_preservation(
+ pipeline_operations=io_token_operations,
+ parent_otel_span=litellm_parent_otel_span,
+ )
+ else:
+ await self.async_increment_reservation_aware_tokens(
+ pipeline_operations=io_token_operations,
+ parent_otel_span=litellm_parent_otel_span,
+ )
except Exception as e:
verbose_proxy_logger.exception("Error in rate limit success event: %s", e)
@@ -3232,9 +4524,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
# already released it (proxy-level rejection that also bubbles up
# here as an LLM-error callback). max_parallel_requests is its
# own counter and is always decremented per call.
- reserved_tokens = 0
- if stash is not None and not stash.reservation_released:
- reserved_tokens = stash.reserved_tokens
+ reserved_tokens, itpm_reserved, otpm_reserved = (
+ (0, 0, 0)
+ if stash is None or stash.reservation_released
+ else (stash.reserved_tokens, stash.itpm_reserved_tokens, stash.otpm_reserved_tokens)
+ )
+
if stash is not None and reserved_tokens > 0:
verbose_proxy_logger.debug("Releasing reserved TPM tokens on failure: %s", reserved_tokens)
# Refund only against the scopes the reservation actually
@@ -3251,12 +4546,64 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
)
+ # Refund project ITPM/OTPM reservations the same way -- full
+ # refund, since a failed call has no billable usage to reconcile
+ # against.
+ itpm_operations: Final = (
+ self._build_project_reservation_ops(
+ targets=tuple(stash.itpm_reserved_scopes),
+ reserved_scopes=stash.itpm_reserved_scopes,
+ actual_tokens=0,
+ reserved_tokens=itpm_reserved,
+ reservation_window_identities=stash.itpm_reserved_window_identities,
+ )
+ if stash is not None and itpm_reserved > 0 and stash.itpm_reserved_window_identities
+ else self._build_reservation_aware_tpm_ops(
+ targets=tuple(stash.itpm_reserved_scopes),
+ reserved_scopes=stash.itpm_reserved_scopes,
+ actual_tokens=0,
+ reserved_tokens=itpm_reserved,
+ )
+ if stash is not None and itpm_reserved > 0
+ else ()
+ )
+
+ otpm_operations: Final = (
+ self._build_project_reservation_ops(
+ targets=tuple(stash.otpm_reserved_scopes),
+ reserved_scopes=stash.otpm_reserved_scopes,
+ actual_tokens=0,
+ reserved_tokens=otpm_reserved,
+ reservation_window_identities=stash.otpm_reserved_window_identities,
+ )
+ if stash is not None and otpm_reserved > 0 and stash.otpm_reserved_window_identities
+ else self._build_reservation_aware_tpm_ops(
+ targets=tuple(stash.otpm_reserved_scopes),
+ reserved_scopes=stash.otpm_reserved_scopes,
+ actual_tokens=0,
+ reserved_tokens=otpm_reserved,
+ )
+ if stash is not None and otpm_reserved > 0
+ else ()
+ )
+
if pipeline_operations:
await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline(
increment_list=pipeline_operations,
litellm_parent_otel_span=litellm_parent_otel_span,
)
- if stash is not None and reserved_tokens > 0:
+ for project_operations in (itpm_operations, otpm_operations):
+ if isinstance(project_operations, list):
+ await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline(
+ increment_list=project_operations,
+ litellm_parent_otel_span=litellm_parent_otel_span,
+ )
+ elif project_operations:
+ await self.async_increment_reservation_aware_tokens(
+ pipeline_operations=project_operations,
+ parent_otel_span=litellm_parent_otel_span,
+ )
+ if stash is not None and (reserved_tokens > 0 or itpm_reserved > 0 or otpm_reserved > 0):
stash.reservation_released = True
except Exception as e:
verbose_proxy_logger.exception("Error in rate limit failure event: %s", e)
@@ -3334,19 +4681,19 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
traceback_str: str | None = None,
) -> None:
"""
- Release the parallel-request slot and any TPM reservation when the
- request is rejected after the pre-call hook acquired them but before
- the LLM call ran (e.g. a downstream guardrail/auth hook raised).
- Without this, those resources are stranded — async_log_failure_event
- is a litellm completion-level callback and never fires for proxy-side
- rejections, so a leaked slot would occupy the gauge for the full
- PARALLEL_REQUEST_SLOT_TTL_SECONDS.
+ Release the parallel-request slot and any TPM/ITPM/OTPM reservation
+ when the request is rejected after the pre-call hook acquired them
+ but before the LLM call ran (e.g. a downstream guardrail/auth hook
+ raised). Without this, those resources are stranded —
+ async_log_failure_event is a litellm completion-level callback and
+ never fires for proxy-side rejections, so a leaked slot would occupy
+ the gauge for the full PARALLEL_REQUEST_SLOT_TTL_SECONDS.
Idempotent: the slot release clears the stashed acquisition (and slot
- removal is a no-op ZREM on a second run), and the TPM refund is
- guarded by the stash's ``reservation_released`` flag — if both this
- hook and async_log_failure_event end up running in the same flow, only
- the first release/refund applies.
+ removal is a no-op ZREM on a second run), and the TPM/ITPM/OTPM
+ refund is guarded by the stash's ``reservation_released`` flag — if
+ both this hook and async_log_failure_event end up running in the same
+ flow, only the first release/refund applies.
"""
try:
stash: Final = get_request_stash()
@@ -3362,23 +4709,80 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
if stash.reservation_released:
return
reserved_tokens: Final = stash.reserved_tokens
- if reserved_tokens <= 0:
+ itpm_reserved: Final = stash.itpm_reserved_tokens
+ otpm_reserved: Final = stash.otpm_reserved_tokens
+ if reserved_tokens <= 0 and itpm_reserved <= 0 and otpm_reserved <= 0:
return
- ops: Final = self._build_reservation_aware_tpm_ops(
- targets=list(stash.reserved_scopes),
- reserved_scopes=stash.reserved_scopes,
- actual_tokens=0,
- reserved_tokens=reserved_tokens,
- )
- if ops:
- verbose_proxy_logger.debug(
- "Releasing reserved TPM tokens on proxy-level rejection: %s", reserved_tokens
+ combined_ops: Final = (
+ self._build_reservation_aware_tpm_ops(
+ targets=tuple(stash.reserved_scopes),
+ reserved_scopes=stash.reserved_scopes,
+ actual_tokens=0,
+ reserved_tokens=reserved_tokens,
)
+ if reserved_tokens > 0
+ else ()
+ )
+ itpm_ops: Final = (
+ self._build_project_reservation_ops(
+ targets=tuple(stash.itpm_reserved_scopes),
+ reserved_scopes=stash.itpm_reserved_scopes,
+ actual_tokens=0,
+ reserved_tokens=itpm_reserved,
+ reservation_window_identities=stash.itpm_reserved_window_identities,
+ )
+ if itpm_reserved > 0 and stash.itpm_reserved_window_identities
+ else self._build_reservation_aware_tpm_ops(
+ targets=tuple(stash.itpm_reserved_scopes),
+ reserved_scopes=stash.itpm_reserved_scopes,
+ actual_tokens=0,
+ reserved_tokens=itpm_reserved,
+ )
+ if itpm_reserved > 0
+ else ()
+ )
+ otpm_ops: Final = (
+ self._build_project_reservation_ops(
+ targets=tuple(stash.otpm_reserved_scopes),
+ reserved_scopes=stash.otpm_reserved_scopes,
+ actual_tokens=0,
+ reserved_tokens=otpm_reserved,
+ reservation_window_identities=stash.otpm_reserved_window_identities,
+ )
+ if otpm_reserved > 0 and stash.otpm_reserved_window_identities
+ else self._build_reservation_aware_tpm_ops(
+ targets=tuple(stash.otpm_reserved_scopes),
+ reserved_scopes=stash.otpm_reserved_scopes,
+ actual_tokens=0,
+ reserved_tokens=otpm_reserved,
+ )
+ if otpm_reserved > 0
+ else ()
+ )
+ if combined_ops or itpm_ops or otpm_ops:
+ verbose_proxy_logger.debug(
+ "Releasing reserved tokens on proxy-level rejection: tpm=%s, itpm=%s, otpm=%s",
+ reserved_tokens,
+ itpm_reserved,
+ otpm_reserved,
+ )
+ if combined_ops:
await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline(
- increment_list=ops,
+ increment_list=combined_ops,
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
)
+ for project_ops in (itpm_ops, otpm_ops):
+ if isinstance(project_ops, list):
+ await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline(
+ increment_list=project_ops,
+ litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
+ )
+ elif project_ops:
+ await self.async_increment_reservation_aware_tokens(
+ pipeline_operations=project_ops,
+ parent_otel_span=user_api_key_dict.parent_otel_span,
+ )
stash.reservation_released = True
except Exception as e:
verbose_proxy_logger.exception("Error releasing TPM reservation on post-call failure: %s", e)
diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py
index 8edb3b42dce..781fe264eb8 100644
--- a/litellm/proxy/management_endpoints/common_daily_activity.py
+++ b/litellm/proxy/management_endpoints/common_daily_activity.py
@@ -594,13 +594,16 @@ def _build_aggregated_where_clause(
sql_params.append(adjusted_end)
p += 1
- # Optional entity filter
+ # Optional entity filter; an empty list must match nothing, not everything
if entity_id is not None:
if isinstance(entity_id, list):
- placeholders = ", ".join(f"${p + i}" for i in range(len(entity_id)))
- sql_conditions.append(f'"{entity_id_field}" IN ({placeholders})')
- sql_params.extend(entity_id)
- p += len(entity_id)
+ if entity_id:
+ placeholders = ", ".join(f"${p + i}" for i in range(len(entity_id)))
+ sql_conditions.append(f'"{entity_id_field}" IN ({placeholders})')
+ sql_params.extend(entity_id)
+ p += len(entity_id)
+ else:
+ sql_conditions.append("FALSE")
else:
sql_conditions.append(f'"{entity_id_field}" = ${p}')
sql_params.append(entity_id)
diff --git a/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py b/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py
index 49c0135ff10..8e8545a51cc 100644
--- a/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py
+++ b/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py
@@ -57,20 +57,37 @@ def _model_table(prisma_client: PrismaClient) -> _ModelTableClient:
return ModelRepository(prisma_client).table
-def validate_models_exist(model_names: list[str], llm_router: "Router | None") -> tuple[bool, list[str]]:
+def validate_models_exist(model_names: Sequence[str], llm_router: "Router | None") -> tuple[bool, Sequence[str]]:
"""
Validate that all requested model names exist in the router.
Checks only exact model name matches.
Returns:
- Tuple[bool, List[str]]: (all_valid, missing_models)
+ (all_valid, missing_models)
"""
if llm_router is None:
return False, model_names
- router_model_names: Final = set(llm_router.get_model_names())
- missing: Final = [m for m in model_names if m not in router_model_names]
- return (len(missing) == 0, missing)
+ router_model_names: Final = frozenset(llm_router.get_model_names())
+ missing: Final = tuple(m for m in model_names if m not in router_model_names)
+ return (not missing, missing)
+
+
+async def _missing_models_after_read_through(
+ model_names: Sequence[str], llm_router: "Router | None"
+) -> tuple[str, ...]:
+ from litellm.proxy import proxy_server
+ from litellm.proxy.common_utils.registry_read_through import (
+ model_registry_read_through,
+ )
+
+ _, missing = validate_models_exist(model_names=model_names, llm_router=llm_router)
+ if not missing:
+ return ()
+ for name in missing:
+ await model_registry_read_through.attempt(name)
+ _, still_missing = validate_models_exist(model_names=model_names, llm_router=proxy_server.llm_router)
+ return tuple(still_missing)
def add_access_group_to_deployment(model_info: dict[str, Any], access_group: str) -> tuple[dict[str, Any], bool]:
@@ -101,13 +118,21 @@ def _raise_http_if_reload_degraded_serving(
before: frozenset[str],
written_models: Sequence[tuple[str, object]],
access_group: str,
+ still_desired: frozenset[str] | None,
+ live_after: frozenset[str] | None,
) -> None:
"""Same verdict as the model-write endpoints, expressed through this file's
HTTPException error convention, with the metadata-only obligation: these writes
change group membership, not the models themselves, so a row that was already not
serving before the reload is never blamed here; only a model this reload stopped
serving is reported."""
- missing, collateral = reload_serving_verdict(before=before, written_models=written_models, written_must_serve=False)
+ missing, collateral = reload_serving_verdict(
+ before=before,
+ written_models=written_models,
+ written_must_serve=False,
+ still_desired=still_desired,
+ live_after=live_after,
+ )
gone: Final = tuple(dict.fromkeys((*missing, *collateral)))
if not gone:
return
@@ -390,12 +415,12 @@ async def create_model_group(
# Validate model_names exist in router (only if using model_names path)
if not use_model_ids and has_model_names:
assert data.model_names is not None
- all_valid, missing_models = validate_models_exist(
+ missing_models: Final = await _missing_models_after_read_through(
model_names=data.model_names,
llm_router=llm_router,
)
- if not all_valid:
+ if missing_models:
raise HTTPException(
status_code=400,
detail={"error": f"Model(s) not found: {', '.join(missing_models)}"},
@@ -439,11 +464,13 @@ async def create_model_group(
live_before_reload: Final = live_model_ids_snapshot()
- await clear_cache()
+ reload_outcome: Final = await clear_cache()
_raise_http_if_reload_degraded_serving(
before=live_before_reload,
written_models=updated_pairs,
access_group=data.access_group,
+ still_desired=reload_outcome.still_desired,
+ live_after=reload_outcome.live_after,
)
verbose_proxy_logger.info(
@@ -654,12 +681,12 @@ async def update_access_group(
# Validation: Check if all new models exist (only if using model_names path)
if not use_model_ids and has_model_names:
assert data.model_names is not None
- all_valid, missing_models = validate_models_exist(
+ missing_models: Final = await _missing_models_after_read_through(
model_names=data.model_names,
llm_router=llm_router,
)
- if not all_valid:
+ if missing_models:
raise HTTPException(
status_code=400,
detail={"error": f"Model(s) not found: {', '.join(missing_models)}"},
@@ -699,11 +726,13 @@ async def update_access_group(
# Clear cache and reload models to pick up the access group changes
live_before_reload: Final = live_model_ids_snapshot()
- await clear_cache()
+ reload_outcome: Final = await clear_cache()
_raise_http_if_reload_degraded_serving(
before=live_before_reload,
written_models=list({**dict(stripped_pairs), **dict(updated_pairs)}.items()),
access_group=access_group,
+ still_desired=reload_outcome.still_desired,
+ live_after=reload_outcome.live_after,
)
verbose_proxy_logger.info(
@@ -801,11 +830,13 @@ async def delete_access_group(
# Clear cache and reload models to pick up the access group changes
live_before_reload: Final = live_model_ids_snapshot()
- await clear_cache()
+ reload_outcome: Final = await clear_cache()
_raise_http_if_reload_degraded_serving(
before=live_before_reload,
written_models=removed_pairs,
access_group=access_group,
+ still_desired=reload_outcome.still_desired,
+ live_after=reload_outcome.live_after,
)
verbose_proxy_logger.info(
diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
index 635767f4db7..c5ab7f1fc63 100644
--- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
+++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
@@ -50,7 +50,7 @@ from litellm.proxy.vector_store_endpoints.utils import (
get_litellm_managed_vector_store,
is_allowed_to_call_vector_store_endpoint,
)
-from litellm.secret_managers.main import get_secret_str
+from litellm.secret_managers.main import get_secret_str, str_to_bool
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY,
LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
@@ -1016,15 +1016,21 @@ async def bedrock_proxy_route(
raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
aws_region_name: Final = litellm.utils.get_secret(secret_name="AWS_REGION_NAME")
- if _is_bedrock_agent_runtime_route(endpoint=endpoint): # handle bedrock agents
- base_target_url: Final = f"https://bedrock-agent-runtime.{aws_region_name}.amazonaws.com"
- else:
+ if not _is_bedrock_agent_runtime_route(endpoint=endpoint):
return await bedrock_llm_proxy_route(
endpoint=endpoint,
request=request,
fastapi_response=fastapi_response,
user_api_key_dict=user_api_key_dict,
)
+
+ if _is_bedrock_agent_runtime_passthrough_disabled():
+ raise HTTPException(
+ status_code=403,
+ detail="bedrock-agent-runtime pass-through is disabled on this proxy.",
+ )
+
+ base_target_url: Final = f"https://bedrock-agent-runtime.{aws_region_name}.amazonaws.com"
encoded_endpoint = httpx.URL(endpoint).path
# Ensure endpoint starts with '/' for proper URL construction
@@ -1292,6 +1298,15 @@ def _is_bedrock_agent_runtime_route(endpoint: str) -> bool:
return False
+def _is_bedrock_agent_runtime_passthrough_disabled() -> bool:
+ from litellm.proxy.proxy_server import general_settings
+
+ setting: Final = general_settings.get("disable_bedrock_agent_runtime_passthrough")
+ if isinstance(setting, str):
+ return str_to_bool(setting) is True
+ return setting is True
+
+
@router.api_route(
"/assemblyai/{endpoint:path}",
methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py
index 1c8bce28454..5f6489a69ca 100644
--- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py
+++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py
@@ -229,6 +229,25 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
verbose_proxy_logger.warning("Error calculating image editing cost: %s", e)
return 0.0
+ @staticmethod
+ def _calculate_embeddings_cost(
+ litellm_model_response: EmbeddingResponse,
+ model: str,
+ custom_llm_provider: str,
+ ) -> float:
+ try:
+ return litellm.completion_cost(
+ completion_response=litellm_model_response,
+ model=model,
+ custom_llm_provider=custom_llm_provider,
+ call_type="aembedding",
+ )
+ except Exception as e: # noqa: BLE001 # completion_cost raises bare Exception for unmapped models; cost failure must never drop the spend log
+ verbose_proxy_logger.warning(
+ "Error calculating embeddings cost for model %s, logging spend with cost 0: %s", model, e
+ )
+ return 0.0
+
@staticmethod
def _build_responses_api_response_and_cost(
model: str,
@@ -351,11 +370,10 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
model_response_object=EmbeddingResponse(),
response_type="embedding",
)
- response_cost = litellm.completion_cost(
- completion_response=litellm_model_response,
+ response_cost = OpenAIPassthroughLoggingHandler._calculate_embeddings_cost(
+ litellm_model_response=litellm_model_response,
model=model,
custom_llm_provider=custom_llm_provider,
- call_type="aembedding",
)
litellm_model_response._hidden_params["response_cost"] = response_cost
elif is_image_generation:
@@ -471,6 +489,12 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
except Exception as e:
verbose_proxy_logger.error("Error in OpenAI passthrough cost tracking: %s", e)
+ if not is_chat_completions:
+ unbilled_result: Final[PassThroughEndpointLoggingTypedDict] = {
+ "result": None,
+ "kwargs": kwargs,
+ }
+ return unbilled_result
# Fall back to base handler without cost tracking
base_handler = OpenAIPassthroughLoggingHandler()
return base_handler.passthrough_chat_handler(
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 56036713fa9..be1914ea50c 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -4120,6 +4120,31 @@ def _swap_in_model_cost_map(new_model_cost_map: dict) -> int:
return fetched_model_count
+def should_load_db_object(object_type: str | SupportedDBObjectType) -> bool:
+ """
+ Check if an object type should be loaded from the database based on general_settings.supported_db_objects.
+
+ Args:
+ object_type: Type of object to check (e.g., SupportedDBObjectType.MODELS, "models", etc.)
+
+ Returns:
+ True if the object should be loaded, False otherwise
+ """
+ supported_db_objects: Final = general_settings.get("supported_db_objects", None)
+
+ if supported_db_objects is None:
+ return True
+
+ if not isinstance(supported_db_objects, list):
+ verbose_proxy_logger.warning(
+ "supported_db_objects is not a list, got %s. Loading all objects.", type(supported_db_objects)
+ )
+ return True
+
+ object_type_str: Final = str(object_type)
+ return any(str(obj) == object_type_str for obj in supported_db_objects)
+
+
class ProxyConfig:
"""
Abstraction class on top of config loading/updating logic. Gives us one place to control all config updating logic.
@@ -6522,36 +6547,7 @@ class ProxyConfig:
return config
def _should_load_db_object(self, object_type: str | SupportedDBObjectType) -> bool:
- """
- Check if an object type should be loaded from the database based on general_settings.supported_db_objects.
-
- Args:
- object_type: Type of object to check (e.g., SupportedDBObjectType.MODELS, "models", etc.)
-
- Returns:
- True if the object should be loaded, False otherwise
- """
- global general_settings
-
- # Get the supported_db_objects configuration
- supported_db_objects: Final = general_settings.get("supported_db_objects", None)
-
- # If supported_db_objects is not set, load all objects (default behavior)
- if supported_db_objects is None:
- return True
-
- # If supported_db_objects is set, only load specified objects
- if not isinstance(supported_db_objects, list):
- verbose_proxy_logger.warning(
- "supported_db_objects is not a list, got %s. Loading all objects.", type(supported_db_objects)
- )
- return True
-
- # Convert object_type to string for comparison (handles both str and enum)
- object_type_str: Final = str(object_type)
-
- # Check if the object type is in the list (supports both str and enum values)
- return any(str(obj) == object_type_str for obj in supported_db_objects)
+ return should_load_db_object(object_type=object_type)
async def _get_models_from_db(self, prisma_client: PrismaClient) -> list | None:
"""
@@ -7094,38 +7090,40 @@ class ProxyConfig:
async def _init_guardrails_in_db(self, prisma_client: PrismaClient):
from litellm.proxy.guardrails.guardrail_registry import (
+ GUARDRAIL_RECONCILE_LOCK,
IN_MEMORY_GUARDRAIL_HANDLER,
Guardrail,
GuardrailRegistry,
)
try:
- guardrails_in_db: Final[list[Guardrail]] = await GuardrailRegistry.get_all_guardrails_from_db(
- prisma_client=prisma_client
- )
- verbose_proxy_logger.debug("guardrails from the DB %s", str(guardrails_in_db))
- db_guardrail_ids: Final[set] = set()
- for guardrail in guardrails_in_db:
- guardrail_id = guardrail.get("guardrail_id")
- if guardrail_id:
- db_guardrail_ids.add(guardrail_id)
- try:
- IN_MEMORY_GUARDRAIL_HANDLER.sync_guardrail_from_db(
- guardrail=cast(Guardrail, guardrail),
- )
- except Exception as e: # noqa: BLE001 # one unloadable row must not stop the remaining guardrails
- verbose_proxy_logger.error(
- "litellm.proxy.proxy_server.py::ProxyConfig:_init_guardrails_in_db - "
- "skipping guardrail '%s' (ID: %s): %s: %s",
- guardrail.get("guardrail_name"),
- guardrail_id,
- type(e).__name__,
- e,
- )
+ async with GUARDRAIL_RECONCILE_LOCK:
+ guardrails_in_db: Final[list[Guardrail]] = await GuardrailRegistry.get_all_guardrails_from_db(
+ prisma_client=prisma_client
+ )
+ verbose_proxy_logger.debug("guardrails from the DB %s", str(guardrails_in_db))
+ db_guardrail_ids: Final[set] = set()
+ for guardrail in guardrails_in_db:
+ guardrail_id = guardrail.get("guardrail_id")
+ if guardrail_id:
+ db_guardrail_ids.add(guardrail_id)
+ try:
+ IN_MEMORY_GUARDRAIL_HANDLER.sync_guardrail_from_db(
+ guardrail=cast(Guardrail, guardrail),
+ )
+ except Exception as e: # noqa: BLE001 # one unloadable row must not stop the remaining guardrails
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.py::ProxyConfig:_init_guardrails_in_db - "
+ "skipping guardrail '%s' (ID: %s): %s: %s",
+ guardrail.get("guardrail_name"),
+ guardrail_id,
+ type(e).__name__,
+ e,
+ )
- # Drop in-memory DB-backed entries whose row was deleted on another
- # pod. Config-loaded entries are never touched.
- IN_MEMORY_GUARDRAIL_HANDLER.reconcile_db_guardrails(db_guardrail_ids=db_guardrail_ids)
+ # Drop in-memory DB-backed entries whose row was deleted on another
+ # pod. Config-loaded entries are never touched.
+ IN_MEMORY_GUARDRAIL_HANDLER.reconcile_db_guardrails(db_guardrail_ids=db_guardrail_ids)
except Exception as e:
verbose_proxy_logger.exception("litellm.proxy.proxy_server.py::ProxyConfig:_init_guardrails_in_db - %s", e)
@@ -7278,13 +7276,17 @@ class ProxyConfig:
)
async def _init_agents_in_db(self, prisma_client: PrismaClient):
+ from litellm.proxy.agent_endpoints.agent_registry import (
+ AGENT_RECONCILE_LOCK,
+ )
from litellm.proxy.agent_endpoints.agent_registry import (
global_agent_registry as AGENT_REGISTRY,
)
try:
- db_agents: Final = await AGENT_REGISTRY.get_all_agents_from_db(prisma_client=prisma_client)
- AGENT_REGISTRY.load_agents_from_db_and_config(db_agents=db_agents)
+ async with AGENT_RECONCILE_LOCK:
+ db_agents: Final = await AGENT_REGISTRY.get_all_agents_from_db(prisma_client=prisma_client)
+ AGENT_REGISTRY.load_agents_from_db_and_config(db_agents=db_agents)
except Exception as e:
verbose_proxy_logger.exception("litellm.proxy.proxy_server.py::ProxyConfig:_init_agents_in_db - %s", e)
@@ -10254,11 +10256,9 @@ async def embeddings(
"""
global proxy_logging_obj
- data: Any = {}
+ data: Final = await _read_request_body(request=request)
+ base_llm_response_processor: Final = ProxyBaseLLMRequestProcessing(data=data)
try:
- # Use shared request body reading helper (same as chat/completions)
- data = await _read_request_body(request=request)
-
### HANDLE TOKEN ARRAY INPUT DECODING ###
# This must happen BEFORE base_process_llm_request() since it modifies the input
router_model_names: Final = llm_router.model_names if llm_router is not None else []
@@ -10302,10 +10302,6 @@ async def embeddings(
if hasattr(user_api_key_dict, "agent_id") and user_api_key_dict.agent_id is not None:
data["metadata"]["agent_id"] = user_api_key_dict.agent_id
- # Use unified request processor (same as chat/completions and responses)
- base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data)
-
- # Process the request with all optimizations (shared sessions, network tuning, etc.)
response: Final = await base_llm_response_processor.base_process_llm_request(
request=request,
fastapi_response=fastapi_response,
@@ -10327,8 +10323,6 @@ async def embeddings(
return response
except Exception as e:
- # Use unified error handler
- base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data)
raise await base_llm_response_processor._handle_llm_api_exception(
e=e,
user_api_key_dict=user_api_key_dict,
diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py
index c85325b6fa9..91a0c68fd58 100644
--- a/litellm/proxy/route_llm_request.py
+++ b/litellm/proxy/route_llm_request.py
@@ -146,7 +146,8 @@ ROUTE_ENDPOINT_MAPPING: Final = {
class ProxyModelNotFoundError(HTTPException):
- def __init__(self, route: str, model_name: str):
+ def __init__(self, route: str, model_name: str, retryable_with_model_read_through: bool = True):
+ self.retryable_with_model_read_through: Final = retryable_with_model_read_through
detail: Final = {
"error": f"{route}: Invalid model name passed in model={model_name}. Call `/v1/models` to view available models for your key."
}
@@ -320,112 +321,150 @@ async def add_shared_session_to_data(data: dict) -> None:
pass
+RouteType = Literal[
+ "acompletion",
+ "atext_completion",
+ "aembedding",
+ "aimage_generation",
+ "aspeech",
+ "atranscription",
+ "amoderation",
+ "arerank",
+ "aresponses",
+ "aget_responses",
+ "adelete_responses",
+ "acancel_responses",
+ "acompact_responses",
+ "acreate_response_reply",
+ "alist_input_items",
+ "_arealtime", # private function for realtime API
+ "acreate_realtime_client_secret",
+ "arealtime_calls",
+ "acreate_realtime_transcription_session",
+ "_aresponses_websocket", # private function for responses WebSocket mode
+ "aimage_edit",
+ "agenerate_content",
+ "agenerate_content_stream",
+ "allm_passthrough_route",
+ "acreate_batch",
+ "aretrieve_batch",
+ "alist_batches",
+ "afile_content",
+ "afile_retrieve",
+ "acreate_fine_tuning_job",
+ "acancel_fine_tuning_job",
+ "alist_fine_tuning_jobs",
+ "aretrieve_fine_tuning_job",
+ "avector_store_search",
+ "avector_store_create",
+ "avector_store_retrieve",
+ "avector_store_list",
+ "avector_store_update",
+ "avector_store_delete",
+ "avector_store_file_create",
+ "avector_store_file_list",
+ "avector_store_file_retrieve",
+ "avector_store_file_content",
+ "avector_store_file_update",
+ "avector_store_file_delete",
+ "aocr",
+ "asearch",
+ "avideo_generation",
+ "avideo_list",
+ "avideo_status",
+ "avideo_content",
+ "avideo_remix",
+ "avideo_create_character",
+ "avideo_get_character",
+ "avideo_edit",
+ "avideo_extension",
+ "acreate_container",
+ "alist_containers",
+ "aretrieve_container",
+ "adelete_container",
+ "aupload_container_file",
+ "alist_container_files",
+ "aretrieve_container_file",
+ "adelete_container_file",
+ "aretrieve_container_file_content",
+ "acreate_skill",
+ "alist_skills",
+ "aget_skill",
+ "adelete_skill",
+ "aingest",
+ "anthropic_messages",
+ "acreate_interaction",
+ "aget_interaction",
+ "adelete_interaction",
+ "acancel_interaction",
+ "acreate_agent",
+ "alist_agents",
+ "aget_agent",
+ "adelete_agent",
+ "alist_agent_versions",
+ "asend_message",
+ "call_mcp_tool",
+ "acancel_batch",
+ "afile_delete",
+ "acreate_eval",
+ "alist_evals",
+ "aget_eval",
+ "aupdate_eval",
+ "adelete_eval",
+ "acancel_eval",
+ "acreate_run",
+ "alist_runs",
+ "aget_run",
+ "acancel_run",
+ "adelete_run",
+]
+
+
async def route_request(
data: dict,
llm_router: LitellmRouter | None,
user_model: str | None,
- route_type: Literal[
- "acompletion",
- "atext_completion",
- "aembedding",
- "aimage_generation",
- "aspeech",
- "atranscription",
- "amoderation",
- "arerank",
- "aresponses",
- "aget_responses",
- "adelete_responses",
- "acancel_responses",
- "acompact_responses",
- "acreate_response_reply",
- "alist_input_items",
- "_arealtime", # private function for realtime API
- "acreate_realtime_client_secret",
- "arealtime_calls",
- "acreate_realtime_transcription_session",
- "_aresponses_websocket", # private function for responses WebSocket mode
- "aimage_edit",
- "agenerate_content",
- "agenerate_content_stream",
- "allm_passthrough_route",
- "acreate_batch",
- "aretrieve_batch",
- "alist_batches",
- "afile_content",
- "afile_retrieve",
- "acreate_fine_tuning_job",
- "acancel_fine_tuning_job",
- "alist_fine_tuning_jobs",
- "aretrieve_fine_tuning_job",
- "avector_store_search",
- "avector_store_create",
- "avector_store_retrieve",
- "avector_store_list",
- "avector_store_update",
- "avector_store_delete",
- "avector_store_file_create",
- "avector_store_file_list",
- "avector_store_file_retrieve",
- "avector_store_file_content",
- "avector_store_file_update",
- "avector_store_file_delete",
- "aocr",
- "asearch",
- "avideo_generation",
- "avideo_list",
- "avideo_status",
- "avideo_content",
- "avideo_remix",
- "avideo_create_character",
- "avideo_get_character",
- "avideo_edit",
- "avideo_extension",
- "acreate_container",
- "alist_containers",
- "aretrieve_container",
- "adelete_container",
- "aupload_container_file",
- "alist_container_files",
- "aretrieve_container_file",
- "adelete_container_file",
- "aretrieve_container_file_content",
- "acreate_skill",
- "alist_skills",
- "aget_skill",
- "adelete_skill",
- "aingest",
- "anthropic_messages",
- "acreate_interaction",
- "aget_interaction",
- "adelete_interaction",
- "acancel_interaction",
- "acreate_agent",
- "alist_agents",
- "aget_agent",
- "adelete_agent",
- "alist_agent_versions",
- "asend_message",
- "call_mcp_tool",
- "acancel_batch",
- "afile_delete",
- "acreate_eval",
- "alist_evals",
- "aget_eval",
- "aupdate_eval",
- "adelete_eval",
- "acancel_eval",
- "acreate_run",
- "alist_runs",
- "aget_run",
- "acancel_run",
- "adelete_run",
- ],
+ route_type: RouteType,
user_api_key_dict: UserAPIKeyAuth | None = None,
):
"""
Common helper to route the request
"""
+ try:
+ return await _route_request_single_attempt(
+ data=data,
+ llm_router=llm_router,
+ user_model=user_model,
+ route_type=route_type,
+ user_api_key_dict=user_api_key_dict,
+ )
+ except ProxyModelNotFoundError as e:
+ requested_model: Final = data.get("model", "")
+ if not e.retryable_with_model_read_through or not isinstance(requested_model, str) or not requested_model:
+ raise
+ from litellm.proxy import proxy_server
+ from litellm.proxy.common_utils.registry_read_through import (
+ model_registry_read_through,
+ )
+
+ if not await model_registry_read_through.attempt(requested_model):
+ raise
+ return await _route_request_single_attempt(
+ data=data,
+ llm_router=proxy_server.llm_router,
+ user_model=user_model,
+ route_type=route_type,
+ user_api_key_dict=user_api_key_dict,
+ )
+
+
+async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited provider coroutines; the inferred union keeps route_request's callers typed
+ data: dict, # mutable-ok: request body is the proxy-wide mutable dict contract shared with route_request
+ llm_router: LitellmRouter | None,
+ user_model: str | None,
+ route_type: RouteType,
+ user_api_key_dict: UserAPIKeyAuth | None = None,
+):
raise_if_required_body_param_missing(route_type=route_type, data=data)
await add_shared_session_to_data(data)
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index 2ad7180bd5f..a743526e975 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -40,7 +40,7 @@ from litellm.proxy._types import (
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.proxy.model_listing import ModelInfoResponse
-from litellm.types.utils import CallTypes, CallTypesLiteral, ModelInfo
+from litellm.types.utils import CallTypes, CallTypesLiteral, ModelInfo, Usage
try:
from litellm_enterprise.enterprise_callbacks.send_emails.base_email import (
@@ -403,6 +403,120 @@ def _exception_changes_request_flow(exc: BaseException) -> bool:
return isinstance(exc, (SensitiveDataRouteException, ModifyResponseException))
+def _prompt_block_text(block: object) -> str:
+ if isinstance(block, str):
+ return block
+ if not isinstance(block, dict):
+ return ""
+ block_text: Final = block.get("text")
+ return block_text if isinstance(block_text, str) else ""
+
+
+def _system_prompt_text(system_input: object) -> str:
+ if isinstance(system_input, str):
+ return system_input
+ if not isinstance(system_input, list):
+ return ""
+ return "".join(_prompt_block_text(block) for block in system_input)
+
+
+def _count_request_input_tokens(model: str, request_input: object, system_input: object) -> int:
+ system_text: Final = _system_prompt_text(system_input)
+ system_tokens: Final = litellm.token_counter(model=model, text=system_text) if system_text else 0
+ if isinstance(request_input, str):
+ return system_tokens + litellm.token_counter(model=model, text=request_input)
+ if not isinstance(request_input, list) or not request_input:
+ return system_tokens
+ text_entries: Final = tuple(entry for entry in request_input if isinstance(entry, str))
+ if len(text_entries) == len(request_input):
+ return system_tokens + litellm.token_counter(model=model, text="".join(text_entries))
+ return system_tokens + litellm.token_counter(
+ model=model, messages=request_input, use_default_image_token_count=True
+ )
+
+
+def _estimate_dispatched_failure_usage(model: str, request_input: object, system_input: object) -> Usage | None:
+ """A request that failed after dispatch consumed provider-billed input
+ tokens, but no provider usage ever came back. Estimate the input side with
+ the same tokenizer fallback interrupted streams use, so the spend log's
+ failure row records what was sent instead of zero."""
+ try:
+ input_tokens: Final = _count_request_input_tokens(
+ model=model, request_input=request_input, system_input=system_input
+ )
+ except Exception:
+ return None
+ if input_tokens <= 0:
+ return None
+ return Usage(prompt_tokens=input_tokens, completion_tokens=0, total_tokens=input_tokens)
+
+
+_INPUT_ESTIMABLE_CALL_TYPES: Final = frozenset(
+ call_type.value
+ for call_type in (
+ CallTypes.completion,
+ CallTypes.acompletion,
+ CallTypes.text_completion,
+ CallTypes.atext_completion,
+ CallTypes.anthropic_messages,
+ CallTypes.aanthropic_messages,
+ CallTypes.responses,
+ CallTypes.aresponses,
+ CallTypes.embedding,
+ CallTypes.aembedding,
+ CallTypes.moderation,
+ CallTypes.amoderation,
+ CallTypes.image_generation,
+ CallTypes.aimage_generation,
+ CallTypes.speech,
+ CallTypes.aspeech,
+ CallTypes.rerank,
+ CallTypes.arerank,
+ CallTypes.generate_content,
+ CallTypes.agenerate_content,
+ CallTypes.generate_content_stream,
+ CallTypes.agenerate_content_stream,
+ )
+)
+
+
+def _failure_usage_to_lift(
+ model_call_details: Mapping[str, object],
+ request_body: Mapping[str, object],
+ dispatched: bool,
+) -> tuple[object, object] | None:
+ """A stream that broke mid-flight still billed the provider for the chunks
+ already delivered; the streaming handler stashes that recovered usage and
+ cost in model_call_details, so prefer it. Otherwise a request that was
+ dispatched to a provider and failed without upstream usage gets an
+ estimated input-side Usage with zero cost. The raw request body backfills
+ the system prompt when the SDK bridges an endpoint (e.g. /v1/messages on a
+ chat-completions provider) without filling optional_params. Returns the
+ (combined_usage_object, response_cost) pair to lift, or None."""
+ recovered_usage: Final = model_call_details.get("combined_usage_object")
+ if recovered_usage is not None:
+ return recovered_usage, model_call_details.get("response_cost")
+ if not dispatched or model_call_details.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL):
+ return None
+ if str(model_call_details.get("call_type")) not in _INPUT_ESTIMABLE_CALL_TYPES:
+ return None
+ optional_params: Final = model_call_details.get("optional_params")
+ dispatched_system: Final = (
+ (optional_params.get("system") or optional_params.get("instructions"))
+ if isinstance(optional_params, dict)
+ else None
+ )
+ system_input: Final = dispatched_system or request_body.get("system") or request_body.get("instructions")
+ estimated_usage: Final = _estimate_dispatched_failure_usage(
+ model=str(model_call_details.get("model") or ""),
+ request_input=model_call_details.get("messages"),
+ system_input=system_input,
+ )
+ if estimated_usage is None:
+ return None
+ return estimated_usage, 0.0
+
+
@dataclass(frozen=True)
class _CallbackCapabilities:
"""Cached per-hook capability flags derived from ``litellm.callbacks``.
@@ -2190,15 +2304,19 @@ class ProxyLogging:
if _first_handoff is not None:
request_data["first_api_call_start_time"] = _first_handoff
- # A stream that broke mid-flight still billed the provider for the
- # chunks already delivered; the streaming handler stashes that
- # recovered usage and cost here. Lift them onto request_data so the
+ # Lift recovered partial-stream usage, or an estimated input-side
+ # usage for a dispatched failure, onto request_data so the
# failure-path spend callbacks (which run after the logging object
- # is popped) record the real partial spend instead of zero.
- _recovered_usage: Final = _model_call_details.get("combined_usage_object")
- if _recovered_usage is not None:
- request_data["combined_usage_object"] = _recovered_usage
- request_data["response_cost"] = _model_call_details.get("response_cost")
+ # is popped) record real token counts instead of zero.
+ _usage_to_lift: Final = _failure_usage_to_lift(
+ model_call_details=_model_call_details,
+ request_body=request_data,
+ dispatched=_first_handoff is not None,
+ )
+ if _usage_to_lift is not None:
+ _lifted_usage, _lifted_cost = _usage_to_lift
+ request_data["combined_usage_object"] = _lifted_usage
+ request_data["response_cost"] = _lifted_cost
# Remove before callbacks iterate — not serialisable
request_data.pop("litellm_logging_obj", None)
diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py
index 25e5fcb6976..e678fba2852 100644
--- a/litellm/responses/streaming_iterator.py
+++ b/litellm/responses/streaming_iterator.py
@@ -20,7 +20,7 @@ from litellm.constants import (
LITELLM_MAX_STREAMING_DURATION_SECONDS,
STREAM_SSE_DONE_STRING,
)
-from litellm.exceptions import MidStreamFallbackError
+from litellm.exceptions import MidStreamFallbackError, RateLimitError
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@@ -50,6 +50,16 @@ if TYPE_CHECKING:
)
+class ProjectQuotaCallback(Protocol):
+ async def enforce_project_io_token_quota_for_frame(
+ self,
+ user_api_key_dict: UserAPIKeyAuth | None,
+ requested_model: str | None,
+ estimated_input_tokens: int,
+ estimated_output_tokens: int,
+ ) -> None: ...
+
+
@lru_cache(maxsize=1)
def _get_openai_response_types():
from litellm.types.llms import openai as openai_types
@@ -1326,6 +1336,84 @@ def _build_synthetic_response_events(
from litellm._logging import verbose_logger
+# Conservative per-frame output-token floor used when a response.create
+# frame omits max_output_tokens, so a project OTPM quota can't be bypassed
+# by simply never declaring an output cap.
+_FRAME_NO_MAX_OUTPUT_TOKENS_FLOOR: Final = 1024
+
+# Rough chars-per-token ratio for estimating a frame's input tokens without
+# resolving a real per-model tokenizer, matching the conservative estimate
+# the proxy's own rate limiter uses for the same purpose.
+_FRAME_CHARS_PER_TOKEN_ESTIMATE: Final = 4
+
+
+def _extract_frame_quota_estimate_inputs(msg_obj: Mapping[str, object]) -> tuple[int, int | None]:
+ """Extract a rough input-token count and any explicit max_output_tokens
+ from a ``response.create`` frame, handling both wire shapes:
+ flat: {"type": "response.create", "input": ..., "max_output_tokens": ...}
+ nested: {"type": "response.create", "response": {"input": ..., "max_output_tokens": ...}}
+ """
+ nested: Final = msg_obj.get("response")
+ params: Final[Mapping[str, object]] = (
+ nested
+ if _is_json_object(nested) and nested
+ else MappingProxyType( # mutable-ok: immediately frozen filtered frame
+ {k: v for k, v in msg_obj.items() if k != "type"}
+ )
+ )
+ text_parts: Final[list[str]] = [] # mutable-ok: local accumulator built in one pass, not shared
+ pending: Final[list[object]] = [ # mutable-ok: explicit worklist avoids recursion
+ params.get("input"),
+ params.get("instructions"),
+ ]
+ while pending:
+ value = pending.pop()
+ if isinstance(value, str):
+ text_parts.append(value)
+ elif _is_json_array(value):
+ for item in value:
+ if isinstance(item, str):
+ text_parts.append(item)
+ elif _is_json_object(item):
+ pending.append(item.get("content"))
+ pending.append(item.get("text"))
+ total_chars: Final = sum(len(part) for part in text_parts)
+ estimated_input_tokens: Final = max(1, total_chars // _FRAME_CHARS_PER_TOKEN_ESTIMATE) if total_chars else 0
+
+ max_output_tokens: Final = params.get("max_output_tokens")
+ return estimated_input_tokens, max_output_tokens if isinstance(max_output_tokens, int) else None
+
+
+async def _enforce_frame_project_quota(
+ quota_callbacks: Sequence[ProjectQuotaCallback],
+ user_api_key_dict: UserAPIKeyAuth | None,
+ model: str | None,
+ raw_message: str,
+) -> None:
+ """Charge one response.create frame's estimated tokens against every
+ registered project ITPM/OTPM quota callback, in isolation from PII
+ masking / logging so a malformed frame still reaches those callbacks."""
+ if not quota_callbacks:
+ return
+ try:
+ msg_obj = json.loads(raw_message)
+ except (json.JSONDecodeError, TypeError):
+ return
+ if not _is_json_object(msg_obj) or msg_obj.get("type") != "response.create":
+ return
+ estimated_input_tokens, explicit_max_output_tokens = _extract_frame_quota_estimate_inputs(msg_obj)
+ estimated_output_tokens: Final = (
+ explicit_max_output_tokens if explicit_max_output_tokens is not None else _FRAME_NO_MAX_OUTPUT_TOKENS_FLOOR
+ )
+ for callback in quota_callbacks:
+ await callback.enforce_project_io_token_quota_for_frame(
+ user_api_key_dict=user_api_key_dict,
+ requested_model=model,
+ estimated_input_tokens=estimated_input_tokens,
+ estimated_output_tokens=estimated_output_tokens,
+ )
+
+
RESPONSES_WS_LOGGED_EVENT_TYPES: Final = [
"response.created",
"response.completed",
@@ -1360,6 +1448,7 @@ class ResponsesWebSocketStreaming:
first_message: str | None = None,
guardrail_callbacks: list[Any] | None = None,
output_guardrail_callbacks: list[PresidioGuardrailCallback] | None = None,
+ quota_callbacks: Sequence[ProjectQuotaCallback] | None = None,
authorized_model: str | None = None,
):
self.websocket = websocket
@@ -1372,6 +1461,7 @@ class ResponsesWebSocketStreaming:
self.first_message = first_message
self.guardrail_callbacks: list[Any] = guardrail_callbacks or []
self.output_guardrail_callbacks: list[PresidioGuardrailCallback] = output_guardrail_callbacks or []
+ self.quota_callbacks: tuple[ProjectQuotaCallback, ...] = tuple(quota_callbacks) if quota_callbacks else ()
# Model name authorized at connection time; enforced on every
# response.create frame to prevent deployment-substitution attacks.
self.authorized_model: str | None = authorized_model
@@ -1781,10 +1871,39 @@ class ResponsesWebSocketStreaming:
return json.dumps(evt_obj) if modified else response_str
+ async def _enforce_or_reject_frame(self, message: str) -> bool:
+ """Run the per-frame project quota check.
+
+ On rejection, sends an ``error`` event to the client and reports that
+ the frame must be dropped instead of forwarded, so the connection
+ stays open for the client to retry once the window resets.
+ """
+ try:
+ await _enforce_frame_project_quota(
+ self.quota_callbacks, self.user_api_key_dict, self.authorized_model, message
+ )
+ except RateLimitError as e:
+ try:
+ await self.websocket.send_text(
+ json.dumps( # mutable-ok: WebSocket wire payload requires JSON objects
+ { # mutable-ok: WebSocket wire payload requires JSON objects
+ "type": "error",
+ "error": { # mutable-ok: nested WebSocket error object
+ "type": "rate_limit_exceeded",
+ "message": str(e),
+ },
+ }
+ )
+ )
+ except Exception: # noqa: BLE001, S110 # client may already be gone
+ pass
+ return False
+ return True
+
async def client_to_backend(self) -> None:
"""Forward response.create events from client to backend."""
try:
- if self.first_message is not None:
+ if self.first_message is not None and await self._enforce_or_reject_frame(self.first_message):
masked_first: Final = await self._mask_response_create(self.first_message)
self._store_input(masked_first)
self._store_event(masked_first)
@@ -1792,6 +1911,8 @@ class ResponsesWebSocketStreaming:
while True:
message = await self.websocket.receive_text()
+ if not await self._enforce_or_reject_frame(message):
+ continue
masked = await self._mask_response_create(message)
self._store_input(masked)
self._store_event(masked)
@@ -1871,6 +1992,7 @@ class ManagedResponsesWebSocketHandler:
timeout: float | None = None,
custom_llm_provider: str | None = None,
first_message: str | None = None,
+ quota_callbacks: Sequence[ProjectQuotaCallback] | None = None,
**kwargs: object,
) -> None:
self.websocket = websocket
@@ -1887,6 +2009,7 @@ class ManagedResponsesWebSocketHandler:
self.custom_llm_provider = custom_llm_provider
self._connection_provider = self._resolve_provider(model) or custom_llm_provider
self.first_message = first_message
+ self.quota_callbacks: tuple[ProjectQuotaCallback, ...] = tuple(quota_callbacks) if quota_callbacks else ()
# Carry through safe pass-through kwargs (e.g. extra_headers)
self.extra_kwargs: dict[str, object] = {k: v for k, v in kwargs.items() if k not in _MANAGED_WS_SKIP_KWARGS}
# In-memory session history: response_id → full accumulated message list.
@@ -2292,6 +2415,14 @@ class ManagedResponsesWebSocketHandler:
verbose_logger.debug("ManagedResponsesWS: error sending warmup ack: %s", exc)
return
+ try:
+ await _enforce_frame_project_quota(
+ self.quota_callbacks, self.user_api_key_dict, self.model_group or self.model, raw_message
+ )
+ except RateLimitError as e:
+ await self._send_error(str(e), error_type="rate_limit_exceeded")
+ return
+
call_kwargs: Final = self._build_base_call_kwargs(msg_obj)
call_kwargs["stream"] = True
diff --git a/litellm/router.py b/litellm/router.py
index c5881960c80..00c6c4c8b6f 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -8573,11 +8573,9 @@ class Router:
Returns:
- The added/updated deployment
"""
+ _deployment_model_id: Final = deployment.model_info.id or ""
+ _deployment_on_router: Final[Deployment | None] = self.get_deployment(model_id=_deployment_model_id)
try:
- # check if deployment already exists
- _deployment_model_id: Final = deployment.model_info.id or ""
-
- _deployment_on_router: Final[Deployment | None] = self.get_deployment(model_id=_deployment_model_id)
if _deployment_on_router is not None:
# deployment with this model_id exists on the router
if (
@@ -8628,10 +8626,31 @@ class Router:
deployment.model_info.id,
e,
)
+ self._restore_deployment_after_failed_upsert(
+ previous_deployment=_deployment_on_router, model_id=_deployment_model_id
+ )
return None
else:
raise e
+ def _restore_deployment_after_failed_upsert(self, previous_deployment: Deployment | None, model_id: str) -> None:
+ if previous_deployment is None or self.has_model_id(model_id):
+ return
+ try:
+ self.add_deployment(deployment=previous_deployment)
+ verbose_router_logger.info(
+ "Restored deployment %s (id=%s); it keeps serving its previous configuration.",
+ previous_deployment.model_name,
+ model_id,
+ )
+ except Exception as restore_error: # noqa: BLE001 # best-effort restore: a second failure must not abort the reload
+ verbose_router_logger.warning(
+ "Could not restore previously served deployment %s (id=%s) after the failed upsert: %s",
+ previous_deployment.model_name,
+ model_id,
+ restore_error,
+ )
+
@staticmethod
def _backend_cost_map_keys(model: str, custom_llm_provider: str | None) -> tuple[str, ...]:
"""The ``litellm.model_cost`` keys a deployment's shared backend info is registered under."""
@@ -11463,8 +11482,10 @@ class Router:
deployment the strategy was registered from via its (model_name, tags)
pair.
- With tag filtering enabled, strategies that all carry real tags matching
- none of the request's do not capture it when the name also has plain
+ With tag filtering enabled, router-wide or by the request's
+ enable_tag_filtering (which the proxy sets from key/team
+ router_settings), strategies that all carry real tags matching none of
+ the request's do not capture it when the name also has plain
deployments: returning None hands the request to ordinary tag-aware
deployment selection.
"""
@@ -11487,8 +11508,9 @@ class Router:
for tagged in candidates:
if "default" in tagged.tags:
return tagged
+ request_scoped_filtering: Final = request_kwargs.get("enable_tag_filtering") is True
if (
- self.enable_tag_filtering
+ (self.enable_tag_filtering or request_scoped_filtering)
and all(tagged.tags for tagged in candidates)
and self._model_name_has_plain_deployments(model)
):
diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py
index 63bc5203417..3c9a4097321 100644
--- a/litellm/router_utils/fallback_event_handlers.py
+++ b/litellm/router_utils/fallback_event_handlers.py
@@ -253,6 +253,7 @@ def get_fallback_model_group(fallbacks: list[Any], model_group: str) -> tuple[li
PROVIDER_SCOPED_RESOURCE_KEYS: Final = ("input_file_id", "training_file")
+PROVIDER_SCOPED_CREATION_FUNCTION_NAMES: Final = frozenset({"_acreate_file"})
def _get_fallback_target_model_group(fallback_entry: str | Mapping[str, object]) -> str | None:
@@ -274,6 +275,18 @@ def references_provider_scoped_resource(kwargs: Mapping[str, object]) -> bool:
return any(kwargs.get(key) for key in PROVIDER_SCOPED_RESOURCE_KEYS)
+def creates_provider_scoped_resource(kwargs: Mapping[str, object]) -> bool:
+ """
+ True when the request creates a resource that will live under one provider's credentials.
+
+ A file uploaded for batches or fine-tuning is stored in the account of the deployment
+ that handled it, and its id is only usable against the model group the caller named.
+ Letting the upload fall back to a different model group silently stores the file with
+ the wrong provider, and every later use of the returned id fails.
+ """
+ return getattr(kwargs.get("original_function"), "__name__", None) in PROVIDER_SCOPED_CREATION_FUNCTION_NAMES
+
+
async def run_async_fallback(
*args: tuple[Any],
litellm_router: LitellmRouter,
@@ -322,7 +335,9 @@ async def run_async_fallback(
metadata_variable_name: Final = _get_router_metadata_variable_name(
function_name=getattr(kwargs.get("original_function"), "__name__", None)
)
- same_model_group_only: Final = references_provider_scoped_resource(kwargs)
+ same_model_group_only: Final = references_provider_scoped_resource(kwargs) or creates_provider_scoped_resource(
+ kwargs
+ )
# Read out of kwargs and narrowed here rather than declared as a parameter: every caller
# reaches this function by spreading a loosely-typed kwargs dict, so a declared parameter
# would carry an annotation that no call site can actually be checked against.
diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py
index aeeeca21d3b..d09503cdc4d 100644
--- a/litellm/types/mcp_server/mcp_server_manager.py
+++ b/litellm/types/mcp_server/mcp_server_manager.py
@@ -224,6 +224,14 @@ class MCPServer(BaseModel):
JWT) but forwards the caller's separate upstream ``Authorization`` unchanged, minting nothing."""
return self.auth_type == MCPAuth.oauth_delegate
+ @property
+ def is_client_forwarded_token(self) -> bool:
+ """True for the two modes whose upstream credential is the caller's own bearer, forwarded
+ unchanged: the gateway mints nothing for them and holds no OAuth client identity, so a
+ discovered ``authorization_url`` / ``token_url`` enriches only the gateway's own OAuth front
+ door and is never a precondition for opening a session."""
+ return self.is_true_passthrough or self.is_oauth_delegate
+
@property
def is_dcr_bridge(self) -> bool:
"""True when this client-forwarded-token server serves the gateway-hosted DCR front door
@@ -231,7 +239,7 @@ class MCPServer(BaseModel):
authorize, and token relays) instead of relaying the upstream's own OAuth discovery
verbatim. ``dcr_bridge`` is rejected on every other auth type at create, update, and
config load, so the mode gate here only defends rows edited outside those paths."""
- return bool(self.dcr_bridge) and (self.is_true_passthrough or self.is_oauth_delegate)
+ return bool(self.dcr_bridge) and self.is_client_forwarded_token
@property
def requires_per_user_auth(self) -> bool:
@@ -248,7 +256,7 @@ class MCPServer(BaseModel):
if self.needs_user_oauth_token:
return True
- if self.is_true_passthrough or self.is_oauth_delegate:
+ if self.is_client_forwarded_token:
return True
# PAT passthrough: auth_type is none but extra_headers includes auth headers
diff --git a/litellm/types/utils.py b/litellm/types/utils.py
index 6f6f317b257..5b925e610e7 100644
--- a/litellm/types/utils.py
+++ b/litellm/types/utils.py
@@ -1631,8 +1631,20 @@ class PromptTokensDetailsWrapper(
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
+ extra_fields: Final = self.model_extra
+ nested_cache_creation_input_tokens: Final = (
+ extra_fields.get("cache_creation_input_tokens") if extra_fields is not None else None
+ )
self.cache_write_tokens = (
- self.cache_write_tokens if self.cache_write_tokens is not None else self.cache_creation_tokens
+ self.cache_write_tokens
+ if self.cache_write_tokens is not None
+ else (
+ self.cache_creation_tokens
+ if self.cache_creation_tokens is not None
+ else (
+ nested_cache_creation_input_tokens if isinstance(nested_cache_creation_input_tokens, int) else None
+ )
+ )
)
if self.character_count is None:
del self.character_count
diff --git a/litellm/utils.py b/litellm/utils.py
index 1c880ee9521..a7b70c4129a 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -65,6 +65,7 @@ from litellm.constants import (
DEFAULT_EMBEDDING_PARAM_VALUES,
DEFAULT_MAX_LRU_CACHE_SIZE,
DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT,
+ DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
DEFAULT_TRIM_RATIO,
FUNCTION_DEFINITION_TOKEN_COUNT,
INITIAL_RETRY_DELAY,
@@ -7638,12 +7639,20 @@ def validate_and_fix_openai_tools(tools: list | None) -> list[dict] | None:
def validate_and_fix_thinking_param(
- thinking: AnthropicThinkingParam | None,
+ thinking: AnthropicThinkingParam | bool | None,
) -> AnthropicThinkingParam | None:
"""
- Normalizes camelCase keys in the thinking param to snake_case.
+ Coerces bool thinking values (True becomes enabled with the default medium budget, False becomes None)
+ and normalizes camelCase keys in the thinking param to snake_case.
Handles clients that send budgetTokens instead of budget_tokens.
"""
+ if thinking is True:
+ return cast(
+ "AnthropicThinkingParam",
+ {"type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET},
+ )
+ if thinking is False:
+ return None
if thinking is None or not isinstance(thinking, dict):
return thinking
normalized: Final = dict(thinking)
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 409022016b0..07f9027313b 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -14456,6 +14456,25 @@
"supports_tool_choice": true,
"supports_output_config": true
},
+ "databricks/databricks-claude-opus-4-6": {
+ "input_cost_per_token": 5.00003e-06,
+ "input_dbu_cost_per_token": 7.1429e-05,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 2.5000010000000002e-05,
+ "output_dbu_cost_per_token": 0.000357143,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+ "supports_assistant_prefill": true,
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true
+ },
"databricks/databricks-claude-sonnet-4": {
"input_cost_per_token": 2.9999900000000002e-06,
"input_dbu_cost_per_token": 4.2857e-05,
@@ -14513,6 +14532,25 @@
"supports_reasoning": true,
"supports_tool_choice": true
},
+ "databricks/databricks-claude-sonnet-4-6": {
+ "input_cost_per_token": 2.9999900000000002e-06,
+ "input_dbu_cost_per_token": 4.2857e-05,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 64000,
+ "max_tokens": 64000,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 1.5000020000000002e-05,
+ "output_dbu_cost_per_token": 0.000214286,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+ "supports_assistant_prefill": true,
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true
+ },
"databricks/databricks-gemini-2-5-flash": {
"input_cost_per_token": 3.0001999999999996e-07,
"input_dbu_cost_per_token": 4.285999999999999e-06,
@@ -14547,6 +14585,74 @@
"supports_function_calling": true,
"supports_tool_choice": true
},
+ "databricks/databricks-gemini-3-1-flash-lite": {
+ "input_cost_per_token": 3.1248e-07,
+ "input_dbu_cost_per_token": 4.464e-06,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 1.87502e-06,
+ "output_dbu_cost_per_token": 2.6786e-05,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+ "supports_function_calling": true,
+ "supports_tool_choice": true
+ },
+ "databricks/databricks-gemini-3-1-pro": {
+ "input_cost_per_token": 2.49998e-06,
+ "input_dbu_cost_per_token": 3.5714e-05,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 1.5000020000000002e-05,
+ "output_dbu_cost_per_token": 0.000214286,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+ "supports_function_calling": true,
+ "supports_tool_choice": true
+ },
+ "databricks/databricks-gemini-3-flash": {
+ "input_cost_per_token": 6.2503e-07,
+ "input_dbu_cost_per_token": 8.929e-06,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 3.74997e-06,
+ "output_dbu_cost_per_token": 5.3571e-05,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+ "supports_function_calling": true,
+ "supports_tool_choice": true
+ },
+ "databricks/databricks-gemini-3-pro": {
+ "input_cost_per_token": 2.49998e-06,
+ "input_dbu_cost_per_token": 3.5714e-05,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 1048576,
+ "max_output_tokens": 65536,
+ "max_tokens": 65536,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 1.5000020000000002e-05,
+ "output_dbu_cost_per_token": 0.000214286,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+ "supports_function_calling": true,
+ "supports_tool_choice": true
+ },
"databricks/databricks-gemma-3-12b": {
"input_cost_per_token": 1.5000999999999998e-07,
"input_dbu_cost_per_token": 2.1429999999999996e-06,
@@ -14592,6 +14698,126 @@
"output_dbu_cost_per_token": 0.000142857,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
},
+ "databricks/databricks-gpt-5-1-codex-max": {
+ "input_cost_per_token": 1.24999e-06,
+ "input_dbu_cost_per_token": 1.7857e-05,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 272000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 9.999990000000002e-06,
+ "output_dbu_cost_per_token": 0.000142857,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+ },
+ "databricks/databricks-gpt-5-1-codex-mini": {
+ "input_cost_per_token": 2.4997e-07,
+ "input_dbu_cost_per_token": 3.571e-06,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 272000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 1.99997e-06,
+ "output_dbu_cost_per_token": 2.8571e-05,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+ },
+ "databricks/databricks-gpt-5-2": {
+ "input_cost_per_token": 1.75e-06,
+ "input_dbu_cost_per_token": 2.5e-05,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 272000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 1.4e-05,
+ "output_dbu_cost_per_token": 0.0002,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+ },
+ "databricks/databricks-gpt-5-2-codex": {
+ "input_cost_per_token": 1.75e-06,
+ "input_dbu_cost_per_token": 2.5e-05,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 272000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 1.4e-05,
+ "output_dbu_cost_per_token": 0.0002,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+ },
+ "databricks/databricks-gpt-5-3-codex": {
+ "input_cost_per_token": 1.75e-06,
+ "input_dbu_cost_per_token": 2.5e-05,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 272000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 1.4e-05,
+ "output_dbu_cost_per_token": 0.0002,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+ },
+ "databricks/databricks-gpt-5-4": {
+ "input_cost_per_token": 2.49998e-06,
+ "input_dbu_cost_per_token": 3.5714e-05,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 272000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 1.5000020000000002e-05,
+ "output_dbu_cost_per_token": 0.000214286,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+ },
+ "databricks/databricks-gpt-5-4-mini": {
+ "input_cost_per_token": 7.4998e-07,
+ "input_dbu_cost_per_token": 1.0714e-05,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 272000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 4.50002e-06,
+ "output_dbu_cost_per_token": 6.4286e-05,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+ },
+ "databricks/databricks-gpt-5-4-nano": {
+ "input_cost_per_token": 1.9999e-07,
+ "input_dbu_cost_per_token": 2.857e-06,
+ "litellm_provider": "databricks",
+ "max_input_tokens": 272000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "metadata": {
+ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+ },
+ "mode": "chat",
+ "output_cost_per_token": 1.24999e-06,
+ "output_dbu_cost_per_token": 1.7857e-05,
+ "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+ },
"databricks/databricks-gpt-5-mini": {
"input_cost_per_token": 2.4997000000000006e-07,
"input_dbu_cost_per_token": 3.571e-06,
diff --git a/pyproject.toml b/pyproject.toml
index 275343ccef6..ffbc96eefb9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "litellm"
-version = "1.98.0"
+version = "1.99.0"
description = "Library to easily interface with LLM API providers"
readme = "README.md"
requires-python = ">=3.10, <3.15"
@@ -67,8 +67,8 @@ proxy = [
"azure-identity>=1.25.2,<2.0",
"azure-storage-blob>=12.28.0,<13.0",
"mcp>=1.28.1,<2.0",
- "litellm-proxy-extras==0.4.86",
- "litellm-enterprise==0.1.56",
+ "litellm-proxy-extras==0.4.87",
+ "litellm-enterprise==0.1.57",
"RestrictedPython>=8.1,<9.0",
"rich>=13.9.4,<14.0",
"InquirerPy>=0.3.4,<1.0",
@@ -306,7 +306,7 @@ members = ["enterprise", "litellm-proxy-extras"]
profile = "black"
[tool.commitizen]
-version = "1.98.0"
+version = "1.99.0"
version_files = [
"pyproject.toml:^version",
]
diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json
index 6882479a344..096e039b8aa 100644
--- a/ruff-strict-budget.json
+++ b/ruff-strict-budget.json
@@ -12,7 +12,7 @@
"limit": 2017
},
"ANN202": {
- "limit": 855
+ "limit": 854
},
"ANN204": {
"limit": 711
diff --git a/tests/batches_tests/test_batch_rate_limits.py b/tests/batches_tests/test_batch_rate_limits.py
index 0baff6c17be..bb41c1137da 100644
--- a/tests/batches_tests/test_batch_rate_limits.py
+++ b/tests/batches_tests/test_batch_rate_limits.py
@@ -27,6 +27,16 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import (
from litellm.proxy.utils import InternalUsageCache
+def _build_batch_limiter() -> _PROXY_BatchRateLimiter:
+ internal_usage_cache = InternalUsageCache(dual_cache=DualCache())
+ return _PROXY_BatchRateLimiter(
+ internal_usage_cache=internal_usage_cache,
+ parallel_request_limiter=_PROXY_MaxParallelRequestsHandler_v3(
+ internal_usage_cache=internal_usage_cache
+ ),
+ )
+
+
def get_expected_batch_file_usage(file_path: str) -> tuple[int, int]:
"""
Helper function to calculate expected request count and token count from a batch JSONL file.
@@ -69,10 +79,7 @@ async def test_batch_rate_limits():
"""
litellm._turn_on_debug()
CUSTOM_LLM_PROVIDER = "openai"
- BATCH_LIMITER = _PROXY_BatchRateLimiter(
- internal_usage_cache=None,
- parallel_request_limiter=None,
- )
+ BATCH_LIMITER = _build_batch_limiter()
file_name = "openai_batch_completions.jsonl"
_current_dir = os.path.dirname(os.path.abspath(__file__))
@@ -580,10 +587,7 @@ async def test_batch_rate_limiter_without_user_context(tmp_path):
CUSTOM_LLM_PROVIDER = "openai"
# Setup
- BATCH_LIMITER = _PROXY_BatchRateLimiter(
- internal_usage_cache=None,
- parallel_request_limiter=None,
- )
+ BATCH_LIMITER = _build_batch_limiter()
# Create a simple batch file
batch_content = """{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hello"}]}}"""
diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml
index ebbfd3415a5..b50551ec105 100644
--- a/tests/e2e/coverage_registry/reliability.yaml
+++ b/tests/e2e/coverage_registry/reliability.yaml
@@ -18,6 +18,16 @@
- {id: reliability.routing.usage_based.picks_under_tpm, module: reliability, tier: P0, behavior: routing, variant: usage_based, assertions: [picks_under_tpm], exercised_on: [chat_completions, messages], source: "router_strategy/lowest_tpm_rpm_v2.py", rationale: "Routes to lowest-TPM deployment; prevents over-allocation"}
- {id: reliability.routing.least_busy.picks_lowest_traffic, module: reliability, tier: P1, behavior: routing, variant: least_busy, assertions: [picks_lowest_traffic], exercised_on: [chat_completions, messages], source: "router_strategy/least_busy.py", rationale: "Fewest in-flight requests"}
- {id: reliability.routing.complexity_llm_classifier.routes_by_llm_tier, module: reliability, tier: P1, behavior: routing, variant: complexity_llm_classifier, assertions: [routes_by_llm_tier], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py", fail_before_fix: proven, rationale: "v2 auto-router LLM complexity classifier runs over the proxy and routes by semantic tier instead of silently crashing on absent litellm_metadata and falling back to heuristic scoring"}
+- {id: reliability.routing.tagged_marker.request_tag_selects_marker, module: reliability, tier: P0, behavior: routing, variant: tagged_marker, assertions: [request_tag_selects_marker], exercised_on: [chat_completions], source: "litellm/router.py:11445", rationale: "Tagged request selects the tagged strategy marker under a shared model_name instead of the plain deployment registered first (GitHub issue #36619)"}
+- {id: reliability.routing.tagged_marker.untagged_request_served_by_plain_deployment, module: reliability, tier: P0, behavior: routing, variant: tagged_marker, assertions: [untagged_request_served_by_plain_deployment], exercised_on: [chat_completions, messages, responses], source: "litellm/router.py:11445", rationale: "Untagged requests to a shared model_name are served by the plain deployment on every call, never captured or errored by the tagged marker (GitHub issue #36620)"}
+- {id: reliability.routing.tagged_marker.header_tag_selects_marker, module: reliability, tier: P0, behavior: routing, variant: tagged_marker, assertions: [header_tag_selects_marker], exercised_on: [messages], source: "litellm/router.py:11445", rationale: "A request tagged only via the x-litellm-tags header selects the tagged marker on Anthropic-native /v1/messages (GitHub issue #36621)"}
+- {id: reliability.routing.tagged_marker.untagged_tier_deployments_still_served, module: reliability, tier: P1, behavior: routing, variant: tagged_marker, assertions: [untagged_tier_deployments_still_served], exercised_on: [chat_completions, messages], source: "litellm/router_strategy/tag_based_routing.py:433", rationale: "Routing tags the marker consumed no longer constrain deployment selection inside the routed tier group, so untagged tier deployments serve the rewrite (GitHub issue #36621)"}
+- {id: reliability.routing.tagged_marker.tag_semantics_stay_strict, module: reliability, tier: P1, behavior: routing, variant: tagged_marker, assertions: [tag_semantics_stay_strict], exercised_on: [chat_completions], source: "litellm/router_strategy/tag_based_routing.py:299", rationale: "Tag consumption must not loosen strict semantics: a tagged call aimed straight at an untagged deployment still gets the 401 tags-configuration denial"}
+- {id: reliability.routing.tagged_marker.responses_input_routes_through_marker, module: reliability, tier: P0, behavior: routing, variant: tagged_marker, assertions: [responses_input_routes_through_marker], exercised_on: [responses], source: "litellm/router.py:11489", rationale: "Tagged /v1/responses (header or litellm_metadata.tags, string or list input) routes through the marker to its tier, extending the GitHub issues #36620/#36621 tag split to the Responses surface"}
+- {id: reliability.routing.tagged_marker.alias_connection_params_stay_with_tier, module: reliability, tier: P0, behavior: routing, variant: tagged_marker, assertions: [alias_connection_params_stay_with_tier], exercised_on: [chat_completions], source: "litellm/router.py:11567", rationale: "An api_key or api_base on the marker alias is never forwarded onto the routed request; the tier deployment calls its provider with its own credential (GitHub PR #36626)"}
+- {id: reliability.routing.semantic_auto_router.responses_input_routed, module: reliability, tier: P0, behavior: routing, variant: semantic_auto_router, assertions: [responses_input_routed], exercised_on: [responses], source: "litellm/router_strategy/auto_router/auto_router.py:131", fail_before_fix: proven, rationale: "/v1/responses input is resolved into messages for the semantic auto-router pre-routing hook instead of failing 400 Unmapped LLM provider auto_router (GitHub PR #37333)"}
+- {id: reliability.routing.strategy_alias.custom_pricing_ignored, module: reliability, tier: P1, behavior: routing, variant: strategy_alias, assertions: [custom_pricing_ignored], exercised_on: [chat_completions], source: "litellm/router.py:11489", rationale: "Custom pricing on a strategy-router alias never prices the routed request; spend logs at the routed tier deployment's own rate (GitHub PR #36691)"}
+- {id: reliability.routing.complexity_heuristic.scores_current_ask_only, module: reliability, tier: P1, behavior: routing, variant: complexity_heuristic, assertions: [scores_current_ask_only], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:942", rationale: "The heuristic complexity classifier scores the caller's current ask only, so a keyword-heavy agent system prompt cannot inflate the tier (GitHub PR #36721)"}
- {id: reliability.cache.exact.returns_cached, module: reliability, tier: P1, behavior: cache, variant: exact, assertions: [returns_cached], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/caching.py", rationale: "Response cache returns cached on exact match"}
- {id: reliability.cache.prompt_caching_model_select.returns_cached, module: reliability, tier: P1, behavior: cache, variant: prompt_caching_model_select, assertions: [returns_cached], exercised_on: [chat_completions], source: "router_utils/prompt_caching_cache.py", rationale: "Selects model supporting prompt caching for cacheable prefix"}
- {id: reliability.circuit_breaker.redis.trips_then_recovers, module: reliability, tier: P0, behavior: circuit_breaker, variant: redis, assertions: [trips_then_recovers], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/redis_cache.py:99", rationale: "Redis breaker CLOSED->OPEN->HALF_OPEN; guards all cache/rate-limit ops"}
diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py
index f4db88b1e19..cb6fc7a01e5 100644
--- a/tests/e2e/e2e_http.py
+++ b/tests/e2e/e2e_http.py
@@ -75,6 +75,8 @@ class NetworkError(BaseModel):
class UnauthorizedError(BaseModel):
kind: Literal["unauthorized"] = "unauthorized"
+ # litellm 401s for key auth, model access, and tag routing alike, so keep the body to tell them apart.
+ body: str = ""
class RateLimitedError(BaseModel):
@@ -289,7 +291,7 @@ def _classify[R: BaseModel](
resp: requests.Response, response_type: type[R]
) -> Result[R]:
if resp.status_code == 401:
- return UnauthorizedError()
+ return UnauthorizedError(body=resp.text)
if resp.status_code == 429:
return RateLimitedError(body=resp.text)
if not resp.ok:
diff --git a/tests/e2e/models.py b/tests/e2e/models.py
index 734e63a94e6..df5cb841fad 100644
--- a/tests/e2e/models.py
+++ b/tests/e2e/models.py
@@ -73,6 +73,7 @@ class KeyGenerateBody(BaseModel):
allowed_passthrough_routes: list[str] | None = None
metadata: KeyMetadata | None = None
object_permission: ObjectPermission | None = None
+ router_settings: "RouterSettingsOverride | None" = None
class KeyGenerateResponse(BaseModel):
@@ -234,16 +235,18 @@ class ChatBody(BaseModel):
class RouterSettingsOverride(BaseModel):
- """Per-request `router_settings_override` in a /chat/completions body: the
- reliability knobs (fallbacks by trigger, retry count) the reliability suite
- drives per call instead of via static router config. Serialized exclude_none, so
- an override sets only the strategies a test exercises. Each fallbacks map is
- model_name -> the ordered fallback model_names to try."""
+ """Router settings a test scopes below the global config: sent per request as
+ `router_settings_override` in a /chat/completions body (the reliability suite's
+ fallback and retry knobs) or stored on a key as `router_settings` at
+ /key/generate (the auto-router suite's tag filtering switch). Serialized
+ exclude_none, so an override sets only the knobs a test exercises. Each
+ fallbacks map is model_name -> the ordered fallback model_names to try."""
fallbacks: list[dict[str, list[str]]] | None = None
context_window_fallbacks: list[dict[str, list[str]]] | None = None
content_policy_fallbacks: list[dict[str, list[str]]] | None = None
num_retries: int | None = None
+ enable_tag_filtering: bool | None = None
class ReliabilityChatBody(ChatBody):
@@ -744,6 +747,10 @@ class LiteLLMParamsBody(BaseModel):
extra_headers: dict[str, str] | None = None
use_in_pass_through: bool | None = None
complexity_router_config: dict[str, object] | None = None
+ auto_router_config: str | None = None
+ auto_router_default_model: str | None = None
+ auto_router_embedding_model: str | None = None
+ tags: list[str] | None = None
mock_response: str | None = None
timeout: float | None = None
tpm: int | None = None
diff --git a/tests/e2e/router/test_auto_router_regressions_e2e.py b/tests/e2e/router/test_auto_router_regressions_e2e.py
new file mode 100644
index 00000000000..35ba2c8d3d1
--- /dev/null
+++ b/tests/e2e/router/test_auto_router_regressions_e2e.py
@@ -0,0 +1,620 @@
+"""Live e2e regression pins for strategy-router (auto-router) routing.
+
+A strategy marker (an ``auto_router/complexity_router`` deployment) and a plain
+deployment can share one ``model_name``, split by tags once
+``enable_tag_filtering`` is on: tagged requests route through the marker to its
+tier models, untagged requests go to the plain deployment. That split, and the
+strategy-router alias behaviors around it, regressed repeatedly; each test here
+pins one fixed behavior:
+
+- GitHub issue #36619: a tagged request selects the tagged marker under a
+ shared name even when a plain deployment was registered first.
+- GitHub issue #36620: untagged requests keep being served by the plain
+ deployment on every call, never captured or 400'd by the tagged marker.
+- GitHub issue #36621: a request tagged via the ``x-litellm-tags`` header
+ routes through the marker even when the tier deployments carry no tags
+ (the marker consumes the routing tags before deployment selection), while a
+ tagged call aimed straight at an untagged deployment stays denied.
+- GitHub issues #36620/#36621 on /v1/responses: the same tag split holds for
+ string and list input, whether the tag arrives in litellm_metadata or the
+ x-litellm-tags header.
+- GitHub PR #37333: /v1/responses input is resolved into messages for a
+ semantic ``auto_router`` deployment's pre-routing hook; such requests used
+ to fail with 400 "Unmapped LLM provider auto_router" because only chat
+ messages fed the route matcher.
+- GitHub PR #36691: custom pricing on the marker alias never prices the routed
+ request; spend logs at the routed tier deployment's own rate.
+- GitHub PR #36721: the heuristic complexity classifier scores the caller's
+ current ask only, so a large agent system prompt cannot inflate the tier.
+- GitHub PR #36626: connection params on the marker alias (``api_key``,
+ ``api_base``) stay with the alias; the routed tier calls its provider with
+ its own credentials.
+
+Every deployment is registered via /model/new (stage has no static config for
+these) and ``enable_tag_filtering`` is enabled through key-level
+``router_settings`` on the keys the tag tests mint, so the switch rides only
+this module's own requests and the rest of the suite is never filtered.
+The served deployment is always read back from the spend log's ``model``,
+which stores either the registered alias or the provider-prefixed form.
+"""
+
+import json
+import os
+from collections.abc import Iterator
+from dataclasses import dataclass
+from typing import Final
+
+import pytest
+from pydantic import BaseModel, ConfigDict, Field
+
+from e2e_config import unique_marker
+from e2e_http import AnthropicHeaders, AuthHeaders, UnauthorizedError, unwrap
+from lifecycle import ResourceManager
+from models import (
+ AnthropicMessagesBody,
+ AnthropicMessagesResponse,
+ ChatBody,
+ ChatMessage,
+ ChatMetadata,
+ KeyGenerateBody,
+ LiteLLMParamsBody,
+ RouterSettingsOverride,
+ SpendLogRow,
+)
+from proxy_client import ProxyClient
+
+pytestmark = pytest.mark.e2e
+
+PLAIN_MODEL = "anthropic/claude-sonnet-5"
+CHEAP_MODEL = "anthropic/claude-haiku-4-5"
+STRONG_MODEL = "openai/gpt-5.6"
+MAX_TOKENS = 16
+TAG_DENIAL_MESSAGE = "Not allowed to access model due to tags configuration"
+PLAIN_SERVED = frozenset({PLAIN_MODEL, "claude-sonnet-5"})
+CHEAP_SERVED = frozenset({CHEAP_MODEL, "claude-haiku-4-5"})
+EMBEDDING_MODEL = "openai/text-embedding-3-small"
+SEMANTIC_ROUTE_UTTERANCE = "summarize this quarterly revenue report into three bullet points"
+
+KEYWORD_HEAVY_SYSTEM_PROMPT = (
+ "You are the principal architecture assistant for a distributed systems platform. "
+ "Analyze every request step by step: design the algorithm, prove its correctness, "
+ "evaluate time and space complexity, and reason about concurrency, consistency, and "
+ "fault tolerance tradeoffs. When asked, refactor and debug multi-threaded code, "
+ "optimize database query plans, derive mathematical proofs, and explain the theorem "
+ "or lemma behind each optimization. Think through edge cases rigorously before answering. "
+) * 4
+
+
+class TaggedAuthHeaders(AuthHeaders):
+ x_litellm_tags: str | None = Field(default=None, serialization_alias="x-litellm-tags")
+
+
+class TaggedAnthropicHeaders(AnthropicHeaders):
+ x_litellm_tags: str | None = Field(default=None, serialization_alias="x-litellm-tags")
+
+
+class ResponsesTagMetadata(BaseModel):
+ tags: list[str]
+
+
+class ResponsesInputItem(BaseModel):
+ role: str
+ content: str
+
+
+class ResponsesBody(BaseModel):
+ model: str
+ input: str | list[ResponsesInputItem]
+ max_output_tokens: int | None = None
+ litellm_metadata: ResponsesTagMetadata | None = None
+
+
+class ResponsesApiResponse(BaseModel):
+ """Minimal /v1/responses answer shape; routing is proven from spend logs,
+ so only the fields the assertions read are modeled."""
+
+ model_config = ConfigDict(extra="allow")
+ id: str | None = None
+ status: str | None = None
+ model: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class TagSplitDeployments:
+ """Scenario A mirrors the customer-shaped config from GitHub issue #36619:
+ plain deployment registered first, tier deployment and marker both tagged.
+ Scenario B flips both axes for GitHub issue #36621: marker registered first
+ and its tier deployment left untagged, so routing depends neither on
+ registration order nor on tier deployments carrying tags."""
+
+ tag_a: str
+ shared_a: str
+ tier_a: str
+ tag_b: str
+ shared_b: str
+ tier_b: str
+
+
+@dataclass(frozen=True, slots=True)
+class ZeroPricedAlias:
+ alias: str
+ tier: str
+
+
+@dataclass(frozen=True, slots=True)
+class HeuristicSplit:
+ alias: str
+ cheap: str
+ strong: str
+
+
+@dataclass(frozen=True, slots=True)
+class SemanticAutoRouter:
+ marker: str
+ target: str
+ fallback: str
+ embedding: str
+
+
+@dataclass(frozen=True, slots=True)
+class CredentialedAlias:
+ alias: str
+ tier: str
+
+
+def _provider_key(env_var: str) -> str:
+ return os.environ.get(env_var) or f"os.environ/{env_var}"
+
+
+def _uniform_tier_config(tier_model: str) -> dict[str, object]:
+ return {
+ "classifier_type": "heuristic",
+ "tiers": {"SIMPLE": tier_model, "MEDIUM": tier_model, "COMPLEX": tier_model, "REASONING": tier_model},
+ }
+
+
+def _key_for(
+ proxy: ProxyClient, resources: ResourceManager, models: list[str], tag_filtering: bool = False
+) -> str:
+ key: Final = proxy.generate_key(
+ KeyGenerateBody(
+ models=models,
+ user_id="e2e-auto-router-regressions",
+ router_settings=RouterSettingsOverride(enable_tag_filtering=True) if tag_filtering else None,
+ )
+ )
+ resources.defer(lambda: proxy.delete_key(key))
+ return key
+
+
+def _hello_chat_body(model: str, tags: list[str] | None = None) -> ChatBody:
+ return ChatBody(
+ model=model,
+ messages=[ChatMessage(role="user", content=f"say hello {unique_marker()}")],
+ max_tokens=MAX_TOKENS,
+ metadata=ChatMetadata(tags=tags) if tags is not None else None,
+ )
+
+
+def _hello_messages_body(model: str) -> AnthropicMessagesBody:
+ return AnthropicMessagesBody(
+ model=model,
+ messages=[ChatMessage(role="user", content=f"say hello {unique_marker()}")],
+ max_tokens=MAX_TOKENS,
+ )
+
+
+def _assert_served_only_by(rows: list[SpendLogRow], allowed: frozenset[str], context: str) -> None:
+ served: Final = tuple(row.model for row in rows)
+ assert served and all(model in allowed for model in served), (
+ f"{context}: expected every request to be served by one of {sorted(allowed)}, spend logs show {served}"
+ )
+
+
+@pytest.fixture(scope="module")
+def split(proxy: ProxyClient) -> Iterator[TagSplitDeployments]:
+ marker: Final = unique_marker()
+ deployments: Final = TagSplitDeployments(
+ tag_a=f"e2e-split-a-{marker}",
+ shared_a=f"e2e-autoroute-a-{marker}",
+ tier_a=f"e2e-tier-a-{marker}",
+ tag_b=f"e2e-split-b-{marker}",
+ shared_b=f"e2e-autoroute-b-{marker}",
+ tier_b=f"e2e-tier-b-{marker}",
+ )
+ anthropic_key: Final = _provider_key("ANTHROPIC_API_KEY")
+ marker_params_a: Final = LiteLLMParamsBody(
+ model="auto_router/complexity_router",
+ complexity_router_config=_uniform_tier_config(deployments.tier_a),
+ tags=[deployments.tag_a],
+ )
+ marker_params_b: Final = LiteLLMParamsBody(
+ model="auto_router/complexity_router",
+ complexity_router_config=_uniform_tier_config(deployments.tier_b),
+ tags=[deployments.tag_b],
+ )
+ registrations: Final[tuple[tuple[str, LiteLLMParamsBody], ...]] = (
+ (deployments.shared_a, LiteLLMParamsBody(model=PLAIN_MODEL, api_key=anthropic_key)),
+ (deployments.tier_a, LiteLLMParamsBody(model=CHEAP_MODEL, api_key=anthropic_key, tags=[deployments.tag_a])),
+ (deployments.shared_a, marker_params_a),
+ (deployments.shared_b, marker_params_b),
+ (deployments.tier_b, LiteLLMParamsBody(model=CHEAP_MODEL, api_key=anthropic_key)),
+ (deployments.shared_b, LiteLLMParamsBody(model=PLAIN_MODEL, api_key=anthropic_key)),
+ )
+ created: Final = tuple(proxy.create_model(name, params) for name, params in registrations)
+ try:
+ yield deployments
+ finally:
+ for model_id in created:
+ proxy.delete_model(model_id)
+
+
+@pytest.fixture(scope="module")
+def zero_priced_alias(proxy: ProxyClient) -> Iterator[ZeroPricedAlias]:
+ marker: Final = unique_marker()
+ named: Final = ZeroPricedAlias(alias=f"e2e-priced-alias-{marker}", tier=f"e2e-priced-tier-{marker}")
+ alias_params: Final = LiteLLMParamsBody(
+ model="auto_router/complexity_router",
+ complexity_router_config=_uniform_tier_config(named.tier),
+ input_cost_per_token=0.0,
+ output_cost_per_token=0.0,
+ )
+ registrations: Final[tuple[tuple[str, LiteLLMParamsBody], ...]] = (
+ (named.tier, LiteLLMParamsBody(model=CHEAP_MODEL, api_key=_provider_key("ANTHROPIC_API_KEY"))),
+ (named.alias, alias_params),
+ )
+ created: Final = tuple(proxy.create_model(name, params) for name, params in registrations)
+ try:
+ yield named
+ finally:
+ for model_id in created:
+ proxy.delete_model(model_id)
+
+
+@pytest.fixture(scope="module")
+def heuristic_split(proxy: ProxyClient) -> Iterator[HeuristicSplit]:
+ marker: Final = unique_marker()
+ named: Final = HeuristicSplit(
+ alias=f"e2e-heuristic-router-{marker}",
+ cheap=f"e2e-heuristic-cheap-{marker}",
+ strong=f"e2e-heuristic-strong-{marker}",
+ )
+ config: Final[dict[str, object]] = {
+ "classifier_type": "heuristic",
+ "token_thresholds": {"simple": 15, "complex": 400},
+ "tiers": {"SIMPLE": named.cheap, "MEDIUM": named.strong, "COMPLEX": named.strong, "REASONING": named.strong},
+ }
+ registrations: Final[tuple[tuple[str, LiteLLMParamsBody], ...]] = (
+ (named.cheap, LiteLLMParamsBody(model=CHEAP_MODEL, api_key=_provider_key("ANTHROPIC_API_KEY"))),
+ (named.strong, LiteLLMParamsBody(model=STRONG_MODEL, api_key=_provider_key("OPENAI_API_KEY"))),
+ (named.alias, LiteLLMParamsBody(model="auto_router/complexity_router", complexity_router_config=config)),
+ )
+ created: Final = tuple(proxy.create_model(name, params) for name, params in registrations)
+ try:
+ yield named
+ finally:
+ for model_id in created:
+ proxy.delete_model(model_id)
+
+
+@pytest.fixture(scope="module")
+def semantic_auto_router(proxy: ProxyClient) -> Iterator[SemanticAutoRouter]:
+ marker: Final = unique_marker()
+ named: Final = SemanticAutoRouter(
+ marker=f"e2e-semantic-router-{marker}",
+ target=f"e2e-semantic-target-{marker}",
+ fallback=f"e2e-semantic-fallback-{marker}",
+ embedding=f"e2e-semantic-embedding-{marker}",
+ )
+ router_config: Final = json.dumps(
+ {"routes": [{"name": named.target, "utterances": [SEMANTIC_ROUTE_UTTERANCE], "score_threshold": 0.3}]}
+ )
+ marker_params: Final = LiteLLMParamsBody(
+ model=f"auto_router/{named.marker}",
+ auto_router_config=router_config,
+ auto_router_default_model=named.fallback,
+ auto_router_embedding_model=named.embedding,
+ )
+ registrations: Final[tuple[tuple[str, LiteLLMParamsBody], ...]] = (
+ (named.embedding, LiteLLMParamsBody(model=EMBEDDING_MODEL, api_key=_provider_key("OPENAI_API_KEY"))),
+ (named.target, LiteLLMParamsBody(model=CHEAP_MODEL, api_key=_provider_key("ANTHROPIC_API_KEY"))),
+ (named.fallback, LiteLLMParamsBody(model=PLAIN_MODEL, api_key=_provider_key("ANTHROPIC_API_KEY"))),
+ (named.marker, marker_params),
+ )
+ created: Final = tuple(proxy.create_model(name, params) for name, params in registrations)
+ try:
+ yield named
+ finally:
+ for model_id in created:
+ proxy.delete_model(model_id)
+
+
+@pytest.fixture(scope="module")
+def credentialed_alias(proxy: ProxyClient) -> Iterator[CredentialedAlias]:
+ marker: Final = unique_marker()
+ named: Final = CredentialedAlias(alias=f"e2e-cred-alias-{marker}", tier=f"e2e-cred-tier-{marker}")
+ alias_params: Final = LiteLLMParamsBody(
+ model="auto_router/complexity_router",
+ complexity_router_config=_uniform_tier_config(named.tier),
+ api_key=f"sk-alias-never-used-{marker}",
+ )
+ registrations: Final[tuple[tuple[str, LiteLLMParamsBody], ...]] = (
+ (named.tier, LiteLLMParamsBody(model=CHEAP_MODEL, api_key=_provider_key("ANTHROPIC_API_KEY"))),
+ (named.alias, alias_params),
+ )
+ created: Final = tuple(proxy.create_model(name, params) for name, params in registrations)
+ try:
+ yield named
+ finally:
+ for model_id in created:
+ proxy.delete_model(model_id)
+
+
+class TestTagSplitRouting:
+ @pytest.mark.covers("reliability.routing.tagged_marker.request_tag_selects_marker")
+ def test_body_tagged_chat_routes_through_the_marker_to_its_tier(
+ self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments
+ ) -> None:
+ """Pins GitHub issue #36619: with tag filtering on, a chat request whose
+ body metadata tags match the tagged marker under a shared model name is
+ answered by the marker's tier deployment, not by the plain deployment
+ that was registered under the name first."""
+ key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True)
+ chat: Final = unwrap(proxy.chat(key, _hello_chat_body(split.shared_a, tags=[split.tag_a])))
+ assert chat.choices, "tagged chat through the shared name returned no choices"
+ rows: Final = proxy.poll_logs_for_key(key, min_rows=1)
+ _assert_served_only_by(rows, CHEAP_SERVED | {split.tier_a}, "body-tagged chat on the shared name")
+
+ @pytest.mark.covers("reliability.routing.tagged_marker.untagged_request_served_by_plain_deployment")
+ def test_untagged_chat_is_always_served_by_the_plain_deployment(
+ self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments
+ ) -> None:
+ """Pins GitHub issue #36620: untagged chat requests to the shared name
+ succeed on every call and are all served by the plain deployment; the
+ tagged marker never captures them, so no intermittent auto-router
+ errors and no tier hijacking."""
+ key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True)
+ for _ in range(5):
+ chat = unwrap(proxy.chat(key, _hello_chat_body(split.shared_a)))
+ assert chat.choices, "untagged chat through the shared name returned no choices"
+ rows: Final = proxy.poll_logs_for_key(key, min_rows=5)
+ _assert_served_only_by(rows, PLAIN_SERVED | {split.shared_a}, "untagged chat on the shared name")
+
+ @pytest.mark.covers("reliability.routing.tagged_marker.untagged_request_served_by_plain_deployment")
+ def test_untagged_messages_is_served_by_the_plain_deployment(
+ self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments
+ ) -> None:
+ """Pins GitHub issue #36620 on the /v1/messages surface: an untagged
+ Anthropic-native request to the shared name is served by the plain
+ deployment, not captured by the tagged marker."""
+ key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True)
+ answer: Final = unwrap(proxy.messages(key, _hello_messages_body(split.shared_a)))
+ assert answer.content or answer.choices, "untagged /v1/messages returned neither content nor choices"
+ rows: Final = proxy.poll_logs_for_key(key, min_rows=1)
+ _assert_served_only_by(rows, PLAIN_SERVED | {split.shared_a}, "untagged /v1/messages on the shared name")
+
+
+class TestUntaggedTierDeployments:
+ @pytest.mark.covers("reliability.routing.tagged_marker.header_tag_selects_marker")
+ def test_header_tagged_messages_routes_through_the_marker_to_an_untagged_tier(
+ self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments
+ ) -> None:
+ """Pins GitHub issue #36621: a /v1/messages request tagged only via the
+ x-litellm-tags header selects the tagged marker, and the rewrite still
+ lands on the tier deployment even though that deployment carries no
+ tags, because the marker consumed the routing tags."""
+ key: Final = _key_for(proxy, resources, [split.shared_b, split.tier_b], tag_filtering=True)
+ headers: Final = TaggedAnthropicHeaders(authorization=f"Bearer {key}", x_litellm_tags=split.tag_b)
+ answer: Final = unwrap(
+ proxy.transport.post(
+ "/v1/messages",
+ headers=headers,
+ json=_hello_messages_body(split.shared_b),
+ response_type=AnthropicMessagesResponse,
+ )
+ )
+ assert answer.content or answer.choices, "header-tagged /v1/messages returned neither content nor choices"
+ rows: Final = proxy.poll_logs_for_key(key, min_rows=1)
+ _assert_served_only_by(rows, CHEAP_SERVED | {split.tier_b}, "header-tagged /v1/messages on the shared name")
+
+ @pytest.mark.covers("reliability.routing.tagged_marker.untagged_tier_deployments_still_served")
+ def test_body_tagged_chat_reaches_the_untagged_tier_after_marker_rewrite(
+ self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments
+ ) -> None:
+ """Pins the tag-consumption half of GitHub issue #36621: after the
+ tagged marker rewrites the request to its tier model, the consumed
+ routing tags no longer constrain deployment selection, so the untagged
+ tier deployment serves the request instead of a strict-tag denial."""
+ key: Final = _key_for(proxy, resources, [split.shared_b, split.tier_b], tag_filtering=True)
+ chat: Final = unwrap(proxy.chat(key, _hello_chat_body(split.shared_b, tags=[split.tag_b])))
+ assert chat.choices, "body-tagged chat through the marker-first shared name returned no choices"
+ rows: Final = proxy.poll_logs_for_key(key, min_rows=1)
+ _assert_served_only_by(rows, CHEAP_SERVED | {split.tier_b}, "body-tagged chat with untagged tier")
+
+ @pytest.mark.covers("reliability.routing.tagged_marker.tag_semantics_stay_strict")
+ def test_tagged_call_straight_at_an_untagged_deployment_stays_denied(
+ self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments
+ ) -> None:
+ """The tag-consumption fix must not loosen strict tag semantics: a
+ tagged request aimed directly at an untagged deployment (no marker
+ involved) is still rejected with the 401 tags-configuration error."""
+ key: Final = _key_for(proxy, resources, [split.tier_b], tag_filtering=True)
+ result: Final = proxy.chat(key, _hello_chat_body(split.tier_b, tags=[split.tag_b]))
+ assert isinstance(result, UnauthorizedError), (
+ f"expected the tagged direct call to an untagged deployment to be denied with 401, got {result}"
+ )
+ assert TAG_DENIAL_MESSAGE in result.body, (
+ f"expected the denial to come from tag routing, got a 401 reading {result.body[:300]}"
+ )
+
+
+class TestResponsesApiTagRouting:
+ @pytest.mark.covers("reliability.routing.tagged_marker.responses_input_routes_through_marker")
+ def test_header_tagged_responses_with_string_input_routes_to_the_tier(
+ self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments
+ ) -> None:
+ """Pins the /v1/responses surface of the tag split (GitHub issues
+ #36620/#36621): a /v1/responses request with string input, tagged via
+ the x-litellm-tags header, succeeds and routes through the tagged
+ marker to its tier."""
+ key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True)
+ headers: Final = TaggedAuthHeaders(authorization=f"Bearer {key}", x_litellm_tags=split.tag_a)
+ body: Final = ResponsesBody(
+ model=split.shared_a, input=f"say hello {unique_marker()}", max_output_tokens=64
+ )
+ answer: Final = unwrap(
+ proxy.transport.post("/v1/responses", headers=headers, json=body, response_type=ResponsesApiResponse)
+ )
+ assert answer.id, "header-tagged /v1/responses returned no response id"
+ rows: Final = proxy.poll_logs_for_key(key, min_rows=1)
+ _assert_served_only_by(rows, CHEAP_SERVED | {split.tier_a}, "header-tagged /v1/responses string input")
+
+ @pytest.mark.covers("reliability.routing.tagged_marker.responses_input_routes_through_marker")
+ def test_body_tagged_responses_with_list_input_routes_to_the_tier(
+ self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments
+ ) -> None:
+ """Pins the body-tag and list-input combination of the same split:
+ /v1/responses with litellm_metadata.tags and structured input items
+ routes through the tagged marker to its tier."""
+ key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True)
+ body: Final = ResponsesBody(
+ model=split.shared_a,
+ input=[ResponsesInputItem(role="user", content=f"say hello {unique_marker()}")],
+ max_output_tokens=64,
+ litellm_metadata=ResponsesTagMetadata(tags=[split.tag_a]),
+ )
+ answer: Final = unwrap(
+ proxy.transport.post(
+ "/v1/responses",
+ headers=proxy.transport.bearer(key),
+ json=body,
+ response_type=ResponsesApiResponse,
+ )
+ )
+ assert answer.id, "body-tagged /v1/responses returned no response id"
+ rows: Final = proxy.poll_logs_for_key(key, min_rows=1)
+ _assert_served_only_by(rows, CHEAP_SERVED | {split.tier_a}, "body-tagged /v1/responses list input")
+
+ @pytest.mark.covers("reliability.routing.tagged_marker.untagged_request_served_by_plain_deployment")
+ def test_untagged_responses_is_served_by_the_plain_deployment(
+ self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments
+ ) -> None:
+ """Pins the untagged half of the /v1/responses tag split: an untagged
+ request to the shared name is served by the plain deployment, matching
+ the chat and messages surfaces."""
+ key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True)
+ body: Final = ResponsesBody(
+ model=split.shared_a, input=f"say hello {unique_marker()}", max_output_tokens=64
+ )
+ answer: Final = unwrap(
+ proxy.transport.post(
+ "/v1/responses",
+ headers=proxy.transport.bearer(key),
+ json=body,
+ response_type=ResponsesApiResponse,
+ )
+ )
+ assert answer.id, "untagged /v1/responses returned no response id"
+ rows: Final = proxy.poll_logs_for_key(key, min_rows=1)
+ _assert_served_only_by(rows, PLAIN_SERVED | {split.shared_a}, "untagged /v1/responses on the shared name")
+
+
+class TestStrategyAliasPricing:
+ @pytest.mark.covers("reliability.routing.strategy_alias.custom_pricing_ignored")
+ def test_zero_priced_alias_still_logs_spend_at_the_tier_rate(
+ self, proxy: ProxyClient, resources: ResourceManager, zero_priced_alias: ZeroPricedAlias
+ ) -> None:
+ """Pins GitHub PR #36691: custom pricing registered on a strategy-router
+ alias never prices the routed request. The alias here carries explicit
+ zero pricing, so any zero-spend row would prove the alias pricing was
+ applied; the routed tier deployment's real rate must produce spend > 0."""
+ key: Final = _key_for(proxy, resources, [zero_priced_alias.alias, zero_priced_alias.tier])
+ chat: Final = unwrap(proxy.chat(key, _hello_chat_body(zero_priced_alias.alias)))
+ assert chat.choices, "chat through the zero-priced alias returned no choices"
+ rows: Final = proxy.poll_logs_for_key(
+ key, min_rows=1, predicate=lambda logged: all((row.spend or 0.0) > 0.0 for row in logged)
+ )
+ _assert_served_only_by(rows, CHEAP_SERVED | {zero_priced_alias.tier}, "chat through the zero-priced alias")
+ priced: Final = tuple((row.model, row.spend) for row in rows)
+ assert all((row.spend or 0.0) > 0.0 for row in rows), (
+ f"expected spend at the tier deployment's own rate, got zero-spend rows: {priced}"
+ )
+
+
+class TestComplexityHeuristicScope:
+ @pytest.mark.covers("reliability.routing.complexity_heuristic.scores_current_ask_only")
+ def test_trivial_ask_behind_keyword_heavy_system_prompt_stays_on_the_cheap_tier(
+ self, proxy: ProxyClient, resources: ResourceManager, heuristic_split: HeuristicSplit
+ ) -> None:
+ """Pins GitHub PR #36721: the heuristic complexity classifier scores the
+ caller's current ask alone. The trivial ask scores SIMPLE on its own,
+ while the accompanying ~2KB agent system prompt is packed with enough
+ reasoning and complexity keywords that scoring the combined text lands
+ in REASONING; only ask-only scoring keeps this on the cheap tier."""
+ key: Final = _key_for(
+ proxy, resources, [heuristic_split.alias, heuristic_split.cheap, heuristic_split.strong]
+ )
+ body: Final = ChatBody(
+ model=heuristic_split.alias,
+ messages=[
+ ChatMessage(role="system", content=KEYWORD_HEAVY_SYSTEM_PROMPT),
+ ChatMessage(role="user", content=f"hi {unique_marker()}"),
+ ],
+ max_tokens=MAX_TOKENS,
+ )
+ chat: Final = unwrap(proxy.chat(key, body))
+ assert chat.choices, "chat through the heuristic router returned no choices"
+ rows: Final = proxy.poll_logs_for_key(key, min_rows=1)
+ _assert_served_only_by(
+ rows, CHEAP_SERVED | {heuristic_split.cheap}, "trivial ask behind a keyword-heavy system prompt"
+ )
+
+
+class TestSemanticAutoRouterResponses:
+ @pytest.mark.covers("reliability.routing.semantic_auto_router.responses_input_routed")
+ def test_responses_input_reaches_the_semantic_auto_router(
+ self, proxy: ProxyClient, resources: ResourceManager, semantic_auto_router: SemanticAutoRouter
+ ) -> None:
+ """Pins GitHub PR #37333: /v1/responses input is resolved into messages
+ for the semantic auto-router's pre-routing hook, so the marker embeds
+ the input, matches its route, and the target deployment serves the
+ request; before the fix the hook saw no messages and the request
+ failed with 400 "Unmapped LLM provider auto_router"."""
+ key: Final = _key_for(
+ proxy,
+ resources,
+ [semantic_auto_router.marker, semantic_auto_router.target, semantic_auto_router.fallback],
+ )
+ body: Final = ResponsesBody(
+ model=semantic_auto_router.marker, input=SEMANTIC_ROUTE_UTTERANCE, max_output_tokens=64
+ )
+ answer: Final = unwrap(
+ proxy.transport.post(
+ "/v1/responses",
+ headers=proxy.transport.bearer(key),
+ json=body,
+ response_type=ResponsesApiResponse,
+ )
+ )
+ assert answer.id, "/v1/responses through the semantic auto-router returned no response id"
+ rows: Final = proxy.poll_logs_for_key(key, min_rows=1)
+ _assert_served_only_by(
+ rows, CHEAP_SERVED | {semantic_auto_router.target}, "semantic auto-router /v1/responses string input"
+ )
+
+
+class TestAliasParamForwarding:
+ @pytest.mark.covers("reliability.routing.tagged_marker.alias_connection_params_stay_with_tier")
+ def test_alias_api_key_never_overrides_the_tier_credential(
+ self, proxy: ProxyClient, resources: ResourceManager, credentialed_alias: CredentialedAlias
+ ) -> None:
+ """Pins GitHub PR #36626: an api_key set on the marker alias entry is
+ never forwarded onto the routed request, so the tier deployment calls
+ its provider with its own credential. Before the fix the alias's key
+ was copied into the request, overriding the tier's credential, and
+ every routed call failed provider auth."""
+ key: Final = _key_for(proxy, resources, [credentialed_alias.alias, credentialed_alias.tier])
+ chat: Final = unwrap(proxy.chat(key, _hello_chat_body(credentialed_alias.alias)))
+ assert chat.choices, "chat through the credentialed alias returned no choices"
+ rows: Final = proxy.poll_logs_for_key(key, min_rows=1)
+ _assert_served_only_by(rows, CHEAP_SERVED | {credentialed_alias.tier}, "chat through the credentialed alias")
diff --git a/tests/e2e/ui/tests/mcp/mcpTools.spec.ts b/tests/e2e/ui/tests/mcp/mcpTools.spec.ts
index edaeab196aa..225ca8b9449 100644
--- a/tests/e2e/ui/tests/mcp/mcpTools.spec.ts
+++ b/tests/e2e/ui/tests/mcp/mcpTools.spec.ts
@@ -63,7 +63,7 @@ test.describe("MCP Tools", () => {
// The form is generated from the tool's inputSchema, so `repoName` proves the schema
// round-tripped through the proxy instead of the panel falling back to a generic field.
- const repoInput = page.locator('input[id="repoName"]');
+ const repoInput = page.getByLabel(/repoName/);
await expect(repoInput).toBeVisible();
await repoInput.fill(TOOL_ARG_REPO);
diff --git a/tests/e2e/ui/tests/modelsPage/addModel.spec.ts b/tests/e2e/ui/tests/modelsPage/addModel.spec.ts
index 1b11ea69f97..bd5373e1569 100644
--- a/tests/e2e/ui/tests/modelsPage/addModel.spec.ts
+++ b/tests/e2e/ui/tests/modelsPage/addModel.spec.ts
@@ -338,8 +338,8 @@ test.describe("Add Model", () => {
await page.getByRole("button", { name: "Add Model" }).last().click();
- // Scope to antd's notification container so a stale toast can't satisfy this.
- await expect(page.locator(".ant-notification").getByText("created successfully").last()).toBeVisible({
+ // Scope to the toast container so a stale toast can't satisfy this.
+ await expect(page.locator("[data-sonner-toast]").getByText("created successfully").last()).toBeVisible({
timeout: 15_000,
});
diff --git a/tests/e2e/ui/tests/modelsPage/clearCustomPricing.spec.ts b/tests/e2e/ui/tests/modelsPage/clearCustomPricing.spec.ts
index e67dcb96f36..c532641b238 100644
--- a/tests/e2e/ui/tests/modelsPage/clearCustomPricing.spec.ts
+++ b/tests/e2e/ui/tests/modelsPage/clearCustomPricing.spec.ts
@@ -85,9 +85,9 @@ test.describe("Clear custom pricing on a deployment", () => {
const inputCost = page.getByPlaceholder("Enter input cost");
const outputCost = page.getByPlaceholder("Enter output cost");
// Both cache fields share the same placeholder ("Defaults to Input Cost if blank"),
- // so disambiguate via the Form.Item id (AntD assigns the `name` prop as input id).
- const cacheReadCost = page.locator("#cache_read_cost");
- const cacheWriteCost = page.locator("#cache_write_cost");
+ // so disambiguate via their labels.
+ const cacheReadCost = page.getByLabel(/Cache Read Cost/);
+ const cacheWriteCost = page.getByLabel(/Cache Write Cost/);
await inputCost.waitFor({ timeout: 15_000 });
for (const field of [inputCost, outputCost, cacheReadCost, cacheWriteCost]) {
await field.click({ clickCount: 3 });
diff --git a/tests/e2e/ui/tests/proxy-admin/keys.spec.ts b/tests/e2e/ui/tests/proxy-admin/keys.spec.ts
index d9b0f959c9f..004fedb3263 100644
--- a/tests/e2e/ui/tests/proxy-admin/keys.spec.ts
+++ b/tests/e2e/ui/tests/proxy-admin/keys.spec.ts
@@ -36,9 +36,8 @@ test.describe("Proxy Admin - Keys", () => {
// Wait for the key creation modal
await expect(page.getByText("Key Ownership")).toBeVisible({ timeout: 10_000 });
- // Fill key name (has data-testid="base-input" in the built UI)
const keyName = `e2e-admin-key-${Date.now()}`;
- await page.getByTestId("base-input").fill(keyName);
+ await page.getByLabel(/Key Name/).fill(keyName);
// Select team
const teamSelect = page.getByTestId("team-dropdown").getByRole("combobox");
@@ -192,7 +191,7 @@ test.describe("Proxy Admin - Keys", () => {
await expect(page.getByText("Key Ownership")).toBeVisible({ timeout: 10_000 });
const keyName = `e2e-admin-allproxy-${Date.now()}`;
- await page.getByTestId("base-input").fill(keyName);
+ await page.getByLabel(/Key Name/).fill(keyName);
// No team selection — leave team dropdown empty so the key is owned by the admin user
@@ -220,7 +219,7 @@ test.describe("Proxy Admin - Keys", () => {
await expect(page.getByText("Key Ownership")).toBeVisible({ timeout: 10_000 });
const keyName = `e2e-admin-specific-${Date.now()}`;
- await page.getByTestId("base-input").fill(keyName);
+ await page.getByLabel(/Key Name/).fill(keyName);
// Open the model multi-select and pick a single specific model. Use
// getByRole("option", ...) to avoid the strict-mode collision between
diff --git a/tests/e2e/ui/tests/proxy-admin/teams.spec.ts b/tests/e2e/ui/tests/proxy-admin/teams.spec.ts
index d7c8eb6237e..172fde173df 100644
--- a/tests/e2e/ui/tests/proxy-admin/teams.spec.ts
+++ b/tests/e2e/ui/tests/proxy-admin/teams.spec.ts
@@ -79,7 +79,7 @@ test.describe("Proxy Admin - Teams", () => {
await expect(modal).toBeVisible({ timeout: 5_000 });
// The email field is a Select — type to search, then select from dropdown
- await modal.locator(".ant-select").first().click();
+ await modal.getByRole("combobox").first().click();
await page.keyboard.type("invitable@test.local");
// Wait for the option to appear, then select via keyboard (avoids viewport issues)
diff --git a/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts b/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts
index d71d5e6c0fe..0ef74e71529 100644
--- a/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts
+++ b/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts
@@ -66,7 +66,7 @@ test.describe("Team Admin", () => {
// Use a dedicated invitee user so this doesn't race with the proxy-admin
// "Invite a user" test that adds invitable@test.local to the same team.
- await modal.locator(".ant-select").first().click();
+ await modal.getByRole("combobox").first().click();
await page.keyboard.type("invitable-team@test.local");
const emailOption = page.getByRole("option", { name: "invitable-team@test.local" }).first();
@@ -136,7 +136,7 @@ test.describe("Team Admin", () => {
await expect(page.getByText("Key Ownership")).toBeVisible({ timeout: 10_000 });
const keyName = `e2e-team-admin-key-${Date.now()}`;
- await page.getByTestId("base-input").fill(keyName);
+ await page.getByLabel(/Key Name/).fill(keyName);
// Team selector — same locator pattern as the proxy-admin keys test.
const teamSelect = page.getByTestId("team-dropdown").getByRole("combobox");
diff --git a/tests/image_gen_tests/test_image_generation.py b/tests/image_gen_tests/test_image_generation.py
index f0a73325afa..33dcdbb57a5 100644
--- a/tests/image_gen_tests/test_image_generation.py
+++ b/tests/image_gen_tests/test_image_generation.py
@@ -269,7 +269,7 @@ class TestAimlImageGeneration(BaseImageGenTest):
class TestGoogleImageGen(BaseImageGenTest):
def get_base_image_generation_call_args(self) -> dict:
- return {"model": "gemini/imagen-4.0-generate-001"}
+ return {"model": "gemini/gemini-3.1-flash-image"}
@pytest.mark.skip(reason="Runwayml image generation API only tested locally")
diff --git a/tests/llm_translation/test_groq.py b/tests/llm_translation/test_groq.py
index cf4be9e801e..c720f818eaf 100644
--- a/tests/llm_translation/test_groq.py
+++ b/tests/llm_translation/test_groq.py
@@ -20,7 +20,7 @@ from litellm.llms.groq.chat.transformation import (
class TestGroq(BaseLLMChatTest):
def get_base_completion_call_args(self) -> dict:
return {
- "model": "groq/llama-3.3-70b-versatile",
+ "model": "groq/openai/gpt-oss-120b",
}
def test_tool_call_no_arguments(self, tool_call_no_arguments):
diff --git a/tests/local_testing/test_model_alias_map.py b/tests/local_testing/test_model_alias_map.py
index 14c1de2f6a7..9ef0448e7c6 100644
--- a/tests/local_testing/test_model_alias_map.py
+++ b/tests/local_testing/test_model_alias_map.py
@@ -15,7 +15,7 @@ from litellm import completion, embedding
litellm.set_verbose = True
-model_alias_map = {"good-model": "groq/llama-3.1-8b-instant"}
+model_alias_map = {"good-model": "groq/openai/gpt-oss-120b"}
def test_model_alias_map(caplog):
@@ -34,7 +34,7 @@ def test_model_alias_map(caplog):
if rec.levelname == "ERROR" and rec.name.startswith("LiteLLM"):
pytest.fail(f"Unexpected litellm ERROR log: {rec.getMessage()}")
- assert "llama-3.1-8b-instant" in response.model
+ assert "gpt-oss-120b" in response.model
except litellm.ServiceUnavailableError:
pass
except Exception as e:
diff --git a/tests/local_testing/test_router.py b/tests/local_testing/test_router.py
index af1a052e86d..7bc29517f8c 100644
--- a/tests/local_testing/test_router.py
+++ b/tests/local_testing/test_router.py
@@ -120,7 +120,7 @@ async def test_router_provider_wildcard_routing():
print("response 2 = ", response2)
response3 = await router.acompletion(
- model="groq/llama-3.1-8b-instant",
+ model="groq/openai/gpt-oss-120b",
messages=[{"role": "user", "content": "hello"}],
)
diff --git a/tests/local_testing/test_router_batch_completion.py b/tests/local_testing/test_router_batch_completion.py
index f7a1b41ca29..bb9e1851c61 100644
--- a/tests/local_testing/test_router_batch_completion.py
+++ b/tests/local_testing/test_router_batch_completion.py
@@ -44,7 +44,7 @@ async def test_batch_completion_multiple_models(mode):
{
"model_name": "groq-llama",
"litellm_params": {
- "model": "groq/llama-3.1-8b-instant",
+ "model": "groq/openai/gpt-oss-120b",
},
},
]
@@ -143,7 +143,7 @@ async def test_batch_completion_fastest_response_streaming():
{
"model_name": "groq-llama",
"litellm_params": {
- "model": "groq/llama-3.1-8b-instant",
+ "model": "groq/openai/gpt-oss-120b",
},
},
]
@@ -179,7 +179,7 @@ async def test_batch_completion_multiple_models_multiple_messages():
{
"model_name": "groq-llama",
"litellm_params": {
- "model": "groq/llama-3.1-8b-instant",
+ "model": "groq/openai/gpt-oss-120b",
},
},
]
diff --git a/tests/local_testing/test_stream_chunk_builder.py b/tests/local_testing/test_stream_chunk_builder.py
index 38e04b93f18..664fd936205 100644
--- a/tests/local_testing/test_stream_chunk_builder.py
+++ b/tests/local_testing/test_stream_chunk_builder.py
@@ -871,7 +871,7 @@ def load_env():
}
LLAMA3_3 = {
"messages": messages,
- "model": "groq/llama-3.3-70b-versatile",
+ "model": "groq/openai/gpt-oss-120b",
"api_base": "https://api.groq.com/openai/v1",
"temperature": 0.0,
"tools": tools,
diff --git a/tests/logging_callback_tests/test_logging_redaction_e2e_test.py b/tests/logging_callback_tests/test_logging_redaction_e2e_test.py
index d56d5e51b04..3b42595b959 100644
--- a/tests/logging_callback_tests/test_logging_redaction_e2e_test.py
+++ b/tests/logging_callback_tests/test_logging_redaction_e2e_test.py
@@ -317,110 +317,69 @@ async def test_redaction_responses_api_stream():
@pytest.mark.asyncio
async def test_redaction_responses_api_with_reasoning_summary():
"""Test that reasoning summary in ResponsesAPIResponse output is properly redacted"""
+ import litellm
from litellm.litellm_core_utils.redact_messages import perform_redaction
- # Create a simple mock object with output items that have reasoning summaries
- class MockResponsesAPIResponse:
- def __init__(self):
- self.output = [
- # Reasoning item with summary
- type(
- "obj",
- (object,),
+ response = litellm.ResponsesAPIResponse(
+ id="resp_123",
+ created_at=1234567890,
+ output=[
+ {
+ "type": "reasoning",
+ "id": "rs_123",
+ "summary": [
{
- "type": "reasoning",
- "id": "rs_123",
- "summary": [
- type(
- "obj",
- (object,),
- {
- "text": "This is a detailed reasoning summary that should be redacted",
- "type": "summary_text",
- },
- )()
- ],
- },
- )(),
- # Message item with content
- type(
- "obj",
- (object,),
+ "type": "summary_text",
+ "text": "This is a detailed reasoning summary that should be redacted",
+ }
+ ],
+ },
+ {
+ "type": "message",
+ "id": "msg_123",
+ "status": "completed",
+ "role": "assistant",
+ "content": [
{
- "type": "message",
- "id": "msg_123",
- "content": [
- type(
- "obj",
- (object,),
- {
- "text": "This is the actual message content",
- "type": "output_text",
- },
- )()
- ],
- },
- )(),
- ]
- self.reasoning = {"effort": "low", "summary": "auto"}
+ "type": "output_text",
+ "text": "This is the actual message content",
+ "annotations": [],
+ }
+ ],
+ },
+ ],
+ reasoning={"effort": "low", "summary": "auto"},
+ )
- # Mock as ResponsesAPIResponse so perform_redaction recognizes it
- mock_response = MockResponsesAPIResponse()
- mock_response.__class__.__name__ = "ResponsesAPIResponse"
+ model_call_details = {
+ "messages": [{"role": "user", "content": "test"}],
+ "prompt": "test prompt",
+ "input": "test input",
+ }
- # Patch isinstance to recognize our mock as ResponsesAPIResponse
- import litellm
+ redacted_result = perform_redaction(model_call_details, response)
- original_isinstance = isinstance
+ assert isinstance(
+ redacted_result, litellm.ResponsesAPIResponse
+ ), "Redaction should preserve the ResponsesAPIResponse type"
- def patched_isinstance(obj, cls):
- if (
- cls == litellm.ResponsesAPIResponse
- and obj.__class__.__name__ == "ResponsesAPIResponse"
- ):
- return True
- return original_isinstance(obj, cls)
+ reasoning_item = redacted_result.output[0]
+ assert (
+ reasoning_item.summary[0].text == "redacted-by-litellm"
+ ), "Reasoning summary text should be redacted"
- import builtins
+ message_item = redacted_result.output[1]
+ assert (
+ message_item.content[0].text == "redacted-by-litellm"
+ ), "Message content text should be redacted"
- builtins.isinstance = patched_isinstance
+ assert (
+ redacted_result.reasoning is None
+ ), "Top-level reasoning field should be None"
- try:
- model_call_details = {
- "messages": [{"role": "user", "content": "test"}],
- "prompt": "test prompt",
- "input": "test input",
- }
-
- # Perform redaction
- redacted_result = perform_redaction(model_call_details, mock_response)
-
- # Verify reasoning summary text is redacted
- reasoning_item = redacted_result.output[0]
- assert (
- reasoning_item.summary[0].text == "redacted-by-litellm"
- ), "Reasoning summary text should be redacted"
-
- # Verify message content is also redacted
- message_item = redacted_result.output[1]
- assert (
- message_item.content[0].text == "redacted-by-litellm"
- ), "Message content text should be redacted"
-
- # Verify top-level reasoning field is removed
- assert (
- redacted_result.reasoning is None
- ), "Top-level reasoning field should be None"
-
- # Verify input messages are redacted
- assert (
- model_call_details["messages"][0]["content"] == "redacted-by-litellm"
- ), "Input messages should be redacted"
-
- print("✓ Reasoning summary redaction test passed")
- finally:
- # Restore original isinstance
- builtins.isinstance = original_isinstance
+ assert (
+ model_call_details["messages"][0]["content"] == "redacted-by-litellm"
+ ), "Input messages should be redacted"
@pytest.mark.asyncio
diff --git a/tests/ocr_tests/test_ocr_azure_document_intelligence.py b/tests/ocr_tests/test_ocr_azure_document_intelligence.py
index 09c21842ad7..5736bd797e3 100644
--- a/tests/ocr_tests/test_ocr_azure_document_intelligence.py
+++ b/tests/ocr_tests/test_ocr_azure_document_intelligence.py
@@ -62,7 +62,7 @@ class TestAzureDocumentIntelligencePagesParam:
return AzureDocumentIntelligenceOCRConfig()
def test_get_supported_ocr_params_includes_pages_and_features(self, cfg):
- assert cfg.get_supported_ocr_params("prebuilt-layout") == ["pages", "features"]
+ assert cfg.get_supported_ocr_params("prebuilt-layout") == ["pages", "features", "req_format"]
def test_map_ocr_params_mistral_zero_based_int_list(self, cfg):
mapped = cfg.map_ocr_params({"pages": [0, 1, 2]}, {}, "prebuilt-layout")
diff --git a/tests/pass_through_unit_tests/test_pass_through_unit_tests.py b/tests/pass_through_unit_tests/test_pass_through_unit_tests.py
index c263b8ce381..77fb924c085 100644
--- a/tests/pass_through_unit_tests/test_pass_through_unit_tests.py
+++ b/tests/pass_through_unit_tests/test_pass_through_unit_tests.py
@@ -413,35 +413,54 @@ async def test_pass_through_request_logging_failure_with_stream(
assert response.body == b'{"mock": "response"}'
+PROTOCOL_CONSTRAINED_PASS_THROUGH_ROUTES = {
+ "/comprehendmedical": {"POST"},
+ "/comprehendmedical/{operation}": {"POST"},
+}
+
+
def test_pass_through_routes_support_all_methods():
"""
- Test that all pass-through routes support GET, POST, PUT, DELETE, PATCH methods
+ A pass-through route fronts a whole provider API, so narrowing its method
+ set turns a request the upstream would have accepted into a 405. The
+ exceptions are providers whose wire protocol admits only one method: Amazon
+ Comprehend Medical speaks AWS JSON 1.1, which is POST-only, so there is no
+ other method to forward.
"""
- # Import the routers
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
router as llm_router,
)
- # Expected HTTP methods
expected_methods = {"GET", "POST", "PUT", "DELETE", "PATCH"}
- # Function to check routes in a router
def check_router_methods(router):
for route in router.routes:
if isinstance(route, APIRoute):
- # Get path and methods for this route
path = route.path
methods = set(route.methods)
- print("supported methods for route", path, "are", methods)
- # Assert all expected methods are supported
+ allowed = PROTOCOL_CONSTRAINED_PASS_THROUGH_ROUTES.get(path, expected_methods)
assert (
- methods == expected_methods
- ), f"Route {path} does not support all methods. Supported: {methods}, Expected: {expected_methods}"
+ methods == allowed
+ ), f"Route {path} does not support all methods. Supported: {methods}, Expected: {allowed}"
- # Check both routers
check_router_methods(llm_router)
+def test_protocol_constrained_pass_through_exemptions_are_not_stale():
+ """
+ The exemption list above weakens the method contract, so it must not
+ outlive the routes it covers: a renamed or deleted route has to fail here
+ rather than sit in the list silently exempting nothing.
+ """
+ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
+ router as llm_router,
+ )
+
+ registered_paths = {route.path for route in llm_router.routes if isinstance(route, APIRoute)}
+ unmatched = set(PROTOCOL_CONSTRAINED_PASS_THROUGH_ROUTES) - registered_paths
+ assert not unmatched, f"Exempted pass-through routes no longer exist: {sorted(unmatched)}"
+
+
def test_is_bedrock_agent_runtime_route():
"""
Test that _is_bedrock_agent_runtime_route correctly identifies bedrock agent runtime endpoints
diff --git a/tests/search_tests/test_tinyfish_search.py b/tests/search_tests/test_tinyfish_search.py
index aca28544513..becb8287a29 100644
--- a/tests/search_tests/test_tinyfish_search.py
+++ b/tests/search_tests/test_tinyfish_search.py
@@ -35,11 +35,16 @@ MOCK_TINYFISH_RESPONSE = {
def _make_mock_response(
- json_data: dict, status_code: int = 200, request_url: str | None = None
+ json_data: dict,
+ status_code: int = 200,
+ request_url: str | None = None,
+ headers: dict | None = None,
) -> MagicMock:
mock = MagicMock()
mock.status_code = status_code
mock.json.return_value = json_data
+ # httpx.Headers normalizes keys to lowercase — mirror production behavior.
+ mock.headers = httpx.Headers(headers or {})
if request_url:
mock.request = MagicMock()
mock.request.url = httpx.URL(request_url)
@@ -163,7 +168,7 @@ class TestTinyfishSearch:
@pytest.mark.asyncio
async def test_fetch_param_round_trip(self):
- # End-to-end check: caller passes `fetch=...` (JSON-encoded tf-fetch
+ # End-to-end check: caller passes `fetch=...` (JSON-encoded fetch
# config); param reaches TinyFish on the request side and the nested
# `fetch` object on each result surfaces back to the SearchResult on the
# response side. No LiteLLM-side support code is required.
@@ -235,6 +240,58 @@ class TestTinyfishSearch:
assert result.results[0].title == "Result 0"
assert result.results[2].title == "Result 2"
+ @pytest.mark.asyncio
+ async def test_top_level_extras_surface_end_to_end(self):
+ # Envelope extras (`query`, `total_results`, `page`) must survive the
+ # full asearch dispatch — proves LiteLLM's entry-point plumbing outside
+ # our transformer doesn't accidentally strip them.
+ os.environ["TINYFISH_API_KEY"] = "sk-tinyfish-test"
+
+ mock_response = _make_mock_response(MOCK_TINYFISH_RESPONSE)
+
+ with patch(
+ "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get",
+ new_callable=AsyncMock,
+ ) as mock_get:
+ mock_get.return_value = mock_response
+
+ response = await litellm.asearch(
+ query="web automation tools",
+ search_provider="tinyfish",
+ )
+
+ assert getattr(response, "query", None) == "web automation tools"
+ assert getattr(response, "total_results", None) == 2
+ assert getattr(response, "page", None) == 0
+
+ @pytest.mark.asyncio
+ async def test_response_headers_surface_end_to_end(self):
+ # Response headers must land on `_hidden_params` after the full
+ # asearch dispatch (both raw and sanitized channels).
+ os.environ["TINYFISH_API_KEY"] = "sk-tinyfish-test"
+
+ mock_response = _make_mock_response(
+ MOCK_TINYFISH_RESPONSE,
+ headers={"X-Request-ID": "req-e2e-1"},
+ )
+
+ with patch(
+ "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get",
+ new_callable=AsyncMock,
+ ) as mock_get:
+ mock_get.return_value = mock_response
+
+ response = await litellm.asearch(
+ query="test",
+ search_provider="tinyfish",
+ )
+
+ raw = response._hidden_params["headers"]
+ add = response._hidden_params["additional_headers"]
+ # httpx lowercases; both channels agree on the value.
+ assert raw["x-request-id"] == "req-e2e-1"
+ assert add["llm_provider-x-request-id"] == "req-e2e-1"
+
@pytest.mark.asyncio
async def test_empty_results(self):
os.environ["TINYFISH_API_KEY"] = "sk-tinyfish-test"
diff --git a/tests/test_litellm/batches/test_batch_utils.py b/tests/test_litellm/batches/test_batch_utils.py
index 64b3df4180b..f7067053acb 100644
--- a/tests/test_litellm/batches/test_batch_utils.py
+++ b/tests/test_litellm/batches/test_batch_utils.py
@@ -959,8 +959,14 @@ async def test_handle_completed_vertex_batch_computes_cost_usage_and_models(monk
litellm_params={"vertex_project": "proj-1", "vertex_location": "us-central1"},
)
+ pricing = litellm.model_cost["vertex_ai/gemini-3.6-flash"]
+ batch_input = pricing["input_cost_per_token_batches"]
+ batch_output = pricing["output_cost_per_token_batches"]
+
+ assert batch_input < pricing["input_cost_per_token"]
+ assert batch_output < pricing["output_cost_per_token"]
assert result.cost > 0
- assert result.cost == pytest.approx(30 * 7.5e-07 + 15 * 3.75e-06)
+ assert result.cost == pytest.approx(30 * batch_input + 15 * batch_output)
assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (30, 15, 45)
assert result.models == ["gemini-3.6-flash", "gemini-3.6-flash"]
assert result.successful_requests == 2
diff --git a/tests/test_litellm/conftest.py b/tests/test_litellm/conftest.py
index 0dc8f56f3ce..c0644c88291 100644
--- a/tests/test_litellm/conftest.py
+++ b/tests/test_litellm/conftest.py
@@ -100,6 +100,12 @@ def isolate_host_aws_config(monkeypatch, isolated_aws_credentials_dir):
monkeypatch.delenv("AWS_DEFAULT_REGION", raising=False)
+@pytest.fixture(scope="function", autouse=True)
+def isolate_host_proxy_base_url(monkeypatch):
+ """Prevent a host PROXY_BASE_URL from outranking request-derived URLs during unit tests."""
+ monkeypatch.delenv("PROXY_BASE_URL", raising=False)
+
+
def _run_coroutine_if_needed(result):
if not asyncio.iscoroutine(result):
return
diff --git a/tests/test_litellm/integrations/otel/test_db_endpoint.py b/tests/test_litellm/integrations/otel/test_db_endpoint.py
new file mode 100644
index 00000000000..5ab0a927b52
--- /dev/null
+++ b/tests/test_litellm/integrations/otel/test_db_endpoint.py
@@ -0,0 +1,312 @@
+"""Tests for litellm/integrations/otel/model/db_endpoint.py
+
+Prisma talks to PostgreSQL through a loopback query engine, so a DB span with no
+``server.address`` gets attributed to ``localhost`` by the backend. These cover
+the endpoint derivation that names the real server, for the local engine and for
+remote and read-replica deployments, and pin the rule that no credential is ever
+exported.
+"""
+
+import os
+from unittest.mock import patch
+
+import pytest
+
+from litellm.integrations.otel.model.db_endpoint import (
+ DatabaseEndpoint,
+ db_span_attributes,
+ parse_database_endpoint,
+ postgres_endpoint,
+)
+
+LOCAL_DSN = "postgresql://llmproxy:dbpassword9090@localhost:5432/litellm"
+REMOTE_DSN = "postgresql://llmproxy:s3cr3t@litellm-prod.abc123.us-east-1.rds.amazonaws.com:6432/litellm?schema=reporting&sslmode=require"
+REPLICA_DSN = "postgresql://reader:r3ad0nly@litellm-prod-ro.abc123.us-east-1.rds.amazonaws.com/litellm_replica"
+
+
+def _resolve(service, call_type=None, database_url=None, read_replica_url=None):
+ """Resolve attributes with the two DB env vars set, as the proxy sets them."""
+ env = {k: v for k, v in (("DATABASE_URL", database_url), ("DATABASE_URL_READ_REPLICA", read_replica_url)) if v}
+ with patch.dict(os.environ, env, clear=False):
+ for absent in {"DATABASE_URL", "DATABASE_URL_READ_REPLICA"} - set(env):
+ os.environ.pop(absent, None)
+ return dict(db_span_attributes(service, call_type))
+
+
+def test_local_prisma_engine_endpoint_is_the_postgres_server_not_the_engine():
+ assert parse_database_endpoint(LOCAL_DSN) == DatabaseEndpoint(
+ address="localhost", port=5432, namespace="litellm"
+ )
+
+
+def test_remote_endpoint_keeps_host_port_and_schema_qualified_namespace():
+ assert parse_database_endpoint(REMOTE_DSN) == DatabaseEndpoint(
+ address="litellm-prod.abc123.us-east-1.rds.amazonaws.com",
+ port=6432,
+ namespace="litellm|reporting",
+ )
+
+
+def test_read_replica_dsn_parses_to_the_replica_host_and_database():
+ assert parse_database_endpoint(REPLICA_DSN) == DatabaseEndpoint(
+ address="litellm-prod-ro.abc123.us-east-1.rds.amazonaws.com",
+ port=5432,
+ namespace="litellm_replica",
+ )
+
+
+def test_default_schema_is_not_spelled_out_in_the_namespace():
+ """``?schema=public`` and no schema at all are the same deployment, so they
+ must not split a group-by on db.namespace."""
+ assert parse_database_endpoint("postgresql://u:p@db.internal/litellm?schema=public") == parse_database_endpoint(
+ "postgresql://u:p@db.internal/litellm"
+ )
+
+
+def test_unix_socket_host_parameter_wins_over_the_netloc():
+ """libpq and the Cloud SQL connector both put the real target in ``host=``
+ behind a localhost netloc, which is the attribution this module removes."""
+ assert parse_database_endpoint(
+ "postgresql://u:p@localhost:5432/litellm?host=/cloudsql/proj:us-east1:inst"
+ ) == DatabaseEndpoint(address="/cloudsql/proj:us-east1:inst", port=5432, namespace="litellm")
+
+
+def test_socket_only_dsn_without_a_netloc_host_still_resolves():
+ assert parse_database_endpoint("postgresql:///litellm?host=/var/run/postgresql") == DatabaseEndpoint(
+ address="/var/run/postgresql", port=5432, namespace="litellm"
+ )
+
+
+def test_percent_encoded_database_name_is_decoded():
+ endpoint = parse_database_endpoint("postgresql://u:p@db.internal/litellm%20prod")
+ assert endpoint is not None and endpoint.namespace == "litellm prod"
+
+
+MISPARSED_AUTHORITY_DSNS = (
+ ("postgresql://litellm:/kJ8xQz+9wT@db.internal:5432/litellm", "kJ8xQz+9wT"),
+ ("postgresql://litellm:12345/aBcD@db.internal:5432/litellm", "aBcD"),
+ # '#' sends the tail to the fragment and '?' to the query, so the path is
+ # empty and only the stranded userinfo '@' reveals the mis-split.
+ ("postgresql://litellm:12345#aBcD@db.internal/litellm", "aBcD"),
+ ("postgresql://litellm:12345?aBcD@db.internal/litellm", "aBcD"),
+ # A '?'-stranded tail that happens to parse as parameters, including one
+ # that hijacks the host= parameter into server.address.
+ ("postgresql://litellm:12345?a=aBcD@db.internal/litellm", "aBcD"),
+ ("postgresql://litellm:12345?host=aBcD@db.internal/litellm", "aBcD"),
+ # Both '/' and '?key=value' together: the slash leaves a clean path holding
+ # the password remainder and the query still parses, so only the stranded
+ # at-sign gives it away.
+ ("postgresql://litellm:12345/aBcD?x=1@db.internal/litellm", "aBcD"),
+)
+
+
+@pytest.mark.parametrize(("dsn", "secret"), MISPARSED_AUTHORITY_DSNS)
+def test_unencoded_slash_in_password_never_yields_an_endpoint(dsn, secret):
+ """An unencoded '/' truncates the authority, so urlparse reports the username
+ as the host and the password tail as the database. Postgres drivers reject
+ such a DSN outright, so the only safe reading is no endpoint at all."""
+ assert parse_database_endpoint(dsn) is None
+
+
+@pytest.mark.parametrize(("dsn", "secret"), MISPARSED_AUTHORITY_DSNS)
+def test_unencoded_slash_in_password_never_reaches_a_span(dsn, secret):
+ attrs = _resolve("postgres", "get_data", database_url=dsn)
+ exported = " ".join(str(value) for value in attrs.values())
+ assert secret not in exported
+ assert "db.namespace" not in attrs
+ assert "server.address" not in attrs
+
+
+def test_extra_path_segment_yields_no_endpoint():
+ """A database name cannot hold an unencoded '/', so a second path segment
+ means the authority was mis-split even when no '@' survived into the path."""
+ assert parse_database_endpoint("postgresql://db.internal:5432/litellm/extra") is None
+
+
+@pytest.mark.parametrize("dsn", [d for d, _ in MISPARSED_AUTHORITY_DSNS])
+def test_a_mis_split_authority_never_exports_the_database_username(dsn):
+ """The username lands in ``parsed.hostname`` when the authority truncates, so
+ a span would name the DB user as the server."""
+ attrs = _resolve("postgres", "get_data", database_url=dsn)
+ assert "server.address" not in attrs
+ assert "litellm" not in " ".join(str(v) for v in attrs.values())
+
+
+@pytest.mark.parametrize(
+ "dsn",
+ [
+ "postgresql://db.internal:5432/litellm?application_name=svc@prod",
+ "postgresql://db.internal:5432/litellm?user=admin@company.com",
+ ],
+)
+def test_an_unencoded_at_sign_in_a_query_forfeits_the_endpoint(dsn):
+ """This shape is byte-for-byte indistinguishable from a mis-split password,
+ so it resolves to no endpoint rather than risking a credential fragment.
+ Percent-encoding the at-sign restores the attributes."""
+ assert parse_database_endpoint(dsn) is None
+ assert parse_database_endpoint(dsn.replace("@", "%40")) is not None
+
+
+def test_host_and_port_query_parameters_are_honoured_together():
+ assert parse_database_endpoint("postgresql://ignored/litellm?host=real.internal&port=6543") == DatabaseEndpoint(
+ address="real.internal", port=6543, namespace="litellm"
+ )
+
+
+def test_percent_encoded_password_still_resolves_the_endpoint():
+ """The encoded spelling is the one a driver accepts, so it must keep working."""
+ assert parse_database_endpoint("postgresql://litellm:pa%2Fssw0rd@db.internal:5432/litellm") == DatabaseEndpoint(
+ address="db.internal", port=5432, namespace="litellm"
+ )
+
+
+def test_hostless_socket_dsn_still_names_the_database():
+ """``postgresql:///litellm`` is a valid local-socket DSN that Prisma accepts,
+ so the database is knowable even though no server address is."""
+ assert parse_database_endpoint("postgresql:///litellm") == DatabaseEndpoint(
+ address=None, port=None, namespace="litellm"
+ )
+
+
+def test_hostless_socket_dsn_emits_namespace_without_a_server():
+ attrs = _resolve("postgres", "get_data", database_url="postgresql:///litellm")
+ assert attrs["db.namespace"] == "litellm"
+ assert "server.address" not in attrs
+ assert "server.port" not in attrs
+
+
+def test_dsn_with_neither_host_nor_database_yields_no_endpoint():
+ assert parse_database_endpoint("postgresql://") is None
+
+
+def test_prisma_default_schema_is_left_implicit():
+ endpoint = parse_database_endpoint("postgresql://u:p@db.internal/litellm?schema=public")
+ assert endpoint is not None and endpoint.namespace == "litellm"
+
+
+@pytest.mark.parametrize("spelling", ["PUBLIC", "Public", "reporting"])
+def test_a_non_default_schema_stays_in_the_namespace(spelling):
+ """Prisma quotes the schema name, so ``?schema=PUBLIC`` provisions a second
+ schema alongside ``public`` with its own tables. Case-folding them into one
+ namespace would report two different schemas as the same database."""
+ endpoint = parse_database_endpoint(f"postgresql://u:p@db.internal/litellm?schema={spelling}")
+ assert endpoint is not None and endpoint.namespace == f"litellm|{spelling}"
+
+
+def test_postgres_scheme_alias_is_accepted():
+ assert parse_database_endpoint("postgres://u:p@db.internal/litellm") == DatabaseEndpoint(
+ address="db.internal", port=5432, namespace="litellm"
+ )
+
+
+@pytest.mark.parametrize(
+ "dsn",
+ [
+ None,
+ "",
+ "mysql://u:p@db.internal:3306/litellm",
+ "postgresql://u:p@db.internal:not-a-port/litellm",
+ "not a url at all",
+ ],
+)
+def test_unusable_dsn_degrades_to_no_endpoint(dsn):
+ assert parse_database_endpoint(dsn) is None
+
+
+def test_database_without_name_or_schema_has_no_namespace():
+ assert parse_database_endpoint("postgresql://u:p@db.internal:5432/") == DatabaseEndpoint(
+ address="db.internal", port=5432, namespace=None
+ )
+
+
+def test_postgres_service_span_carries_system_operation_and_endpoint():
+ assert _resolve("postgres", "get_data", database_url=REMOTE_DSN) == {
+ "db.system.name": "postgresql",
+ "db.system": "postgresql",
+ "db.operation.name": "get_data",
+ "server.address": "litellm-prod.abc123.us-east-1.rds.amazonaws.com",
+ "server.port": 6432,
+ "db.namespace": "litellm|reporting",
+ }
+
+
+def test_legacy_db_system_is_dual_emitted_for_datadog():
+ """Datadog's OTLP intake infers the database span type from ``db.system``,
+ not from the semconv-current ``db.system.name``."""
+ assert _resolve("postgres", "get_data", database_url=LOCAL_DSN)["db.system"] == "postgresql"
+ assert _resolve("redis", "set")["db.system"] == "redis"
+
+
+def test_batch_write_service_is_also_attributed_to_postgres():
+ attrs = _resolve("batch_write_to_db", "_PROXY_track_cost_callback", database_url=REMOTE_DSN)
+ assert attrs["db.system.name"] == "postgresql"
+ assert attrs["server.address"] == "litellm-prod.abc123.us-east-1.rds.amazonaws.com"
+
+
+def test_redis_service_never_borrows_the_postgres_endpoint():
+ assert _resolve("redis", "set", database_url=REMOTE_DSN) == {
+ "db.system.name": "redis",
+ "db.system": "redis",
+ "db.operation.name": "set",
+ }
+
+
+def test_non_datastore_service_gets_no_db_attributes():
+ assert _resolve("reset_budget_job", "reset_budget", database_url=REMOTE_DSN) == {}
+
+
+def test_configured_read_replica_suppresses_the_endpoint_rather_than_naming_the_primary():
+ """Reads are routed to the replica per Prisma call, underneath the span, so
+ naming the writer would pin replica latency onto the primary."""
+ attrs = _resolve("postgres", "get_data", database_url=REMOTE_DSN, read_replica_url=REPLICA_DSN)
+ assert attrs == {
+ "db.system.name": "postgresql",
+ "db.system": "postgresql",
+ "db.operation.name": "get_data",
+ }
+
+
+def test_endpoint_attributes_are_omitted_when_database_url_is_unset():
+ assert _resolve("postgres", "get_data") == {
+ "db.system.name": "postgresql",
+ "db.system": "postgresql",
+ "db.operation.name": "get_data",
+ }
+
+
+def test_blank_call_type_does_not_emit_an_empty_operation_attribute():
+ assert "db.operation.name" not in _resolve("postgres", "")
+ assert "db.operation.name" not in _resolve("postgres", None)
+
+
+@pytest.mark.parametrize(
+ ("dsn", "secrets"),
+ [
+ (LOCAL_DSN, ("dbpassword9090", "llmproxy")),
+ (REMOTE_DSN, ("s3cr3t", "llmproxy", "sslmode")),
+ (REPLICA_DSN, ("r3ad0nly", "reader")),
+ ],
+)
+def test_no_credential_reaches_any_exported_attribute(dsn, secrets):
+ attrs = _resolve("postgres", "get_data", database_url=dsn)
+ assert attrs["server.address"]
+ exported = " ".join(str(value) for value in attrs.values())
+ for secret in secrets:
+ assert secret not in exported
+
+
+def test_a_runtime_endpoint_change_is_reflected_on_the_next_span():
+ """The RDS IAM refresh, the reconnect path and the DB-backed
+ environment_variables overlay can all rewrite DATABASE_URL after startup, so
+ a value cached for the process lifetime would report a server the process no
+ longer talks to."""
+ first = _resolve("postgres", "get_data", database_url=LOCAL_DSN)
+ assert first["server.address"] == "localhost"
+ moved = _resolve("postgres", "get_data", database_url=REMOTE_DSN)
+ assert moved["server.address"] == "litellm-prod.abc123.us-east-1.rds.amazonaws.com"
+
+
+def test_a_replica_configured_after_the_first_span_suppresses_the_endpoint():
+ assert _resolve("postgres", "get_data", database_url=REMOTE_DSN)["server.address"]
+ later = _resolve("postgres", "get_data", database_url=REMOTE_DSN, read_replica_url=REPLICA_DSN)
+ assert "server.address" not in later
diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py
index 82b074220fa..bb2d970e9c7 100644
--- a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py
+++ b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py
@@ -8,6 +8,8 @@ hooks, proxy SERVER span lifecycle (start + setters), parent-context resolution
import asyncio
import contextlib
+import os
+from unittest.mock import patch
from datetime import datetime, timedelta, timezone
import pytest
@@ -1521,6 +1523,36 @@ def test_async_service_success_hook_emits_service_span():
assert span.status.status_code is StatusCode.UNSET
+def test_postgres_db_span_names_the_database_server_not_the_prisma_engine():
+ """Prisma reaches Postgres over loopback, so without server.address the
+ backend attributes the wait to localhost."""
+ dsn = "postgresql://llmproxy:dbpassword9090@litellm-prod.abc123.us-east-1.rds.amazonaws.com:6432/litellm?schema=reporting"
+ logger, exporter = _logger()
+ parent = _service_parent(logger)
+ try:
+ with patch.dict(os.environ, {"DATABASE_URL": dsn}, clear=False):
+ os.environ.pop("DATABASE_URL_READ_REPLICA", None)
+ asyncio.run(
+ logger.async_service_success_hook(
+ payload=_ServicePayload("postgres", "get_data"),
+ parent_otel_span=parent,
+ )
+ )
+ finally:
+ parent.end()
+ span = {s.name: s for s in exporter.get_finished_spans()}["postgres get_data"]
+ assert span.kind is SpanKind.CLIENT
+ assert span.attributes["db.system.name"] == "postgresql"
+ assert span.attributes["db.operation.name"] == "get_data"
+ assert span.attributes["server.address"] == "litellm-prod.abc123.us-east-1.rds.amazonaws.com"
+ assert span.attributes["server.port"] == 6432
+ assert span.attributes["db.namespace"] == "litellm|reporting"
+ assert span.attributes["db.system"] == "postgresql"
+ exported = " ".join(str(value) for value in span.attributes.values())
+ assert "dbpassword9090" not in exported
+ assert "llmproxy" not in exported
+
+
def test_async_service_failure_hook_marks_error_status():
logger, exporter = _logger()
parent = _service_parent(logger)
diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py
index fa1c9fa8a79..28d2cb22e7c 100644
--- a/tests/test_litellm/integrations/test_opentelemetry.py
+++ b/tests/test_litellm/integrations/test_opentelemetry.py
@@ -34,6 +34,7 @@ from litellm.integrations.opentelemetry import (
_normalize_team_metadata_keys,
)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
+from litellm.types.services import ServiceLoggerPayload, ServiceTypes
class TestOpenTelemetryGuardrails(unittest.TestCase):
@@ -6212,3 +6213,110 @@ class TestDynamicTracerProviderCache(unittest.TestCase):
self.assertTrue(entry.owns_exporter)
self.assertIsNotNone(entry.provider._atexit_handler)
+class TestOpenTelemetryDatabaseSemconvAttributes(unittest.TestCase):
+ """A Postgres service span must name the PostgreSQL server it reached.
+
+ Without ``db.system`` and ``server.address``, the only host in the trace is
+ the loopback address of Prisma's local query engine, so the backend
+ attributes the wait to ``localhost`` and it cannot be correlated with the
+ database's own metrics.
+ """
+
+ DSN = "postgresql://llmproxy:dbpassword9090@litellm-prod.abc123.us-east-1.rds.amazonaws.com:6432/litellm?schema=reporting"
+ REPLICA_DSN = "postgresql://reader:r3ad0nly@litellm-prod-ro.abc123.us-east-1.rds.amazonaws.com/litellm"
+
+ def _service_span(self, service, call_type, dsn, error=None, replica_dsn=None):
+ exporter = InMemorySpanExporter()
+ provider = TracerProvider()
+ provider.add_span_processor(SimpleSpanProcessor(exporter))
+ otel = OpenTelemetry()
+ otel.tracer = provider.get_tracer(__name__)
+ parent = otel.tracer.start_span("Received Proxy Server Request")
+ payload = ServiceLoggerPayload(
+ is_error=error is not None,
+ error=error,
+ service=service,
+ duration=0.25,
+ call_type=call_type,
+ event_metadata=None,
+ )
+ hook = otel.async_service_failure_hook if error else otel.async_service_success_hook
+ kwargs = {"error": error} if error else {}
+ env = {k: v for k, v in (("DATABASE_URL", dsn), ("DATABASE_URL_READ_REPLICA", replica_dsn)) if v}
+ with patch.dict(os.environ, env, clear=False):
+ for absent in {"DATABASE_URL", "DATABASE_URL_READ_REPLICA"} - set(env):
+ os.environ.pop(absent, None)
+ asyncio.run(
+ hook(
+ payload=payload,
+ parent_otel_span=parent,
+ start_time=datetime.now(),
+ end_time=datetime.now(),
+ **kwargs,
+ )
+ )
+ parent.end()
+ return next(s for s in exporter.get_finished_spans() if s.name == service.value)
+
+ def test_postgres_span_names_the_database_server(self):
+ span = self._service_span(ServiceTypes.DB, "get_data", self.DSN)
+ self.assertEqual(span.attributes["db.system.name"], "postgresql")
+ self.assertEqual(span.attributes["db.operation.name"], "get_data")
+ self.assertEqual(
+ span.attributes["server.address"],
+ "litellm-prod.abc123.us-east-1.rds.amazonaws.com",
+ )
+ self.assertEqual(span.attributes["server.port"], 6432)
+ self.assertEqual(span.attributes["db.namespace"], "litellm|reporting")
+
+ def test_datastore_span_is_a_client_span_carrying_the_legacy_db_system(self):
+ """Datadog types a span as a database call from CLIENT kind plus
+ ``db.system``; an INTERNAL span is classified as custom work."""
+ span = self._service_span(ServiceTypes.DB, "get_data", self.DSN)
+ self.assertEqual(span.kind, trace.SpanKind.CLIENT)
+ self.assertEqual(span.attributes["db.system"], "postgresql")
+
+ def test_internal_service_span_stays_internal(self):
+ span = self._service_span(ServiceTypes.RESET_BUDGET_JOB, "reset_budget", self.DSN)
+ self.assertEqual(span.kind, trace.SpanKind.INTERNAL)
+ self.assertNotIn("db.system.name", span.attributes)
+ self.assertNotIn("server.address", span.attributes)
+
+ def test_existing_service_and_call_type_attributes_are_unchanged(self):
+ span = self._service_span(ServiceTypes.DB, "get_data", self.DSN)
+ self.assertEqual(span.attributes["service"], "postgres")
+ self.assertEqual(span.attributes["call_type"], "get_data")
+
+ def test_failed_postgres_span_also_names_the_database_server(self):
+ span = self._service_span(ServiceTypes.DB, "get_data", self.DSN, error="connection refused")
+ self.assertEqual(span.attributes["db.system.name"], "postgresql")
+ self.assertEqual(span.kind, trace.SpanKind.CLIENT)
+ self.assertEqual(
+ span.attributes["server.address"],
+ "litellm-prod.abc123.us-east-1.rds.amazonaws.com",
+ )
+ self.assertEqual(span.attributes["error"], "connection refused")
+
+ def test_no_credential_from_the_dsn_lands_on_the_span(self):
+ span = self._service_span(ServiceTypes.DB, "get_data", self.DSN)
+ exported = " ".join(str(value) for value in span.attributes.values())
+ self.assertIn("litellm-prod.abc123.us-east-1.rds.amazonaws.com", exported)
+ self.assertNotIn("dbpassword9090", exported)
+ self.assertNotIn("llmproxy", exported)
+
+ def test_redis_span_does_not_borrow_the_postgres_endpoint(self):
+ span = self._service_span(ServiceTypes.REDIS, "async_set_cache", self.DSN)
+ self.assertEqual(span.attributes["db.system.name"], "redis")
+ self.assertEqual(span.kind, trace.SpanKind.CLIENT)
+ self.assertNotIn("server.address", span.attributes)
+
+ def test_configured_read_replica_suppresses_the_endpoint(self):
+ span = self._service_span(ServiceTypes.DB, "get_data", self.DSN, replica_dsn=self.REPLICA_DSN)
+ self.assertEqual(span.attributes["db.system.name"], "postgresql")
+ self.assertNotIn("server.address", span.attributes)
+ self.assertNotIn("db.namespace", span.attributes)
+
+ def test_unset_database_url_leaves_the_span_without_endpoint_attributes(self):
+ span = self._service_span(ServiceTypes.DB, "get_data", None)
+ self.assertEqual(span.attributes["db.system.name"], "postgresql")
+ self.assertNotIn("server.address", span.attributes)
diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py
index af40245ebfa..f9311497729 100644
--- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py
+++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py
@@ -241,10 +241,32 @@ def test_split_concatenated_json_non_dict_value():
assert result == [{}]
-def test_split_concatenated_json_invalid_raises():
- """Completely invalid JSON raises JSONDecodeError."""
- with pytest.raises(json.JSONDecodeError):
- split_concatenated_json_objects("not json at all")
+def test_split_concatenated_json_wholly_invalid_returns_empty():
+ """
+ Wholly unparseable JSON degrades to an empty list instead of raising.
+
+ Regression for https://github.com/BerriAI/litellm/issues/18667: a raise
+ here propagated out of `_convert_to_bedrock_tool_call_invoke` and turned
+ every replayed conversation into a 500.
+ """
+ assert split_concatenated_json_objects("not json at all") == []
+
+
+def test_split_concatenated_json_malformed_object_returns_empty():
+ """
+ A single malformed object (missing comma between keys) degrades to an
+ empty list rather than raising `Expecting ',' delimiter`.
+ """
+ assert split_concatenated_json_objects('{"location": "Boston" "unit": "celsius"}') == []
+
+
+def test_split_concatenated_json_salvages_prefix_before_truncated_tail():
+ """
+ Complete objects parsed before an unparseable/truncated tail are kept;
+ only the bad tail is discarded.
+ """
+ result = split_concatenated_json_objects('{"a": 1}{"b": 2}{"c":')
+ assert result == [{"a": 1}, {"b": 2}]
# ---------------------------------------------------------------------------
diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py
index de5d0a180c6..fffbc884782 100644
--- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py
+++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py
@@ -2287,6 +2287,116 @@ def test_bedrock_tool_call_invoke_non_dict_arguments():
assert result[0]["toolUse"]["input"] == {}
+def test_bedrock_tool_call_invoke_malformed_json_does_not_raise():
+ """
+ Regression for https://github.com/BerriAI/litellm/issues/18667.
+
+ When the model emits malformed JSON in tool-call arguments (here a
+ missing comma between keys), replaying that history must NOT raise
+ `Unable to convert openai tool calls ... Expecting ',' delimiter`.
+ It degrades to an empty-object input so the conversation can continue.
+ """
+ tool_calls = [
+ {
+ "id": "toolu_abc123",
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "arguments": '{"location": "Boston" "unit": "celsius"}',
+ },
+ }
+ ]
+ result = _convert_to_bedrock_tool_call_invoke(tool_calls)
+ assert len(result) == 1
+ assert result[0]["toolUse"]["toolUseId"] == "toolu_abc123"
+ assert result[0]["toolUse"]["name"] == "get_weather"
+ assert result[0]["toolUse"]["input"] == {}
+
+
+def test_bedrock_tool_call_invoke_salvages_valid_prefix_before_truncated_tail():
+ """
+ A valid leading object followed by a truncated tail keeps the valid
+ object rather than dropping everything or raising.
+ """
+ tool_calls = [
+ {
+ "id": "call_partial",
+ "type": "function",
+ "function": {"name": "shell", "arguments": '{"cmd": "ls"}{"cmd":'},
+ }
+ ]
+ result = _convert_to_bedrock_tool_call_invoke(tool_calls)
+ assert len(result) == 1
+ assert result[0]["toolUse"]["input"] == {"cmd": "ls"}
+
+
+def test_bedrock_tool_call_invoke_mixed_turn_survives_one_malformed_call():
+ """
+ Regression for LIT-4574: an assistant turn with several tool calls where only one
+ has malformed/truncated arguments must keep the valid calls intact and degrade just
+ the bad one to empty input, instead of killing the entire turn.
+ """
+ tool_calls = [
+ {
+ "id": "t_good",
+ "type": "function",
+ "function": {
+ "name": "good_tool",
+ "arguments": '{"item_type": "email", "item_id": "AAMkAD=="}',
+ },
+ },
+ {
+ "id": "t_bad",
+ "type": "function",
+ "function": {"name": "bad_tool", "arguments": '{"item_type": "email"'},
+ },
+ ]
+ result = _convert_to_bedrock_tool_call_invoke(tool_calls)
+ tool_uses = [block["toolUse"] for block in result if "toolUse" in block]
+ assert len(tool_uses) == 2
+ by_name = {tool_use["name"]: tool_use for tool_use in tool_uses}
+ assert by_name["good_tool"]["input"] == {"item_type": "email", "item_id": "AAMkAD=="}
+ assert by_name["bad_tool"]["input"] == {}
+
+
+def test_bedrock_tool_call_invoke_truncated_json_arguments():
+ """
+ Truncated tool call arguments (issue #35303) must not raise. A client replaying a
+ partially streamed tool call would otherwise trigger a pre-network exception that the
+ router maps to a retryable APIConnectionError and retries through the fallback graph.
+ """
+ tool_calls = [
+ {
+ "id": "tooluse_MAh2QLVjBRkvi5QJkLQ08V",
+ "type": "function",
+ "function": {
+ "name": "replace_note_content",
+ "arguments": '{"note_id": "999af35c-4061-4ece-8581-7d43fc988ba4", "title": "WG"',
+ },
+ }
+ ]
+ result = _convert_to_bedrock_tool_call_invoke(tool_calls)
+ assert len(result) == 1
+ assert result[0]["toolUse"]["toolUseId"] == "tooluse_MAh2QLVjBRkvi5QJkLQ08V"
+ assert result[0]["toolUse"]["input"] == {}
+
+
+def test_bedrock_tool_call_invoke_unconvertible_raises_non_retryable_bad_request():
+ """
+ Conversion failures are client input errors, so they must surface as a non-retryable
+ BadRequestError instead of a bare Exception that maps to APIConnectionError, and the
+ message must not embed the tool call payload (issue #35303).
+ """
+ tool_calls = [{"id": "call_bad", "type": "function", "function": None}]
+
+ with pytest.raises(litellm.BadRequestError) as exc_info:
+ _convert_to_bedrock_tool_call_invoke(tool_calls)
+
+ assert exc_info.value.status_code == 400
+ assert "call_bad" in str(exc_info.value)
+ assert "function" not in str(exc_info.value).split("Received error=")[0]
+
+
def test_make_valid_bedrock_tool_name_preserves_hyphens():
assert make_valid_bedrock_tool_name("my-tool") == "my-tool"
assert (
diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
index e46057fe11a..99781a2f211 100644
--- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
+++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
@@ -4918,3 +4918,51 @@ def test_payload_without_guardrail_cost_is_unchanged(logging_obj):
assert payload is not None
assert payload["response_cost"] == pytest.approx(0.0000429)
assert payload["cost_breakdown"] is None
+
+
+_AWS_SECRET = "wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY"
+_GEMINI_KEY = "AIzaSyC0000000000000000000000000000000"
+
+
+def test_empty_api_base_does_not_dump_call_state(logging_obj):
+ """Direct (non-HTTP) providers pass api_base='', which used to echo model_call_details."""
+ logging_obj.model_call_details["litellm_params"] = {
+ "api_key": "sk-proj-hunter2hunter2hunter2hunter2",
+ "aws_secret_access_key": _AWS_SECRET,
+ }
+
+ curl_command = logging_obj._get_request_curl_command(
+ api_base="",
+ headers={},
+ additional_args={},
+ data={"model": "some-model"},
+ )
+
+ assert "litellm_call_id" not in curl_command
+ assert _AWS_SECRET not in curl_command
+ assert "hunter2" not in curl_command
+
+
+def test_pre_call_redacts_and_masks_raw_request(logging_obj):
+ """log_raw_request_response echoes the request body and api_base back to loggers/UI."""
+ metadata = {"user_api_key_alias": "qa-key"}
+ logging_obj.model_call_details["litellm_params"] = {"metadata": metadata}
+ logging_obj.log_raw_request_response = True
+
+ logging_obj.pre_call(
+ input="hi",
+ api_key="",
+ additional_args={
+ "api_base": f"https://generativelanguage.googleapis.com/v1beta/models/x:generateContent?key={_GEMINI_KEY}",
+ "headers": {},
+ "complete_input_dict": {"aws_secret_access_key": _AWS_SECRET},
+ },
+ )
+
+ raw_request = metadata["raw_request"]
+ assert _AWS_SECRET not in raw_request
+ assert "REDACTED" in raw_request
+
+ raw_api_base = logging_obj.model_call_details["raw_request_typed_dict"]["raw_request_api_base"]
+ assert _GEMINI_KEY not in raw_api_base
+ assert "key=*****" in raw_api_base
diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py
index 2509f6480d5..298360789eb 100644
--- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py
+++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py
@@ -6043,3 +6043,12 @@ def test_streaming_usage_chunk_is_transformed():
assert chunk.usage.prompt_tokens == 11
assert chunk.usage.completion_tokens == 4
assert chunk.usage.total_tokens == 15
+
+
+def test_update_optional_params_with_thinking_tokens_bool_thinking_does_not_crash():
+ config = AmazonConverseConfig()
+ optional_params = {"thinking": True}
+ config.update_optional_params_with_thinking_tokens(
+ non_default_params={"thinking": True}, optional_params=optional_params
+ )
+ assert "maxTokens" not in optional_params
diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py
index 2dda8bf722a..69f2312f203 100644
--- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py
+++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py
@@ -1,7 +1,9 @@
import asyncio
import json
+import logging
import os
import sys
+import time
from unittest.mock import AsyncMock, Mock, patch
import httpx
@@ -9,6 +11,7 @@ import pytest
sys.path.insert(0, os.path.abspath("../../../..")) # Adds the parent directory to the system path
import litellm
+from litellm._logging import verbose_logger
from litellm.integrations.code_interpreter_interception.handler import (
CodeInterpreterInterceptionLogger,
LITELLM_CODE_EXECUTION_TOOL_NAME,
@@ -2158,3 +2161,68 @@ async def test_vector_store_search_handler_direct_config_async_skips_http():
pre_call_args = logging_obj.pre_call.call_args.kwargs["additional_args"]
assert pre_call_args["query"] == ["q1", "q2"]
assert pre_call_args["vector_store_id"] == "vs_direct"
+
+
+def _direct_vector_store_debug_logging_obj():
+ from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
+
+ logging_obj = LitellmLogging(
+ model="valkey",
+ messages=[{"role": "user", "content": "q"}],
+ stream=False,
+ call_type="vector_store_search",
+ start_time=time.time(),
+ litellm_call_id="vs-debug-call-id",
+ function_id="vs-debug-function-id",
+ log_raw_request_response=True,
+ )
+ logging_obj.update_environment_variables(
+ model="valkey",
+ optional_params={"vector_store_id": "vs_direct", "query": "q"},
+ litellm_params={
+ "litellm_call_id": "vs-debug-call-id",
+ "vector_store_id": "vs_direct",
+ "litellm_request_debug": True,
+ "metadata": {"user_api_key_alias": "vs-test-key"},
+ "valkey_host": "valkey.internal",
+ "valkey_password": "sup3r-s3cret-valkey-pw",
+ "litellm_embedding_config": {"api_key": "sk-embedding-s3cret"},
+ },
+ )
+ return logging_obj
+
+
+@pytest.mark.parametrize("is_async", [False, True])
+def test_direct_vector_store_search_debug_log_omits_stored_credentials(caplog, is_async):
+ """Regression: an empty api_base made pre_call dump the whole model_call_details, so every
+ search shipped the stored valkey_password / embedding api_key into the raw_request metadata."""
+ handler = BaseLLMHTTPHandler()
+ stub_response = {"object": "vector_store.search_results.page", "search_query": "q", "data": []}
+ config = _make_stub_direct_vector_store_config(stub_response)
+ logging_obj = _direct_vector_store_debug_logging_obj()
+
+ with caplog.at_level(logging.DEBUG, logger=verbose_logger.name):
+ result = handler.vector_store_search_handler(
+ vector_store_id="vs_direct",
+ query="q",
+ vector_store_search_optional_params={"max_num_results": 4},
+ vector_store_provider_config=config,
+ custom_llm_provider="valkey",
+ litellm_params=GenericLiteLLMParams(
+ valkey_host="valkey.internal",
+ valkey_password="sup3r-s3cret-valkey-pw",
+ ),
+ logging_obj=logging_obj,
+ _is_async=is_async,
+ )
+ if is_async:
+ result = asyncio.run(result)
+
+ assert result is stub_response
+ raw_request = logging_obj.model_call_details["litellm_params"]["metadata"]["raw_request"]
+ assert "sup3r-s3cret-valkey-pw" not in raw_request
+ assert "sk-embedding-s3cret" not in raw_request
+ assert "valkey://vs_direct" in raw_request
+ logged = "\n".join(record.getMessage() for record in caplog.records)
+ assert "sup3r-s3cret-valkey-pw" not in logged
+ assert "sk-embedding-s3cret" not in logged
diff --git a/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py b/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py
index 6f5aaabae06..510776ddfdf 100644
--- a/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py
+++ b/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py
@@ -271,6 +271,47 @@ class TestDashscopeCostCalculator:
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
+ def test_dashscope_nested_cache_creation_input_tokens_bill_at_cache_write_rate(self):
+ """
+ Regression (LIT-5757): DashScope nests cache_creation_input_tokens inside
+ prompt_tokens_details; those tokens must bill at the tier's cache-creation
+ rate instead of being folded into text tokens at the input rate.
+ """
+ self._register_tiered_model(
+ "dashscope/qwen-nested-cache-write-test",
+ [
+ {
+ "range": [0, 128000],
+ "input_cost_per_token": 4e-07,
+ "cache_read_input_token_cost": 1.6e-07,
+ "cache_creation_input_token_cost": 5e-07,
+ "output_cost_per_token": 1.6e-06,
+ }
+ ],
+ )
+
+ usage = Usage(
+ prompt_tokens=2059,
+ completion_tokens=201,
+ total_tokens=2260,
+ prompt_tokens_details={
+ "cached_tokens": 0,
+ "text_tokens": 2059,
+ "cache_type": "ephemeral",
+ "cache_creation_input_tokens": 2048,
+ "cache_creation": {"ephemeral_5m_input_tokens": 2048},
+ },
+ completion_tokens_details={"reasoning_tokens": 170},
+ )
+
+ prompt_cost, _ = dashscope_cost_per_token(
+ model="qwen-nested-cache-write-test", usage=usage
+ )
+
+ assert math.isclose(
+ prompt_cost, (2048 * 5e-07) + (11 * 4e-07), rel_tol=1e-10
+ )
+
def test_dashscope_tiered_cache_creation_falls_back_to_tier_input_rate(self):
"""
Tiers without a cache_creation_input_token_cost bill cache-creation tokens at
diff --git a/tests/test_litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py b/tests/test_litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py
index ec51e5d303d..d5783e3567f 100644
--- a/tests/test_litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py
+++ b/tests/test_litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py
@@ -101,3 +101,8 @@ async def test_async_transform_request_strips_unsupported_tools_from_body():
assert [tool["type"] for tool in body["tools"]] == ["function"]
assert body["tools"][0]["function"]["name"] == "shell"
+
+
+def test_thinking_mode_active_bool_thinking_returns_false_without_crashing():
+ config = DeepSeekChatConfig()
+ assert config._thinking_mode_active(model="deepseek-reasoner", optional_params={"thinking": True}) is False
diff --git a/tests/test_litellm/llms/tinyfish/test_tinyfish_search.py b/tests/test_litellm/llms/tinyfish/test_tinyfish_search.py
index 58363e3baea..2dcccb8ea7e 100644
--- a/tests/test_litellm/llms/tinyfish/test_tinyfish_search.py
+++ b/tests/test_litellm/llms/tinyfish/test_tinyfish_search.py
@@ -47,7 +47,9 @@ def _make_mock_response(
mock = MagicMock()
mock.status_code = status_code
- mock.headers = headers or {}
+ # httpx.Headers normalizes keys to lowercase — mirror production so tests
+ # assert what callers actually see.
+ mock.headers = httpx.Headers(headers or {})
if json_data is not None:
mock.json.return_value = json_data
mock.text = text if text is not None else _json.dumps(json_data)
@@ -222,7 +224,7 @@ class TestTransformSearchRequest:
assert param not in result["_tinyfish_params"]
def test_arbitrary_param_passed_through(self):
- # `fetch` is a TinyFish-specific param (JSON-encoded tf-fetch config).
+ # `fetch` is a TinyFish-specific param (JSON-encoded fetch config).
# The passthrough loop should forward it verbatim without LiteLLM needing
# to know about it.
config = TinyfishSearchConfig()
@@ -237,26 +239,49 @@ class TestTransformSearchRequest:
config = TinyfishSearchConfig()
result = config.transform_search_request(
query="test",
- optional_params={"fetch": {"format": "html", "fetch_path": "fast"}},
- )
- assert (
- result["_tinyfish_params"]["fetch"]
- == '{"format":"html","fetch_path":"fast"}'
+ optional_params={"fetch": {"format": "html"}},
)
+ assert result["_tinyfish_params"]["fetch"] == '{"format":"html"}'
def test_bool_param_serialized_as_lowercase(self):
- # urlencode renders Python bool as capitalized "True"/"False"; ux-labs
- # rejects those (e.g. include_thumbnail must be literal "true"/"false").
- # Normalize before passing through.
+ # urlencode renders Python bool as capitalized "True"/"False"; TinyFish
+ # Search's bool params require lowercase "true"/"false" strings on the
+ # wire. Normalize before passing through.
config = TinyfishSearchConfig()
true_result = config.transform_search_request(
- query="test", optional_params={"include_thumbnail": True}
+ query="test", optional_params={"some_bool_param": True}
)
false_result = config.transform_search_request(
- query="test", optional_params={"include_thumbnail": False}
+ query="test", optional_params={"some_bool_param": False}
)
- assert true_result["_tinyfish_params"]["include_thumbnail"] == "true"
- assert false_result["_tinyfish_params"]["include_thumbnail"] == "false"
+ assert true_result["_tinyfish_params"]["some_bool_param"] == "true"
+ assert false_result["_tinyfish_params"]["some_bool_param"] == "false"
+
+ def test_float_param_passes_through(self):
+ # Float values pass the urlencode adapter and land on the wire as
+ # their decimal string form. If TinyFish's server rejects a float
+ # for a param it expects as int, the server's 400 response is
+ # attributed via _wrap_error (`TinyFish Search: ...`) — better than
+ # a client-side pydantic ValidationError with no context.
+ config = TinyfishSearchConfig()
+ result = config.transform_search_request(
+ query="test",
+ optional_params={"some_float_param": 0.5},
+ )
+ assert result["_tinyfish_params"]["some_float_param"] == 0.5
+
+ def test_list_param_auto_json_encoded(self):
+ # TinyFish Search's JSON-array params arrive on the wire as JSON-
+ # encoded strings. Accept the natural Python list form and serialize
+ # so the caller doesn't have to pre-stringify. Params whose wire
+ # format is a plain comma-separated string are the caller's
+ # responsibility to pass as a Python str.
+ config = TinyfishSearchConfig()
+ result = config.transform_search_request(
+ query="test",
+ optional_params={"some_list_param": ["a.example", "b.example"]},
+ )
+ assert result["_tinyfish_params"]["some_list_param"] == '["a.example","b.example"]'
def test_pre_stringified_param_passed_unchanged(self):
# If the caller already JSON-encoded, don't re-encode.
@@ -422,10 +447,104 @@ class TestTransformSearchResponse:
assert getattr(first, "position", None) == 1
assert getattr(first, "site_name", None) == "tinyfish.ai"
+ def test_top_level_extras_flow_through(self):
+ # TinyFish returns `query`, `total_results`, `page` at the envelope
+ # level. These must ride through to the caller via SearchResponse's
+ # extra="allow" so pagination logic, echo checks, etc. work.
+ config = TinyfishSearchConfig()
+ mock_response = _make_mock_response(MOCK_TINYFISH_RESPONSE)
+ result = config.transform_search_response(
+ raw_response=mock_response, logging_obj=None
+ )
+ assert getattr(result, "query", None) == "web automation tools"
+ assert getattr(result, "total_results", None) == 2
+ assert getattr(result, "page", None) == 0
+
+ def test_top_level_future_extras_flow_through(self):
+ # Any future TinyFish top-level field must ride through unchanged
+ # (design contract: no LiteLLM code change needed for new fields).
+ config = TinyfishSearchConfig()
+ body = {
+ "results": [
+ {"title": "x", "url": "https://x", "snippet": "x"},
+ ],
+ "query": "test",
+ "example_int_extra": 123, # hypothetical future field
+ "example_str_extra": "value", # hypothetical future field
+ "example_id_extra": "abc-def", # hypothetical future field
+ }
+ result = config.transform_search_response(
+ raw_response=_make_mock_response(body), logging_obj=None
+ )
+ assert getattr(result, "example_int_extra", None) == 123
+ assert getattr(result, "example_str_extra", None) == "value"
+ assert getattr(result, "example_id_extra", None) == "abc-def"
+
+ def test_response_headers_stashed_on_hidden_params(self):
+ # TinyFish Search sets X-Request-ID on every success response. Confirm it
+ # lands on both `_hidden_params["headers"]` (raw) and
+ # `_hidden_params["additional_headers"]` (sanitized/prefixed).
+ # httpx.Headers lowercases every key, so assertions use lowercase.
+ config = TinyfishSearchConfig()
+ mock_response = _make_mock_response(
+ MOCK_TINYFISH_RESPONSE,
+ headers={"X-Request-ID": "req-abc-123", "Content-Type": "application/json"},
+ )
+ result = config.transform_search_response(
+ raw_response=mock_response, logging_obj=None
+ )
+ # Raw copy — httpx has normalized keys to lowercase.
+ assert result._hidden_params["headers"]["x-request-id"] == "req-abc-123"
+ # process_response_headers prefixes non-OpenAI-standard keys with "llm_provider-".
+ assert result._hidden_params["additional_headers"]["llm_provider-x-request-id"] == "req-abc-123"
+
+ def test_response_headers_future_headers_flow_through(self):
+ # "Accept extra": any header TinyFish Search adds later must ride
+ # through without a LiteLLM code change.
+ config = TinyfishSearchConfig()
+ mock_response = _make_mock_response(
+ MOCK_TINYFISH_RESPONSE,
+ headers={
+ "X-Request-ID": "req-1",
+ "X-Example-Header-A": "value-a", # hypothetical future header
+ "X-Example-Header-B": "value-b", # hypothetical future header
+ },
+ )
+ result = config.transform_search_response(
+ raw_response=mock_response, logging_obj=None
+ )
+ raw = result._hidden_params["headers"]
+ # httpx lowercases header names on read.
+ assert raw["x-example-header-a"] == "value-a"
+ assert raw["x-example-header-b"] == "value-b"
+
+ def test_response_headers_strips_x_litellm_spoof(self):
+ # A provider setting `x-litellm-*` in its response must not be able to
+ # spoof LiteLLM-internal markers via _hidden_params["additional_headers"].
+ # The raw copy preserves the header (opt-in debug view); the sanitized
+ # copy prefixes it with `llm_provider-` so bare `x-litellm-*` markers
+ # can't be spoofed (values still survive under the prefixed key for
+ # observability).
+ config = TinyfishSearchConfig()
+ mock_response = _make_mock_response(
+ MOCK_TINYFISH_RESPONSE,
+ headers={"x-litellm-attempted-fallbacks": "spoofed", "X-Request-ID": "r1"},
+ )
+ result = config.transform_search_response(
+ raw_response=mock_response, logging_obj=None
+ )
+ # Raw view still has the spoof.
+ assert result._hidden_params["headers"]["x-litellm-attempted-fallbacks"] == "spoofed"
+ # Sanitized view: the spoof survives only under the llm_provider- prefix
+ # (never under the bare x-litellm-* key that LiteLLM downstream trusts).
+ additional = result._hidden_params["additional_headers"]
+ assert "x-litellm-attempted-fallbacks" not in additional
+ assert additional.get("llm_provider-x-litellm-attempted-fallbacks") == "spoofed"
+
def test_fetch_field_rides_through_to_search_result(self):
- # Mirrors browser-search's per-result `fetch` nested object (see
- # api/src/parser.rs SearchResult.fetch). Confirms `fetch=...` requests
- # surface their content to LiteLLM callers without provider changes.
+ # Mirrors TinyFish Search's per-result `fetch` nested object.
+ # Confirms `fetch=...` requests surface their content to LiteLLM
+ # callers without provider changes.
config = TinyfishSearchConfig()
fetched = {
"results": [
@@ -568,7 +687,7 @@ class TestTransformSearchResponse:
class TestErrorHandling:
def test_4xx_response_raises_with_attribution_and_unwrapped_message(self):
- # Reproduces ux-labs' error envelope shape for an INVALID_INPUT response.
+ # Reproduces TinyFish Search's error envelope shape for an INVALID_INPUT response.
config = TinyfishSearchConfig()
body = {
"error": {
@@ -590,7 +709,7 @@ class TestErrorHandling:
def test_429_preserves_status_code_and_headers(self):
config = TinyfishSearchConfig()
- body = {"error": {"code": "RATE_LIMIT_EXCEEDED", "message": "60 rpm"}}
+ body = {"error": {"code": "RATE_LIMIT_EXCEEDED", "message": "rate limit exceeded"}}
mock_response = _make_mock_response(
body, status_code=429, headers={"Retry-After": "60"}
)
@@ -600,10 +719,12 @@ class TestErrorHandling:
)
assert getattr(exc_info.value, "status_code", None) == 429
headers = getattr(exc_info.value, "headers", {}) or {}
- assert headers.get("Retry-After") == "60"
+ # httpx lowercases; the exception carries the same dict shape.
+ assert headers.get("retry-after") == "60"
- def test_5xx_with_non_ux_labs_body_falls_back_to_raw_text(self):
- # Cloudflare-style JSON or any other envelope: unwrap fails, fall back to raw.
+ def test_5xx_with_non_tinyfish_envelope_shape_falls_back_to_raw_text(self):
+ # A JSON body that doesn't match TinyFish Search's error envelope shape:
+ # unwrap fails, fall back to the raw body text.
config = TinyfishSearchConfig()
body = {"errors": [{"code": "10000", "message": "Internal"}]}
mock_response = _make_mock_response(body, status_code=502)
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py
index 3392203dbab..051df30dfcb 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py
@@ -6406,7 +6406,7 @@ async def test_execute_mcp_tool_rest_server_id_injects_requested_server_credenti
with (
patch.dict(
mcp_module.global_mcp_server_manager.tool_name_to_mcp_server_name_mapping,
- {"echo": collision_server.name},
+ {"echo": collision_server.name, "echo_requested-echo": requested_server.name},
),
patch.object(
mcp_module.global_mcp_server_manager,
@@ -8430,3 +8430,72 @@ class TestListFiltersHonorThePrefixBoundary:
assert listed == callable_, f"grants={grants!r} listed={listed} callable={callable_}"
assert listed is expected, f"grants={grants!r} expected={expected} got={listed}"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "auth_type",
+ [MCPAuth.authorization, MCPAuth.bearer_token, MCPAuth.api_key, MCPAuth.basic, MCPAuth.token],
+)
+async def test_list_tools_injects_byok_credential_for_non_oauth2_auth_types(auth_type):
+ """Regression for BYOK servers on a non-oauth2 auth_type: the stored per-user credential must be
+ attached when listing tools, otherwise the upstream 401 is absorbed and the server lists nothing."""
+ try:
+ from litellm.proxy._experimental.mcp_server.server import (
+ _get_tools_from_mcp_servers,
+ set_auth_context,
+ )
+ except ImportError:
+ pytest.skip("MCP server not available")
+
+ user_api_key_auth = UserAPIKeyAuth(api_key="test_key", user_id="byok_user")
+ set_auth_context(user_api_key_auth)
+
+ server = MagicMock()
+ server.server_id = "byok_server"
+ server.name = "byok"
+ server.alias = "byok"
+ server.server_name = "byok"
+ server.auth_type = auth_type
+ server.is_byok = True
+ server.allowed_tools = None
+ server.disallowed_tools = None
+ server.extra_headers = None
+ server.tool_name_to_display_name = None
+ server.tool_name_to_description = None
+
+ seen_auth_headers = []
+
+ async def mock_get_tools_from_server(server, mcp_auth_header=None, add_prefix=False, **kwargs):
+ seen_auth_headers.append(mcp_auth_header)
+ tool = MagicMock()
+ tool.name = f"{server.alias}-toolA" if add_prefix else "toolA"
+ tool.description = "desc"
+ tool.inputSchema = {}
+ return [tool]
+
+ mock_manager = MagicMock()
+ mock_manager.get_allowed_mcp_servers = AsyncMock(return_value=[server.server_id])
+ mock_manager.get_mcp_server_by_id = MagicMock(return_value=server)
+ mock_manager.filter_server_ids_by_ip_with_info = lambda server_ids, client_ip: (server_ids, 0)
+ mock_manager._get_tools_from_server = mock_get_tools_from_server
+
+ with (
+ patch(
+ "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
+ mock_manager,
+ ),
+ patch(
+ "litellm.proxy._experimental.mcp_server.server._get_byok_credential",
+ AsyncMock(return_value="personal-api-key"),
+ ),
+ ):
+ listing = await _get_tools_from_mcp_servers(
+ user_api_key_auth=user_api_key_auth,
+ mcp_auth_header=None,
+ mcp_servers=None,
+ mcp_server_auth_headers=None,
+ )
+
+ assert seen_auth_headers == ["personal-api-key"]
+ assert [tool.name for tool in listing.tools] == ["byok-toolA"]
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
index cd1ef7320dd..9064312fd6d 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
@@ -42,6 +42,7 @@ from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
_deserialize_json_list,
_normalize_mcp_server_cost_info,
_obo_retry_applies,
+ _resolve_openapi_tool_auth,
_should_strip_caller_authorization,
_without_authorization,
)
@@ -4833,6 +4834,74 @@ class TestMCPServerManager:
with pytest.raises(ValueError, match="Tool missing_tool not found"):
manager._resolve_mcp_server_for_tool_call("github", "missing_tool")
+ @staticmethod
+ def _manager_with_deepwiki_and_huggingface() -> MCPServerManager:
+ manager = MCPServerManager()
+ deepwiki = MCPServer(server_id="deepwiki-id", name="deepwiki", server_name="deepwiki", transport=MCPTransport.http)
+ huggingface = MCPServer(
+ server_id="huggingface-id", name="huggingface", server_name="huggingface", transport=MCPTransport.http
+ )
+ manager.registry = {"deepwiki-id": deepwiki, "huggingface-id": huggingface}
+ manager.tool_name_to_mcp_server_name_mapping = {
+ "read_wiki_structure": "deepwiki",
+ "deepwiki-read_wiki_structure": "deepwiki",
+ "hub_repo_search": "huggingface",
+ "huggingface-hub_repo_search": "huggingface",
+ }
+ return manager
+
+ def test_resolve_mcp_server_for_tool_call_rejects_tool_exposed_only_by_another_server(self):
+ manager = self._manager_with_deepwiki_and_huggingface()
+
+ with pytest.raises(ValueError, match="Tool read_wiki_structure not found"):
+ manager._resolve_mcp_server_for_tool_call("huggingface", "read_wiki_structure")
+ with pytest.raises(ValueError, match="Tool hub_repo_search not found"):
+ manager._resolve_mcp_server_for_tool_call("deepwiki", "hub_repo_search")
+
+ assert manager._resolve_mcp_server_for_tool_call("deepwiki", "read_wiki_structure") is manager.registry["deepwiki-id"]
+ assert manager._resolve_mcp_server_for_tool_call("huggingface", "hub_repo_search") is manager.registry["huggingface-id"]
+
+ def test_get_mcp_server_from_tool_name_rejects_other_servers_prefix(self):
+ manager = self._manager_with_deepwiki_and_huggingface()
+
+ assert manager._get_mcp_server_from_tool_name("huggingface-read_wiki_structure") is None
+ assert manager._get_mcp_server_from_tool_name("deepwiki-hub_repo_search") is None
+ assert manager._get_mcp_server_from_tool_name("deepwiki-read_wiki_structure") is manager.registry["deepwiki-id"]
+ assert manager._get_mcp_server_from_tool_name("huggingface-hub_repo_search") is manager.registry["huggingface-id"]
+
+ def test_resolve_mcp_server_for_tool_call_shared_bare_name_resolves_via_own_prefixed_spelling(self):
+ manager = MCPServerManager()
+ zapier = MCPServer(server_id="zapier-id", name="zapier", alias="zapier-alias", transport=MCPTransport.http)
+ other = MCPServer(server_id="other-id", name="other", server_name="other", transport=MCPTransport.http)
+ manager.registry = {"zapier-id": zapier, "other-id": other}
+ manager.tool_name_to_mcp_server_name_mapping = {
+ "create_zap": "other",
+ "other-create_zap": "other",
+ "zapier-alias-create_zap": "zapier-alias",
+ }
+
+ assert manager._resolve_mcp_server_for_tool_call("zapier", "create_zap") is zapier
+ assert manager._resolve_mcp_server_for_tool_call("other", "create_zap") is other
+
+ def test_remove_server_drops_only_its_own_tool_mapping_rows(self):
+ manager = self._manager_with_deepwiki_and_huggingface()
+
+ manager.remove_server(
+ LiteLLM_MCPServerTable(
+ server_id="huggingface-id",
+ alias="huggingface",
+ url="https://huggingface.co/mcp",
+ transport=MCPTransport.http,
+ created_at=datetime.now(),
+ updated_at=datetime.now(),
+ )
+ )
+
+ assert manager.tool_name_to_mcp_server_name_mapping == {
+ "read_wiki_structure": "deepwiki",
+ "deepwiki-read_wiki_structure": "deepwiki",
+ }
+
@pytest.mark.asyncio
async def test_resolve_oauth2_headers_skipped_when_not_user_oauth(self):
"""Returns input headers unchanged when server does not need user OAuth."""
@@ -10457,3 +10526,242 @@ class TestSessionResourceScopeIntersect:
):
fallback = await manager.get_allowed_mcp_servers(auth)
assert fallback == ["granted-id"]
+
+
+class TestClientForwardedDiscoveryFailureIsNotFatal:
+ """A failed OAuth metadata discovery may only brick the flows the gateway runs itself.
+
+ ``true_passthrough`` / ``oauth_delegate`` forward the caller's own bearer and mint nothing, so an
+ upstream that publishes no RFC 9728 metadata (an internal API, or any IdP unreachable from the
+ pod) must still serve sessions instead of 503-ing before the upstream is ever contacted.
+ """
+
+ @staticmethod
+ def _config(auth_type: MCPAuthType, dcr_bridge: bool | None) -> dict[str, dict[str, object]]:
+ entry: Final[dict[str, object]] = {
+ "url": "https://up.example.com/mcp",
+ "transport": MCPTransport.http,
+ "auth_type": auth_type,
+ **({"oauth2_flow": "authorization_code"} if auth_type == MCPAuth.oauth2 else {}),
+ **({"dcr_bridge": dcr_bridge} if dcr_bridge is not None else {}),
+ }
+ return {"upstream": entry}
+
+ async def _registered(self, manager: MCPServerManager, auth_type: MCPAuthType, dcr_bridge: bool | None):
+ with (
+ patch.object(manager, "_discover_oauth_metadata_for_server", new=AsyncMock(return_value=None)),
+ patch.object(manager, "initialize_tool_name_to_mcp_server_name_mapping"),
+ ):
+ await manager.load_servers_from_config(self._config(auth_type, dcr_bridge))
+ return next(iter(manager.config_mcp_servers.values()))
+
+ @pytest.mark.parametrize(
+ "auth_type, dcr_bridge, serves_without_endpoints",
+ [
+ (MCPAuth.true_passthrough, None, True),
+ (MCPAuth.true_passthrough, True, True),
+ (MCPAuth.oauth_delegate, None, True),
+ (MCPAuth.oauth_delegate, True, True),
+ (MCPAuth.oauth2, None, False),
+ (MCPAuth.oauth2_token_exchange, None, False),
+ ],
+ )
+ @pytest.mark.parametrize("failure", ["incomplete", "timed_out"])
+ @pytest.mark.asyncio
+ async def test_discovery_failure_blocks_only_gateway_run_flows(
+ self,
+ auth_type: MCPAuthType,
+ dcr_bridge: bool | None,
+ serves_without_endpoints: bool,
+ failure: str,
+ ):
+ manager = MCPServerManager()
+ server = await self._registered(manager, auth_type, dcr_bridge)
+ manager._set_oauth_discovery_deferred(server.server_id, True)
+
+ async def never_returns(_server):
+ await asyncio.Future()
+
+ discovery_patch: Final = (
+ {"new": AsyncMock(return_value=None)} if failure == "incomplete" else {"side_effect": never_returns}
+ )
+ with (
+ patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.MCP_METADATA_TIMEOUT", 0.01),
+ patch.object(manager, "_discover_oauth_metadata_for_server", **discovery_patch),
+ ):
+ if not serves_without_endpoints:
+ with pytest.raises(HTTPException) as exc:
+ await manager.ensure_oauth_metadata_discovered(server)
+ assert exc.value.status_code == 503
+ return
+
+ resolved = await manager.ensure_oauth_metadata_discovered(server)
+
+ assert resolved is manager.config_mcp_servers[server.server_id]
+ assert resolved.authorization_url is None
+ assert resolved.token_url is None
+ assert manager._oauth_discovery_slot(server.server_id) is not None
+
+ @pytest.mark.parametrize(
+ "auth_type, serves_the_listing",
+ [(MCPAuth.true_passthrough, True), (MCPAuth.oauth_delegate, True), (MCPAuth.oauth2, False)],
+ )
+ @pytest.mark.asyncio
+ async def test_listing_leg_serves_a_forwarding_server_whose_discovery_failed(
+ self, auth_type: MCPAuthType, serves_the_listing: bool
+ ):
+ """The listing leg is where the 503 became an empty tool list, so pin the fix there too.
+
+ ``_get_tools_from_server`` is the per-server leg the aggregate absorbs: a failure here is what
+ the fan-out turns into HTTP 200 with ``tools: []``, which is why the outage carried no
+ diagnostic. A forwarding server must now reach its upstream, and a gateway-run flow must still
+ surface the fault rather than be silently listed as empty.
+ """
+ manager = MCPServerManager()
+ server = await self._registered(manager, auth_type, None)
+ manager._set_oauth_discovery_deferred(server.server_id, True)
+ manager._fetch_tools_with_timeout = AsyncMock(
+ return_value=[MCPTool(name="list_reports", description="d", inputSchema={"type": "object"})]
+ )
+
+ with patch.object(manager, "_discover_oauth_metadata_for_server", new=AsyncMock(return_value=None)):
+ if not serves_the_listing:
+ with pytest.raises(MCPServerListError):
+ await manager._get_tools_from_server(server=server)
+ return
+ tools = await manager._get_tools_from_server(server=server)
+
+ assert [tool.name for tool in tools] == ["upstream-list_reports"]
+ manager._fetch_tools_with_timeout.assert_awaited_once()
+
+ @pytest.mark.parametrize("auth_type", [MCPAuth.true_passthrough, MCPAuth.oauth_delegate])
+ @pytest.mark.asyncio
+ async def test_client_forwarded_servers_keep_discovering_their_front_door_endpoints(
+ self, auth_type: MCPAuthType
+ ):
+ """Exempting these modes from the FAILURE must not exempt them from discovery itself.
+
+ ``/authorize``, ``/token`` and ``/register`` read the discovered endpoints for these servers
+ (``_resolve_ephemeral_dcr_client`` mints for ``true_passthrough`` whatever ``dcr_bridge``
+ says), so an exemption written into the unresolved-endpoints predicate would disarm the slot
+ and silently drop a working front door.
+ """
+ manager = MCPServerManager()
+ metadata: Final = MCPOAuthMetadata(
+ authorization_url="https://idp.example.com/authorize",
+ token_url="https://idp.example.com/token",
+ registration_url="https://idp.example.com/register",
+ )
+ with (
+ patch.object(manager, "_discover_oauth_metadata_for_server", new=AsyncMock(return_value=metadata)),
+ patch.object(manager, "initialize_tool_name_to_mcp_server_name_mapping"),
+ ):
+ await manager.load_servers_from_config(self._config(auth_type, None))
+ server = next(iter(manager.config_mcp_servers.values()))
+ resolved = await manager.ensure_oauth_metadata_discovered(server)
+
+ assert resolved.authorization_url == "https://idp.example.com/authorize"
+ assert resolved.token_url == "https://idp.example.com/token"
+ assert resolved.registration_url == "https://idp.example.com/register"
+ assert manager.config_mcp_servers[server.server_id].authorization_url == "https://idp.example.com/authorize"
+ assert manager._oauth_discovery_slot(server.server_id) is None
+
+
+class TestResolveOpenapiToolAuth:
+ """The credential matrix for a ``spec_path`` server's two OpenAPI dispatch arms.
+
+ A per-server ``x-mcp-{alias}-authorization`` is already a complete header value and is forwarded
+ verbatim; a BYOK credential is a raw secret that takes the auth-type prefix. Conflating them
+ ships ``Bearer Bearer ``, so every cell pins which of the two a value came from.
+ """
+
+ @staticmethod
+ def _server(auth_type: MCPAuthType = MCPAuth.oauth_delegate) -> MCPServer:
+ return MCPServer(
+ server_id="srv-openapi",
+ name="report_api",
+ server_name="report_api",
+ alias="report_api",
+ url="https://api.internal.example.com",
+ transport=MCPTransport.http,
+ auth_type=auth_type,
+ spec_path="https://api.internal.example.com/openapi.json",
+ extra_headers=["X-Tenant"],
+ )
+
+ @pytest.mark.parametrize(
+ "per_server, byok, expected_auth, expected_extra_keys, expected_credential",
+ [
+ (
+ {"report_api": "Bearer caller-token"},
+ None,
+ "Bearer caller-token",
+ {"X-Tenant"},
+ "Bearer caller-token",
+ ),
+ (
+ {"report_api": {"Authorization": "Bearer caller-token", "X-Trace": "abc"}},
+ None,
+ "Bearer caller-token",
+ {"X-Tenant", "X-Trace"},
+ {"Authorization": "Bearer caller-token", "X-Trace": "abc"},
+ ),
+ (
+ {"report_api": {"X-Api-Key": "k1"}},
+ "byok-secret",
+ "Bearer byok-secret",
+ {"X-Tenant", "X-Api-Key"},
+ "byok-secret",
+ ),
+ ({"report_api": {"X-Api-Key": "k1"}}, None, None, {"X-Tenant", "X-Api-Key"}, None),
+ (None, "byok-secret", "Bearer byok-secret", {"X-Tenant"}, "byok-secret"),
+ (None, None, None, {"X-Tenant"}, None),
+ ({"other_server": "Bearer wrong"}, None, None, {"X-Tenant"}, None),
+ ],
+ )
+ def test_credential_matrix(
+ self,
+ per_server: dict | None,
+ byok: str | None,
+ expected_auth: str | None,
+ expected_extra_keys: set,
+ expected_credential: object,
+ ):
+ auth_value, forwarded, credential = _resolve_openapi_tool_auth(
+ mcp_server=self._server(),
+ mcp_auth_header=byok,
+ mcp_server_auth_headers=per_server,
+ raw_headers={"x-tenant": "acme", "authorization": "Bearer admission-key"},
+ user_api_key_auth=None,
+ )
+
+ assert auth_value == expected_auth
+ assert set(forwarded or {}) == expected_extra_keys
+ assert credential == expected_credential
+
+ def test_per_server_value_is_never_re_prefixed(self):
+ """The regression that a naive wiring produces: the caller already sent ``Bearer ``."""
+ auth_value, _, credential = _resolve_openapi_tool_auth(
+ mcp_server=self._server(auth_type=MCPAuth.api_key),
+ mcp_auth_header="byok-secret",
+ mcp_server_auth_headers={"report_api": "Bearer caller-token"},
+ raw_headers=None,
+ user_api_key_auth=None,
+ )
+
+ assert auth_value == "Bearer caller-token"
+ assert credential == "Bearer caller-token"
+ assert not auth_value.startswith("ApiKey ")
+
+ def test_per_server_authorization_is_not_also_left_in_forwarded_headers(self):
+ """``resolve_openapi_upstream_auth`` pops Authorization out of the forwarded headers, so a
+ second copy there would give the passthrough arm two sources to reconcile."""
+ _, forwarded, _ = _resolve_openapi_tool_auth(
+ mcp_server=self._server(),
+ mcp_auth_header=None,
+ mcp_server_auth_headers={"report_api": {"Authorization": "Bearer caller-token"}},
+ raw_headers={"x-tenant": "acme"},
+ user_api_key_auth=None,
+ )
+
+ assert "Authorization" not in (forwarded or {})
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py
index 63473bf3cf5..7bd846aeda4 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py
@@ -189,9 +189,11 @@ async def test_openapi_local_tool_denied_when_server_not_resolvable():
pre_call = AsyncMock(return_value={})
handle_local = AsyncMock(return_value=[])
+ resolve_auth = MagicMock()
# `_get_mcp_server_from_tool_name` returns None — no server context.
with (
+ patch.object(mcp_module, "_resolve_openapi_tool_auth", new=resolve_auth),
patch.object(
mcp_module.global_mcp_server_manager,
"_get_mcp_server_from_tool_name",
@@ -228,6 +230,9 @@ async def test_openapi_local_tool_denied_when_server_not_resolvable():
assert exc.value.status_code == 503
pre_call.assert_not_awaited()
handle_local.assert_not_awaited()
+ # The credential resolver takes a non-optional server, so the 503 guard above it is what keeps
+ # a missing server from ever reaching it. Pinned here so moving the guard reds this test.
+ resolve_auth.assert_not_called()
@pytest.mark.asyncio
@@ -549,3 +554,101 @@ async def test_unknown_tool_name_still_reports_not_found():
assert exc.value.status_code == 404
assert "not found" in str(exc.value.detail)
+
+
+OPENAPI_PER_SERVER_TOKEN = "Bearer per-server-upstream-token"
+
+
+def _spec_path_server() -> MCPServer:
+ return MCPServer(
+ server_id="srv-reports",
+ name="report_api",
+ server_name="report_api",
+ alias="report_api",
+ url="https://api.internal.example.com",
+ transport=MCPTransport.http,
+ auth_type=MCPAuth.oauth_delegate,
+ spec_path="https://api.internal.example.com/openapi.json",
+ )
+
+
+@pytest.mark.parametrize("dispatch_arm", ["local_registry", "call_tool"])
+@pytest.mark.asyncio
+async def test_per_server_auth_header_reaches_both_openapi_dispatch_arms(dispatch_arm: str):
+ """`x-mcp-{alias}-authorization` must survive on BOTH OpenAPI dispatch arms.
+
+ OpenAPI tools live in the local tool registry, so `execute_mcp_tool` serves MCP-protocol and REST
+ tool calls while `MCPServerManager.call_tool` serves the responses-API handler. Both arms sourced
+ the upstream credential only from the deprecated global / BYOK `mcp_auth_header`, so the
+ per-server header was dropped and the upstream API saw no Authorization at all.
+
+ Asserting on the resolver kwarg as well as the ContextVar is deliberate: for the client-forwarded
+ modes the credential has to reach `resolve_openapi_upstream_auth`, whose passthrough arm outranks
+ the ContextVar when it materializes a header.
+ """
+ from litellm.proxy._experimental.mcp_server import server as mcp_module
+ from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import (
+ _request_auth_header,
+ )
+
+ server = _spec_path_server()
+ auth_headers = {"report_api": {"Authorization": OPENAPI_PER_SERVER_TOKEN}}
+ user = UserAPIKeyAuth(api_key="sk-user", user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER.value)
+ captured: dict = {}
+
+ async def fake_resolver(**kwargs):
+ captured["resolver_credential"] = kwargs["mcp_auth_header"]
+ return None, kwargs["forwarded_headers"]
+
+ async def capture_local(_name, _arguments):
+ captured["injected"] = _request_auth_header.get()
+ return []
+
+ async def capture_openapi_handler(_server, _name, _arguments):
+ captured["injected"] = _request_auth_header.get()
+ return []
+
+ manager = mcp_module.global_mcp_server_manager
+ with (
+ patch.object(manager, "resolve_openapi_upstream_auth", new=fake_resolver),
+ patch.object(manager, "pre_call_tool_check", new=AsyncMock(return_value={})),
+ ):
+ if dispatch_arm == "local_registry":
+ fake_tool = MagicMock()
+ fake_tool.name = "list_reports"
+ with (
+ patch.object(manager, "_get_mcp_server_from_tool_name", return_value=server),
+ patch.object(mcp_module.global_mcp_tool_registry, "get_tool", return_value=fake_tool),
+ patch(
+ "litellm.proxy._experimental.mcp_server.server._handle_local_mcp_tool",
+ new=capture_local,
+ ),
+ patch(
+ "litellm.proxy._experimental.mcp_server.server.MCPRequestHandler.is_tool_allowed",
+ return_value=True,
+ ),
+ ):
+ await mcp_module.execute_mcp_tool(
+ name="list_reports",
+ arguments={},
+ allowed_mcp_servers=[server],
+ start_time=datetime.now(timezone.utc),
+ user_api_key_auth=user,
+ mcp_server_auth_headers=auth_headers,
+ )
+ else:
+ with (
+ patch.object(manager, "_resolve_mcp_server_for_tool_call", return_value=server),
+ patch.object(manager, "_call_openapi_tool_handler", new=capture_openapi_handler),
+ ):
+ await manager.call_tool(
+ server_name="report_api",
+ name="list_reports",
+ arguments={},
+ user_api_key_auth=user,
+ mcp_server_auth_headers=auth_headers,
+ )
+
+ assert captured["resolver_credential"] == {"Authorization": OPENAPI_PER_SERVER_TOKEN}
+ assert captured["injected"] == OPENAPI_PER_SERVER_TOKEN
+ assert _request_auth_header.get() is None
diff --git a/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py b/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py
index 0a427df0cb7..9a90daeccb7 100644
--- a/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py
+++ b/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py
@@ -164,6 +164,41 @@ class TestProxyExceptionPassthrough:
mock_logging.post_call_failure_hook.assert_awaited_once()
+class TestFailureHookRequestData:
+ @pytest.mark.asyncio
+ async def test_failure_hook_gets_post_setup_data_with_logging_obj(self):
+ """Request setup replaces the processor's data dict (adding the logging
+ object the failure hook needs to lift token usage from); the exception
+ handler must pass that replaced dict, not the raw request body dict."""
+ import litellm.proxy.anthropic_endpoints.endpoints as ep
+ import litellm.proxy.proxy_server as proxy_server
+ from litellm.proxy._types import ProxyException, UserAPIKeyAuth
+
+ captured = {}
+
+ async def fake_process(self, **kwargs):
+ self.data = {**self.data, "litellm_logging_obj": "logging-obj-sentinel"}
+ captured["processor_data"] = self.data
+ raise RuntimeError("provider timeout")
+
+ with (
+ patch.object(ep, "_read_request_body", new=AsyncMock(return_value={"model": "claude-sonnet"})),
+ patch.object(ep.ProxyBaseLLMRequestProcessing, "base_process_llm_request", new=fake_process),
+ patch.object(proxy_server, "proxy_logging_obj") as mock_logging,
+ ):
+ mock_logging.post_call_failure_hook = AsyncMock()
+ with pytest.raises(ProxyException):
+ await ep.anthropic_response(
+ fastapi_response=MagicMock(),
+ request=MagicMock(),
+ user_api_key_dict=UserAPIKeyAuth(),
+ )
+
+ hook_request_data = mock_logging.post_call_failure_hook.await_args.kwargs["request_data"]
+ assert hook_request_data is captured["processor_data"]
+ assert hook_request_data["litellm_logging_obj"] == "logging-obj-sentinel"
+
+
class TestEventLoggingBatchEndpoint:
"""Test the stubbed event logging batch endpoint"""
diff --git a/tests/test_litellm/proxy/common_utils/test_registry_read_through.py b/tests/test_litellm/proxy/common_utils/test_registry_read_through.py
new file mode 100644
index 00000000000..f0fbdea4e85
--- /dev/null
+++ b/tests/test_litellm/proxy/common_utils/test_registry_read_through.py
@@ -0,0 +1,476 @@
+import asyncio
+from typing import Final
+
+import pytest
+
+from litellm.proxy.common_utils.registry_read_through import RegistryReadThrough
+
+
+class ResyncSpy:
+ def __init__(self, found: bool = True, error: Exception | None = None) -> None:
+ self.found = found
+ self.error = error
+ self.calls: list[str] = []
+
+ async def __call__(self, key: str) -> bool:
+ self.calls.append(key)
+ if self.error is not None:
+ raise self.error
+ return self.found
+
+
+@pytest.mark.asyncio
+async def test_attempt_returns_true_when_resync_finds_object():
+ spy: Final = ResyncSpy(found=True)
+ read_through: Final = RegistryReadThrough(resync=spy)
+
+ assert await read_through.attempt("new-model") is True
+ assert spy.calls == ["new-model"]
+
+
+@pytest.mark.asyncio
+async def test_attempt_found_key_is_not_negative_cached():
+ spy: Final = ResyncSpy(found=True)
+ read_through: Final = RegistryReadThrough(resync=spy)
+
+ assert await read_through.attempt("new-model") is True
+ assert await read_through.attempt("new-model") is True
+ assert spy.calls == ["new-model", "new-model"]
+
+
+@pytest.mark.asyncio
+async def test_missing_key_is_negative_cached_within_ttl():
+ spy: Final = ResyncSpy(found=False)
+ read_through: Final = RegistryReadThrough(resync=spy, miss_ttl_seconds=60.0)
+
+ assert await read_through.attempt("ghost-model") is False
+ assert await read_through.attempt("ghost-model") is False
+ assert spy.calls == ["ghost-model"]
+
+
+@pytest.mark.asyncio
+async def test_negative_cache_expires_and_resync_runs_again():
+ spy: Final = ResyncSpy(found=False)
+ read_through: Final = RegistryReadThrough(resync=spy, miss_ttl_seconds=0.05)
+
+ assert await read_through.attempt("ghost-model") is False
+ await asyncio.sleep(0.1)
+ assert await read_through.attempt("ghost-model") is False
+ assert spy.calls == ["ghost-model", "ghost-model"]
+
+
+@pytest.mark.asyncio
+async def test_resync_exception_returns_false_without_negative_caching():
+ spy: Final = ResyncSpy(error=RuntimeError("db down"))
+ read_through: Final = RegistryReadThrough(resync=spy)
+
+ assert await read_through.attempt("new-model") is False
+ assert await read_through.attempt("new-model") is False
+ assert spy.calls == ["new-model", "new-model"]
+
+
+@pytest.mark.asyncio
+async def test_concurrent_attempts_for_missing_key_resync_once():
+ class SlowResyncSpy(ResyncSpy):
+ async def __call__(self, key: str) -> bool:
+ await asyncio.sleep(0.05)
+ return await super().__call__(key)
+
+ spy: Final = SlowResyncSpy(found=False)
+ read_through: Final = RegistryReadThrough(resync=spy, miss_ttl_seconds=60.0)
+
+ results: Final = await asyncio.gather(*(read_through.attempt("ghost-model") for _ in range(5)))
+ assert results == [False] * 5
+ assert spy.calls == ["ghost-model"]
+
+
+@pytest.mark.asyncio
+async def test_distinct_keys_do_not_share_negative_cache():
+ spy: Final = ResyncSpy(found=False)
+ read_through: Final = RegistryReadThrough(resync=spy, miss_ttl_seconds=60.0)
+
+ assert await read_through.attempt("ghost-a") is False
+ assert await read_through.attempt("ghost-b") is False
+ assert spy.calls == ["ghost-a", "ghost-b"]
+
+
+@pytest.mark.asyncio
+async def test_resync_budget_exhausted_blocks_resync_without_negative_caching():
+ spy: Final = ResyncSpy(found=False)
+ read_through: Final = RegistryReadThrough(
+ resync=spy, miss_ttl_seconds=60.0, max_resyncs_per_window=2, resync_window_seconds=60.0
+ )
+
+ assert await read_through.attempt("ghost-a") is False
+ assert await read_through.attempt("ghost-b") is False
+ assert await read_through.attempt("ghost-c") is False
+ assert spy.calls == ["ghost-a", "ghost-b"]
+ assert read_through._recent_misses.get_cache("ghost-c") is None
+
+
+@pytest.mark.asyncio
+async def test_resync_budget_replenishes_after_window():
+ spy: Final = ResyncSpy(found=True)
+ read_through: Final = RegistryReadThrough(resync=spy, max_resyncs_per_window=1, resync_window_seconds=0.05)
+
+ assert await read_through.attempt("model-a") is True
+ assert await read_through.attempt("model-b") is False
+ await asyncio.sleep(0.1)
+ assert await read_through.attempt("model-b") is True
+ assert spy.calls == ["model-a", "model-b"]
+
+
+class FakeAgentRow:
+ def __init__(self, agent_id: str, agent_name: str) -> None:
+ self.agent_id = agent_id
+ self.agent_name = agent_name
+ self.object_permission = None
+ self.spend = 0.0
+
+ def model_dump(self):
+ return {
+ "agent_id": self.agent_id,
+ "agent_name": self.agent_name,
+ "agent_card_params": {"name": self.agent_name, "url": "http://db-agent"},
+ "litellm_params": {},
+ "object_permission": None,
+ "spend": self.spend,
+ }
+
+
+@pytest.fixture
+def clean_agent_registry():
+ from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
+
+ original_agents: Final = list(global_agent_registry.agent_list)
+ original_config_agents: Final = getattr(global_agent_registry, "config_agents", ())
+ global_agent_registry.agent_list = []
+ global_agent_registry.config_agents = ()
+ try:
+ yield global_agent_registry
+ finally:
+ global_agent_registry.agent_list = original_agents
+ global_agent_registry.config_agents = original_config_agents
+
+
+@pytest.mark.asyncio
+async def test_get_agent_with_read_through_recovers_agent_created_on_sibling_replica(
+ clean_agent_registry, monkeypatch
+):
+ from unittest.mock import AsyncMock, MagicMock
+
+ import litellm.proxy.proxy_server as proxy_server
+ from litellm.proxy.common_utils.registry_read_through import get_agent_with_read_through
+
+ agent_id: Final = "read-through-db-agent-id"
+ prisma_client: Final = MagicMock()
+ prisma_client.db.litellm_agentstable.find_unique = AsyncMock(
+ return_value=FakeAgentRow(agent_id, "read-through-db-agent")
+ )
+ monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+ monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+
+ assert clean_agent_registry.get_agent_by_id(agent_id=agent_id) is None
+ agent: Final = await get_agent_with_read_through(agent_id)
+
+ assert agent is not None
+ assert agent.agent_id == agent_id
+ prisma_client.db.litellm_agentstable.find_unique.assert_awaited_once_with(
+ where={"agent_id": agent_id},
+ include={"object_permission": True},
+ )
+
+
+@pytest.mark.asyncio
+async def test_get_agent_with_read_through_recovers_agent_by_name(clean_agent_registry, monkeypatch):
+ from unittest.mock import AsyncMock, MagicMock
+
+ import litellm.proxy.proxy_server as proxy_server
+ from litellm.proxy.common_utils.registry_read_through import get_agent_with_read_through
+
+ agent_name: Final = "read-through-db-agent-by-name"
+ prisma_client: Final = MagicMock()
+ prisma_client.db.litellm_agentstable.find_unique = AsyncMock(
+ side_effect=[None, FakeAgentRow("read-through-name-lookup-id", agent_name)]
+ )
+ monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+ monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+
+ agent: Final = await get_agent_with_read_through(agent_name)
+
+ assert agent is not None
+ assert agent.agent_name == agent_name
+ prisma_client.db.litellm_agentstable.find_unique.assert_awaited_with(
+ where={"agent_name": agent_name},
+ include={"object_permission": True},
+ )
+
+
+@pytest.mark.asyncio
+async def test_get_agent_with_read_through_returns_none_for_unknown_agent(clean_agent_registry, monkeypatch):
+ from unittest.mock import AsyncMock, MagicMock
+
+ import litellm.proxy.proxy_server as proxy_server
+ from litellm.proxy.common_utils.registry_read_through import get_agent_with_read_through
+
+ prisma_client: Final = MagicMock()
+ prisma_client.db.litellm_agentstable.find_unique = AsyncMock(return_value=None)
+ monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+ monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+
+ assert await get_agent_with_read_through("agent-nobody-created") is None
+ assert prisma_client.db.litellm_agentstable.find_unique.await_count == 2
+
+
+@pytest.mark.asyncio
+async def test_resync_agents_already_registered_skips_db(clean_agent_registry, monkeypatch):
+ from unittest.mock import AsyncMock, MagicMock
+
+ import litellm.proxy.proxy_server as proxy_server
+ from litellm.proxy.common_utils.registry_read_through import _resync_agents
+
+ agent_id: Final = "read-through-dedup-agent-id"
+ prisma_client: Final = MagicMock()
+ prisma_client.db.litellm_agentstable.find_unique = AsyncMock(
+ return_value=FakeAgentRow(agent_id, "read-through-dedup-agent")
+ )
+ monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+ monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+
+ assert await _resync_agents(agent_id) is True
+ assert await _resync_agents(agent_id) is True
+ assert prisma_client.db.litellm_agentstable.find_unique.await_count == 1
+ assert len(clean_agent_registry.agent_list) == 1
+
+
+class FakeGuardrailRow:
+ def __init__(self, guardrail_id: str, guardrail_name: str) -> None:
+ self.guardrail_id = guardrail_id
+ self.guardrail_name = guardrail_name
+
+ def __iter__(self):
+ return iter(
+ {
+ "guardrail_id": self.guardrail_id,
+ "guardrail_name": self.guardrail_name,
+ "litellm_params": {
+ "guardrail": "litellm_content_filter",
+ "mode": "pre_call",
+ "default_on": True,
+ "blocked_words": [{"keyword": "secret", "action": "BLOCK"}],
+ },
+ "guardrail_info": {},
+ "status": "active",
+ }.items()
+ )
+
+
+@pytest.mark.asyncio
+async def test_get_guardrail_with_read_through_recovers_guardrail_created_on_sibling_replica(monkeypatch):
+ from unittest.mock import AsyncMock, MagicMock
+
+ import litellm.proxy.proxy_server as proxy_server
+ from litellm.proxy.common_utils.registry_read_through import (
+ get_initialized_guardrail_with_read_through,
+ )
+ from litellm.proxy.guardrails.guardrail_registry import IN_MEMORY_GUARDRAIL_HANDLER
+
+ guardrail_id: Final = "read-through-db-guardrail-id"
+ guardrail_name: Final = "read-through-db-guardrail"
+ prisma_client: Final = MagicMock()
+ prisma_client.db.litellm_guardrailstable.find_first = AsyncMock(
+ return_value=FakeGuardrailRow(guardrail_id, guardrail_name)
+ )
+ prisma_client.db.litellm_guardrailstable.find_many = AsyncMock(
+ side_effect=AssertionError("full-table guardrail scan on read-through miss")
+ )
+ monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+ monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+
+ try:
+ guardrail: Final = await get_initialized_guardrail_with_read_through(guardrail_name=guardrail_name)
+ assert guardrail is not None
+ assert guardrail.guardrail_name == guardrail_name
+ prisma_client.db.litellm_guardrailstable.find_first.assert_awaited_once_with(
+ where={"guardrail_name": guardrail_name, "status": "active"}
+ )
+ finally:
+ IN_MEMORY_GUARDRAIL_HANDLER.delete_in_memory_guardrail(guardrail_id)
+
+
+@pytest.mark.asyncio
+async def test_get_guardrail_with_read_through_returns_none_for_unknown_guardrail(monkeypatch):
+ from unittest.mock import AsyncMock, MagicMock
+
+ import litellm.proxy.proxy_server as proxy_server
+ from litellm.proxy.common_utils.registry_read_through import (
+ get_initialized_guardrail_with_read_through,
+ )
+
+ prisma_client: Final = MagicMock()
+ prisma_client.db.litellm_guardrailstable.find_first = AsyncMock(return_value=None)
+ monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+ monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+
+ assert await get_initialized_guardrail_with_read_through(guardrail_name="guardrail-nobody-created") is None
+
+
+@pytest.mark.asyncio
+async def test_resync_guardrails_never_loads_non_active_rows(monkeypatch):
+ from unittest.mock import AsyncMock, MagicMock
+
+ import litellm.proxy.proxy_server as proxy_server
+ from litellm.proxy.common_utils.registry_read_through import _resync_guardrails
+
+ pending_name: Final = "pending-review-guardrail"
+ prisma_client: Final = MagicMock()
+ prisma_client.db.litellm_guardrailstable.find_first = AsyncMock(return_value=None)
+ monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+ monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+
+ assert await _resync_guardrails(pending_name) is False
+ prisma_client.db.litellm_guardrailstable.find_first.assert_awaited_once_with(
+ where={"guardrail_name": pending_name, "status": "active"}
+ )
+
+
+@pytest.mark.asyncio
+async def test_resync_guardrails_syncs_under_guardrail_reconcile_lock(monkeypatch):
+ from unittest.mock import AsyncMock, MagicMock
+
+ import litellm.proxy.common_utils.registry_read_through as read_through_module
+ import litellm.proxy.proxy_server as proxy_server
+ from litellm.proxy.common_utils.registry_read_through import _resync_guardrails
+ from litellm.proxy.guardrails.guardrail_registry import (
+ GUARDRAIL_RECONCILE_LOCK,
+ IN_MEMORY_GUARDRAIL_HANDLER,
+ )
+
+ guardrail_name: Final = "lock-scope-guardrail"
+ prisma_client: Final = MagicMock()
+ prisma_client.db.litellm_guardrailstable.find_first = AsyncMock(
+ return_value=FakeGuardrailRow("lock-scope-guardrail-id", guardrail_name)
+ )
+ lock_states: list[bool] = []
+
+ def record_sync(guardrail) -> None:
+ lock_states.append(GUARDRAIL_RECONCILE_LOCK.locked())
+
+ monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+ monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+ monkeypatch.setattr(IN_MEMORY_GUARDRAIL_HANDLER, "sync_guardrail_from_db", record_sync)
+ monkeypatch.setattr(read_through_module, "_initialized_guardrail", lambda guardrail_name: MagicMock())
+
+ assert await _resync_guardrails(guardrail_name) is True
+ assert lock_states == [True]
+ assert not GUARDRAIL_RECONCILE_LOCK.locked()
+
+
+@pytest.mark.asyncio
+async def test_resync_model_deployments_mutates_router_under_model_reconcile_lock(monkeypatch):
+ from unittest.mock import AsyncMock, MagicMock
+
+ import litellm.proxy.proxy_server as proxy_server
+ from litellm.proxy.common_utils.registry_read_through import _resync_model_deployments
+
+ prisma_client: Final = MagicMock()
+ prisma_client.db.litellm_proxymodeltable.find_many = AsyncMock(return_value=[MagicMock()])
+ router: Final = MagicMock()
+ router.get_model_list.return_value = []
+ lock_states: list[bool] = []
+
+ def record_add_deployment(db_models) -> None:
+ lock_states.append(proxy_server.MODEL_RECONCILE_LOCK.locked())
+
+ monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+ monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+ monkeypatch.setattr(proxy_server, "llm_router", router)
+ monkeypatch.setattr(proxy_server, "llm_model_list", None)
+ monkeypatch.setattr(proxy_server.proxy_config, "_add_deployment", record_add_deployment)
+
+ assert await _resync_model_deployments("lock-scope-model") is True
+ assert lock_states == [True]
+ assert not proxy_server.MODEL_RECONCILE_LOCK.locked()
+
+
+@pytest.mark.asyncio
+async def test_resync_model_deployments_respects_supported_db_objects(monkeypatch):
+ from unittest.mock import AsyncMock, MagicMock
+
+ import litellm.proxy.proxy_server as proxy_server
+ from litellm.proxy.common_utils.registry_read_through import _resync_model_deployments
+
+ prisma_client: Final = MagicMock()
+ prisma_client.db.litellm_proxymodeltable.find_many = AsyncMock(
+ side_effect=AssertionError("db hit for an object type this replica does not load")
+ )
+ monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+ monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+ monkeypatch.setattr(proxy_server, "general_settings", {"supported_db_objects": ["guardrails"]})
+
+ assert await _resync_model_deployments("gated-out-model") is False
+
+
+@pytest.mark.asyncio
+async def test_resync_guardrails_respects_supported_db_objects(monkeypatch):
+ from unittest.mock import AsyncMock, MagicMock
+
+ import litellm.proxy.proxy_server as proxy_server
+ from litellm.proxy.common_utils.registry_read_through import _resync_guardrails
+
+ prisma_client: Final = MagicMock()
+ prisma_client.db.litellm_guardrailstable.find_unique = AsyncMock(
+ side_effect=AssertionError("db hit for an object type this replica does not load")
+ )
+ monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+ monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+ monkeypatch.setattr(proxy_server, "general_settings", {"supported_db_objects": ["models"]})
+
+ assert await _resync_guardrails("gated-out-guardrail") is False
+
+
+@pytest.mark.asyncio
+async def test_resync_agents_respects_supported_db_objects(clean_agent_registry, monkeypatch):
+ from unittest.mock import AsyncMock, MagicMock
+
+ import litellm.proxy.proxy_server as proxy_server
+ from litellm.proxy.common_utils.registry_read_through import _resync_agents
+
+ prisma_client: Final = MagicMock()
+ prisma_client.db.litellm_agentstable.find_unique = AsyncMock(
+ side_effect=AssertionError("db hit for an object type this replica does not load")
+ )
+ monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+ monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+ monkeypatch.setattr(proxy_server, "general_settings", {"supported_db_objects": ["models"]})
+
+ assert await _resync_agents("gated-out-agent") is False
+
+
+@pytest.mark.asyncio
+async def test_resync_agents_waits_for_agent_reload_and_skips_duplicate_registration(clean_agent_registry, monkeypatch):
+ from unittest.mock import AsyncMock, MagicMock
+
+ import litellm.proxy.proxy_server as proxy_server
+ from litellm.proxy.agent_endpoints.agent_registry import AGENT_RECONCILE_LOCK
+ from litellm.proxy.common_utils.registry_read_through import _resync_agents
+ from litellm.types.agents import AgentResponse
+
+ agent_id: Final = "reload-race-agent-id"
+ prisma_client: Final = MagicMock()
+ prisma_client.db.litellm_agentstable.find_unique = AsyncMock(
+ side_effect=AssertionError("db hit while the agent reload held the reconcile lock")
+ )
+ monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+ monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+
+ async with AGENT_RECONCILE_LOCK:
+ resync_task: Final = asyncio.ensure_future(_resync_agents(agent_id))
+ await asyncio.sleep(0.05)
+ assert not resync_task.done()
+ clean_agent_registry.register_agent(
+ agent_config=AgentResponse.model_validate(FakeAgentRow(agent_id, "reload-race-agent").model_dump())
+ )
+
+ assert await resync_task is True
+ assert len(clean_agent_registry.agent_list) == 1
diff --git a/tests/test_litellm/proxy/hooks/test_batch_file_validation.py b/tests/test_litellm/proxy/hooks/test_batch_file_validation.py
index 7ff1bc11d81..1ce1a2f3e51 100644
--- a/tests/test_litellm/proxy/hooks/test_batch_file_validation.py
+++ b/tests/test_litellm/proxy/hooks/test_batch_file_validation.py
@@ -1382,6 +1382,315 @@ async def test_check_and_increment_computes_descriptors_when_not_passed():
parallel_request_limiter._create_rate_limit_descriptors.assert_called_once()
+@pytest.mark.asyncio
+async def test_pre_call_enforces_project_otpm_limit_for_batch():
+ """VERIA regression: ``_create_batch_rate_limit_descriptors`` only asked
+ for the generic key/user/team/model descriptors, so a project caller
+ could submit a batch that consumed none of its configured project OTPM
+ quota. The project OTPM descriptor must now be present and charged with
+ the batch's estimated *output* tokens, not its input tokens."""
+ from litellm import DualCache
+ from litellm.proxy.hooks.batch_rate_limiter import _PROXY_BatchRateLimiter
+ from litellm.proxy.hooks.parallel_request_limiter_v3 import (
+ _PROXY_MaxParallelRequestsHandler_v3,
+ )
+ from litellm.proxy.utils import InternalUsageCache
+
+ local_cache = DualCache()
+ parallel_request_limiter = _PROXY_MaxParallelRequestsHandler_v3(
+ internal_usage_cache=InternalUsageCache(local_cache)
+ )
+ rate_limiter = _PROXY_BatchRateLimiter(
+ internal_usage_cache=InternalUsageCache(local_cache),
+ parallel_request_limiter=parallel_request_limiter,
+ )
+
+ user = UserAPIKeyAuth(
+ api_key="sk-project-batch-otpm",
+ models=["*"],
+ project_id="proj-mantle-batch",
+ project_metadata={"model_otpm_limit": {"gpt-4o-mini": 50}},
+ )
+
+ # Two rows each declaring max_tokens=40: 80 output tokens total, over the
+ # configured 50-token project OTPM limit but negligible input tokens.
+ mock_content = MagicMock()
+ mock_content.content = (
+ b'{"body": {"model": "gpt-4o-mini", "max_tokens": 40, '
+ b'"messages": [{"role": "user", "content": "hi"}]}}\n'
+ b'{"body": {"model": "gpt-4o-mini", "max_tokens": 40, '
+ b'"messages": [{"role": "user", "content": "hi"}]}}\n'
+ )
+
+ with (
+ patch("litellm.proxy.proxy_server.general_settings", {}),
+ patch("litellm.proxy.proxy_server.llm_router", MagicMock()),
+ patch(
+ "litellm.proxy.openai_files_endpoints.common_utils.get_credentials_for_model",
+ return_value={"custom_llm_provider": "openai"},
+ ),
+ patch("litellm.afile_content", new=AsyncMock(return_value=mock_content)),
+ ):
+ with pytest.raises(HTTPException) as exc:
+ await rate_limiter.async_pre_call_hook(
+ user_api_key_dict=user,
+ cache=local_cache,
+ data={"input_file_id": "file-abc123", "model": "gpt-4o-mini"},
+ call_type="acreate_batch",
+ )
+
+ assert exc.value.status_code == 429
+ assert "model_per_project_otpm" in str(exc.value.detail)
+
+
+@pytest.mark.asyncio
+async def test_pre_call_enforces_project_itpm_limit_for_batch():
+ """Companion to the OTPM regression above: a project's ITPM quota must
+ also apply to batch submissions."""
+ from litellm import DualCache
+ from litellm.proxy.hooks.batch_rate_limiter import _PROXY_BatchRateLimiter
+ from litellm.proxy.hooks.parallel_request_limiter_v3 import (
+ _PROXY_MaxParallelRequestsHandler_v3,
+ )
+ from litellm.proxy.utils import InternalUsageCache
+
+ local_cache = DualCache()
+ parallel_request_limiter = _PROXY_MaxParallelRequestsHandler_v3(
+ internal_usage_cache=InternalUsageCache(local_cache)
+ )
+ rate_limiter = _PROXY_BatchRateLimiter(
+ internal_usage_cache=InternalUsageCache(local_cache),
+ parallel_request_limiter=parallel_request_limiter,
+ )
+
+ user = UserAPIKeyAuth(
+ api_key="sk-project-batch-itpm",
+ models=["*"],
+ project_id="proj-mantle-batch",
+ project_metadata={"model_itpm_limit": {"gpt-4o-mini": 1}},
+ )
+
+ mock_content = MagicMock()
+ mock_content.content = (
+ b'{"body": {"model": "gpt-4o-mini", "max_tokens": 1, '
+ b'"messages": [{"role": "user", "content": "well over one token of input"}]}}\n'
+ )
+
+ with (
+ patch("litellm.proxy.proxy_server.general_settings", {}),
+ patch("litellm.proxy.proxy_server.llm_router", MagicMock()),
+ patch(
+ "litellm.proxy.openai_files_endpoints.common_utils.get_credentials_for_model",
+ return_value={"custom_llm_provider": "openai"},
+ ),
+ patch("litellm.afile_content", new=AsyncMock(return_value=mock_content)),
+ ):
+ with pytest.raises(HTTPException) as exc:
+ await rate_limiter.async_pre_call_hook(
+ user_api_key_dict=user,
+ cache=local_cache,
+ data={"input_file_id": "file-abc123", "model": "gpt-4o-mini"},
+ call_type="acreate_batch",
+ )
+
+ assert exc.value.status_code == 429
+ assert "model_per_project_itpm" in str(exc.value.detail)
+
+
+@pytest.mark.asyncio
+async def test_pre_call_enforces_project_otpm_limit_for_non_routing_row_model():
+ """VERIA regression: project ITPM/OTPM descriptors were built only for the
+ file-bound/top-level routing model, so a caller could bind the batch file
+ to an unlimited model while a JSONL row's own `body.model` named a
+ different, quota-limited model. That row's tokens must still be charged
+ against its own model's project OTPM quota."""
+ from litellm import DualCache
+ from litellm.proxy.hooks.batch_rate_limiter import _PROXY_BatchRateLimiter
+ from litellm.proxy.hooks.parallel_request_limiter_v3 import (
+ _PROXY_MaxParallelRequestsHandler_v3,
+ )
+ from litellm.proxy.utils import InternalUsageCache
+
+ local_cache = DualCache()
+ parallel_request_limiter = _PROXY_MaxParallelRequestsHandler_v3(
+ internal_usage_cache=InternalUsageCache(local_cache)
+ )
+ rate_limiter = _PROXY_BatchRateLimiter(
+ internal_usage_cache=InternalUsageCache(local_cache),
+ parallel_request_limiter=parallel_request_limiter,
+ )
+
+ # The routing model ("unlimited-model") has no configured quota; only
+ # "quota-limited-model" -- named inside the JSONL row, not the routing
+ # model -- has a project OTPM limit.
+ user = UserAPIKeyAuth(
+ api_key="sk-project-batch-cross-model",
+ models=["*"],
+ project_id="proj-mantle-batch",
+ project_metadata={"model_otpm_limit": {"quota-limited-model": 50}},
+ )
+
+ mock_content = MagicMock()
+ mock_content.content = (
+ b'{"body": {"model": "quota-limited-model", "max_tokens": 80, '
+ b'"messages": [{"role": "user", "content": "hi"}]}}\n'
+ )
+
+ with (
+ patch("litellm.proxy.proxy_server.general_settings", {}),
+ patch("litellm.proxy.proxy_server.llm_router", MagicMock()),
+ patch(
+ "litellm.proxy.openai_files_endpoints.common_utils.get_credentials_for_model",
+ return_value={"custom_llm_provider": "openai"},
+ ),
+ patch("litellm.afile_content", new=AsyncMock(return_value=mock_content)),
+ ):
+ with pytest.raises(HTTPException) as exc:
+ await rate_limiter.async_pre_call_hook(
+ user_api_key_dict=user,
+ cache=local_cache,
+ data={"input_file_id": "file-abc123", "model": "unlimited-model"},
+ call_type="acreate_batch",
+ )
+
+ assert exc.value.status_code == 429
+ assert "model_per_project_otpm" in str(exc.value.detail)
+ assert "quota-limited-model" in str(exc.value.detail)
+
+
+@pytest.mark.asyncio
+async def test_pre_call_charges_each_row_model_against_its_own_project_quota():
+ """A batch whose rows target two different project-quota-limited models
+ must charge each row's tokens only against its own model's quota, never
+ the other model's or the whole batch's combined total. The under-limit
+ model's request must succeed even though the over-limit model's row
+ would fail on its own."""
+ from litellm import DualCache
+ from litellm.proxy.hooks.batch_rate_limiter import _PROXY_BatchRateLimiter
+ from litellm.proxy.hooks.parallel_request_limiter_v3 import (
+ PROJECT_OTPM_DESCRIPTOR_KEY,
+ _PROXY_MaxParallelRequestsHandler_v3,
+ )
+ from litellm.proxy.utils import InternalUsageCache
+
+ local_cache = DualCache()
+ parallel_request_limiter = _PROXY_MaxParallelRequestsHandler_v3(
+ internal_usage_cache=InternalUsageCache(local_cache)
+ )
+ rate_limiter = _PROXY_BatchRateLimiter(
+ internal_usage_cache=InternalUsageCache(local_cache),
+ parallel_request_limiter=parallel_request_limiter,
+ )
+
+ user = UserAPIKeyAuth(
+ api_key="sk-project-batch-two-models",
+ models=["*"],
+ project_id="proj-mantle-batch",
+ project_metadata={
+ "model_otpm_limit": {"model-a": 1000, "model-b": 10},
+ },
+ )
+
+ # model-a stays comfortably under its 1000 OTPM limit; model-b's single
+ # row alone exceeds its 10 OTPM limit. If the two were combined into one
+ # counter (the pre-fix behavior for the routing model), model-a's ample
+ # headroom would mask model-b's overage.
+ mock_content = MagicMock()
+ mock_content.content = (
+ b'{"body": {"model": "model-a", "max_tokens": 5, '
+ b'"messages": [{"role": "user", "content": "hi"}]}}\n'
+ b'{"body": {"model": "model-b", "max_tokens": 40, '
+ b'"messages": [{"role": "user", "content": "hi"}]}}\n'
+ )
+
+ with (
+ patch("litellm.proxy.proxy_server.general_settings", {}),
+ patch("litellm.proxy.proxy_server.llm_router", MagicMock()),
+ patch(
+ "litellm.proxy.openai_files_endpoints.common_utils.get_credentials_for_model",
+ return_value={"custom_llm_provider": "openai"},
+ ),
+ patch("litellm.afile_content", new=AsyncMock(return_value=mock_content)),
+ ):
+ with pytest.raises(HTTPException) as exc:
+ await rate_limiter.async_pre_call_hook(
+ user_api_key_dict=user,
+ cache=local_cache,
+ data={"input_file_id": "file-abc123", "model": "model-a"},
+ call_type="acreate_batch",
+ )
+
+ assert exc.value.status_code == 429
+ assert "model-b" in str(exc.value.detail)
+
+ # model-a's own counter was not touched by model-b's rejection: a
+ # follow-up model-a-only batch well within its own limit must still pass.
+ model_a_status = await parallel_request_limiter.should_rate_limit(
+ descriptors=[
+ {
+ "key": PROJECT_OTPM_DESCRIPTOR_KEY,
+ "value": "proj-mantle-batch:model-a",
+ "rate_limit": {
+ "requests_per_unit": None,
+ "tokens_per_unit": 1000,
+ "window_size": parallel_request_limiter.window_size,
+ },
+ }
+ ],
+ read_only=True,
+ )
+ assert model_a_status["overall_code"] == "OK"
+
+
+def test_should_not_skip_when_project_has_io_limit_for_non_routing_model():
+ """The no-limits skip must not fire just because the file-bound/top-level
+ routing model itself has no configured quota: a JSONL row can name a
+ different model that the project *does* quota, and that isn't knowable
+ without downloading and parsing the file."""
+ rate_limiter = _make_rate_limiter()
+ # No key/team/model-level limits at all -- only a project OTPM limit for a
+ # model unrelated to the routing model below.
+ rate_limiter.parallel_request_limiter._create_rate_limit_descriptors.return_value = [
+ {"rate_limit": {}}
+ ]
+ user = UserAPIKeyAuth(
+ api_key="sk",
+ models=["*"],
+ project_id="proj-mantle-batch",
+ project_metadata={"model_otpm_limit": {"some-other-model": 50}},
+ )
+ with patch("litellm.proxy.proxy_server.general_settings", {}):
+ should_skip, descriptors = rate_limiter._should_skip_batch_input_file_processing(
+ data={"model": "unlimited-model", "input_file_id": "file-abc"},
+ user_api_key_dict=user,
+ )
+ assert should_skip is False
+ assert descriptors is not None
+
+
+def test_should_skip_when_project_has_no_io_limits_and_no_other_limits():
+ """Sanity check for the new project-limits carve-out: a project caller
+ with no ITPM/OTPM configuration anywhere must still get the fast-path
+ skip when no other rate limits apply, exactly as before this fix."""
+ rate_limiter = _make_rate_limiter()
+ rate_limiter.parallel_request_limiter._create_rate_limit_descriptors.return_value = [
+ {"rate_limit": {}}
+ ]
+ user = UserAPIKeyAuth(
+ api_key="sk",
+ models=["*"],
+ project_id="proj-mantle-batch",
+ project_metadata={},
+ )
+ with patch("litellm.proxy.proxy_server.general_settings", {}):
+ should_skip, descriptors = rate_limiter._should_skip_batch_input_file_processing(
+ data={"model": "unlimited-model", "input_file_id": "file-abc"},
+ user_api_key_dict=user,
+ )
+ assert should_skip is True
+ assert descriptors is None
+
+
@pytest.mark.asyncio
async def test_count_input_file_usage_raises_on_non_bytes_content():
from litellm.proxy.hooks.batch_rate_limiter import _PROXY_BatchRateLimiter
@@ -1671,3 +1980,117 @@ async def test_count_input_file_usage_collects_models_after_malformed_line():
)
assert exc.value.status_code == 403
+
+
+# ---------------------------------------------------------------------------
+# VERIA-Low regression: Responses batch rows must not bypass project OTPM
+# ---------------------------------------------------------------------------
+
+
+def _output_estimator():
+ """A `_PROXY_BatchRateLimiter` whose output-token floor is observable:
+ the no-`max_tokens` floor mock returns a distinctive sentinel so tests can
+ tell "floor was used" apart from "an explicit cap was read"."""
+ from litellm.proxy.hooks.batch_rate_limiter import _PROXY_BatchRateLimiter
+ from litellm.proxy.hooks.parallel_request_limiter_v3 import (
+ _PROXY_MaxParallelRequestsHandler_v3,
+ )
+
+ limiter = MagicMock()
+ limiter.no_max_tokens_output_floor.return_value = 999
+ limiter.get_output_candidate_count = _PROXY_MaxParallelRequestsHandler_v3.get_output_candidate_count
+ return _PROXY_BatchRateLimiter(
+ internal_usage_cache=MagicMock(),
+ parallel_request_limiter=limiter,
+ )
+
+
+def test_estimate_entry_output_tokens_zero_for_embeddings_url():
+ """A real `/v1/embeddings` row reserves zero output tokens."""
+ rate_limiter = _output_estimator()
+ entry = {
+ "url": "/v1/embeddings",
+ "body": {"model": "text-embedding-3-small", "input": "hello world"},
+ }
+
+ assert rate_limiter._estimate_entry_output_tokens(entry, None) == 0
+
+
+def test_estimate_entry_output_tokens_does_not_zero_responses_row_with_input():
+ """Pre-fix: a `/v1/responses` row carries `body.input` with no `messages`/
+ `prompt`, so the old body-shape heuristic misclassified it as embeddings
+ and reserved zero output tokens -- a project caller could submit large
+ Responses generations against a quota-limited model without consuming
+ OTPM. The row's own `url` (not body shape) must decide this."""
+ rate_limiter = _output_estimator()
+ entry = {
+ "url": "/v1/responses",
+ "body": {"model": "gpt-4o", "input": "write me an essay"},
+ }
+
+ # No explicit cap on the row, so it must fall back to the no-max-tokens
+ # floor -- never straight to zero.
+ assert rate_limiter._estimate_entry_output_tokens(entry, None) == 999
+ rate_limiter.parallel_request_limiter.no_max_tokens_output_floor.assert_called_once_with(None)
+
+
+def test_estimate_entry_output_tokens_uses_max_output_tokens_for_responses():
+ """`/v1/responses` caps output with `max_output_tokens`, not `max_tokens`/
+ `max_completion_tokens`. Pre-fix this field was never inspected, so a
+ capped Responses row still fell through to the (possibly larger) floor
+ estimate instead of the caller's own declared cap."""
+ rate_limiter = _output_estimator()
+ entry = {
+ "url": "/v1/responses",
+ "body": {"model": "gpt-4o", "input": "hi", "max_output_tokens": 123},
+ }
+
+ assert rate_limiter._estimate_entry_output_tokens(entry, None) == 123
+ rate_limiter.parallel_request_limiter.no_max_tokens_output_floor.assert_not_called()
+
+
+def test_estimate_entry_output_tokens_prefers_max_tokens_over_max_output_tokens():
+ """When a row somehow carries both fields, the chat-style cap wins first --
+ `max_output_tokens` is only consulted once the chat-style caps are absent."""
+ rate_limiter = _output_estimator()
+ entry = {
+ "url": "/v1/chat/completions",
+ "body": {
+ "model": "gpt-4o",
+ "messages": [],
+ "max_tokens": 50,
+ "max_output_tokens": 500,
+ },
+ }
+
+ assert rate_limiter._estimate_entry_output_tokens(entry, None) == 50
+
+
+@pytest.mark.parametrize(
+ ("body_extra", "expected"),
+ [
+ ({"max_tokens": 40, "n": 10}, 400),
+ ({"max_tokens": 40, "best_of": 5}, 200),
+ ({"max_tokens": 40, "n": 3, "best_of": 5}, 200),
+ ({"max_tokens": 40, "n": 0}, 40),
+ ({"max_tokens": 40, "n": -2}, 40),
+ ({"max_tokens": 40, "n": 5.0}, 200),
+ ({"max_tokens": 40, "n": "10"}, 400),
+ ({"max_tokens": 40, "n": "not-a-number"}, 40),
+ ({"max_tokens": 40, "n": 1e309}, 40),
+ ({"max_tokens": 1e309, "n": 3}, 2997),
+ ({"n": 3}, 2997),
+ ],
+)
+def test_estimate_entry_output_tokens_multiplies_candidate_count(body_extra, expected):
+ """A row generating n / best_of candidates consumes that many completions'
+ worth of output tokens, so the OTPM reservation must scale with the
+ effective candidate count. Pre-fix a `max_tokens: 40, n: 10` row consumed
+ up to 400 output tokens while reserving only 40."""
+ rate_limiter = _output_estimator()
+ entry = {
+ "url": "/v1/chat/completions",
+ "body": {"model": "gpt-4o", "messages": [], **body_extra},
+ }
+
+ assert rate_limiter._estimate_entry_output_tokens(entry, None) == expected
diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py
index 194b9dcb217..54366226dfb 100644
--- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py
+++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py
@@ -3116,6 +3116,192 @@ async def test_project_model_rate_limits_not_triggered_for_other_model_v3():
), f"model_per_project should not be added for unrelated model, got: {descriptor_keys}"
+@pytest.mark.asyncio
+async def test_project_model_itpm_otpm_limits_enforced_v3():
+ """
+ Project-level model_itpm_limit/model_otpm_limit must produce distinct
+ Bedrock Mantle-style input and output token descriptors.
+ """
+ _api_key = hash_token("sk-project-io-test")
+ local_cache = DualCache()
+ parallel_request_handler = _PROXY_MaxParallelRequestsHandler(
+ internal_usage_cache=InternalUsageCache(local_cache)
+ )
+
+ captured_descriptors = []
+
+ async def mock_should_rate_limit(descriptors, **kwargs):
+ captured_descriptors.extend(descriptors)
+ return {"overall_code": "OK", "statuses": []}
+
+ parallel_request_handler.should_rate_limit = mock_should_rate_limit
+
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=_api_key,
+ project_id="proj-mantle",
+ project_metadata={
+ "model_itpm_limit": {"bedrock_mantle/claude-opus": 20000000},
+ "model_otpm_limit": {"bedrock_mantle/claude-opus": 4000000},
+ },
+ )
+
+ await parallel_request_handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=local_cache,
+ data={"model": "bedrock_mantle/claude-opus"},
+ call_type="",
+ )
+
+ descriptor_keys = [d["key"] for d in captured_descriptors]
+ assert "model_per_project_itpm" in descriptor_keys
+ assert "model_per_project_otpm" in descriptor_keys
+ assert "model_per_project" not in descriptor_keys
+
+ itpm_descriptor = next(
+ d for d in captured_descriptors if d["key"] == "model_per_project_itpm"
+ )
+ otpm_descriptor = next(
+ d for d in captured_descriptors if d["key"] == "model_per_project_otpm"
+ )
+ assert itpm_descriptor["value"] == "proj-mantle:bedrock_mantle/claude-opus"
+ assert itpm_descriptor["rate_limit"]["tokens_per_unit"] == 20000000
+ assert otpm_descriptor["value"] == "proj-mantle:bedrock_mantle/claude-opus"
+ assert otpm_descriptor["rate_limit"]["tokens_per_unit"] == 4000000
+
+
+@pytest.mark.asyncio
+async def test_project_model_itpm_otpm_limits_not_triggered_for_other_model_v3():
+ """Split project limits must not apply to an unrelated model."""
+ _api_key = hash_token("sk-project-io-test-2")
+ local_cache = DualCache()
+ parallel_request_handler = _PROXY_MaxParallelRequestsHandler(
+ internal_usage_cache=InternalUsageCache(local_cache)
+ )
+
+ captured_descriptors = []
+
+ async def mock_should_rate_limit(descriptors, **kwargs):
+ captured_descriptors.extend(descriptors)
+ return {"overall_code": "OK", "statuses": []}
+
+ parallel_request_handler.should_rate_limit = mock_should_rate_limit
+
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=_api_key,
+ project_id="proj-mantle",
+ project_metadata={
+ "model_itpm_limit": {"bedrock_mantle/claude-opus": 20000000},
+ },
+ )
+
+ await parallel_request_handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=local_cache,
+ data={"model": "gpt-4"},
+ call_type="",
+ )
+
+ descriptor_keys = [d["key"] for d in captured_descriptors]
+ assert "model_per_project_itpm" not in descriptor_keys
+ assert "model_per_project_otpm" not in descriptor_keys
+
+
+@pytest.mark.asyncio
+async def test_project_model_itpm_and_tpm_limits_coexist_v3():
+ """Combined project TPM and split ITPM/OTPM limits are enforced together."""
+ _api_key = hash_token("sk-project-io-test-3")
+ local_cache = DualCache()
+ parallel_request_handler = _PROXY_MaxParallelRequestsHandler(
+ internal_usage_cache=InternalUsageCache(local_cache)
+ )
+
+ captured_descriptors = []
+
+ async def mock_should_rate_limit(descriptors, **kwargs):
+ captured_descriptors.extend(descriptors)
+ return {"overall_code": "OK", "statuses": []}
+
+ parallel_request_handler.should_rate_limit = mock_should_rate_limit
+
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=_api_key,
+ project_id="proj-mantle",
+ project_metadata={
+ "model_tpm_limit": {"bedrock_mantle/claude-opus": 1000},
+ "model_itpm_limit": {"bedrock_mantle/claude-opus": 20000000},
+ "model_otpm_limit": {"bedrock_mantle/claude-opus": 4000000},
+ },
+ )
+
+ await parallel_request_handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=local_cache,
+ data={"model": "bedrock_mantle/claude-opus"},
+ call_type="",
+ )
+
+ descriptor_keys = [d["key"] for d in captured_descriptors]
+ assert "model_per_project" in descriptor_keys
+ assert "model_per_project_itpm" in descriptor_keys
+ assert "model_per_project_otpm" in descriptor_keys
+
+
+@pytest.mark.asyncio
+async def test_enforce_project_io_token_quota_for_frame_blocks_over_limit_otpm():
+ """VERIA regression: the Responses WebSocket connection-level pre-call
+ hook only runs once, but a connection accepts many response.create
+ frames. enforce_project_io_token_quota_for_frame is the per-frame check
+ that closes that gap; it must reserve against the caller's project OTPM
+ limit and reject once a frame's estimated output tokens exceed it."""
+ _api_key = hash_token("sk-ws-frame-otpm")
+ local_cache = DualCache()
+ handler = _PROXY_MaxParallelRequestsHandler(
+ internal_usage_cache=InternalUsageCache(local_cache)
+ )
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=_api_key,
+ project_id="proj-mantle-ws",
+ project_metadata={"model_otpm_limit": {"gpt-4o": 50}},
+ )
+
+ await handler.enforce_project_io_token_quota_for_frame(
+ user_api_key_dict=user_api_key_dict,
+ requested_model="gpt-4o",
+ estimated_input_tokens=1,
+ estimated_output_tokens=30,
+ )
+
+ with pytest.raises(HTTPException) as exc:
+ await handler.enforce_project_io_token_quota_for_frame(
+ user_api_key_dict=user_api_key_dict,
+ requested_model="gpt-4o",
+ estimated_input_tokens=1,
+ estimated_output_tokens=30,
+ )
+
+ assert exc.value.status_code == 429
+ assert "model_per_project_otpm" in str(exc.value.detail)
+
+
+@pytest.mark.asyncio
+async def test_enforce_project_io_token_quota_for_frame_noop_without_project_limits():
+ """A key with no project ITPM/OTPM configured must never be blocked by
+ the per-frame check (no descriptors to reserve against)."""
+ _api_key = hash_token("sk-ws-frame-no-limits")
+ local_cache = DualCache()
+ handler = _PROXY_MaxParallelRequestsHandler(
+ internal_usage_cache=InternalUsageCache(local_cache)
+ )
+ user_api_key_dict = UserAPIKeyAuth(api_key=_api_key)
+
+ await handler.enforce_project_io_token_quota_for_frame(
+ user_api_key_dict=user_api_key_dict,
+ requested_model="gpt-4o",
+ estimated_input_tokens=10_000_000,
+ estimated_output_tokens=10_000_000,
+ )
+
+
@pytest.mark.asyncio
async def test_pre_call_hook_keeps_internal_stash_out_of_request_body():
"""Regression for #27001 / #35197: the limiter's per-request bookkeeping
@@ -3192,7 +3378,7 @@ async def test_responses_route_body_untouched_by_pre_call_hook(caller_metadata):
_api_key = hash_token("sk-responses-regression")
user_api_key_dict = UserAPIKeyAuth(
api_key=_api_key,
- tpm_limit=1000,
+ tpm_limit=100000,
rpm_limit=5,
)
local_cache = DualCache()
@@ -5264,7 +5450,7 @@ async def test_configured_estimate_does_not_apply_to_embeddings(monkeypatch):
local_cache,
user_api_key_dict,
{"model": "text-embedding-3-small", "input": "hello"},
- call_type="embeddings",
+ call_type="embedding",
)
assert reserved == ONE_TOKEN_PROMPT_INPUT_ESTIMATE
diff --git a/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py b/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py
index f7bd37b412a..8d03857c917 100644
--- a/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py
+++ b/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py
@@ -15,7 +15,7 @@ Redis.
"""
import asyncio
-from datetime import datetime
+from datetime import datetime, timedelta
from typing import Any, Dict
import pytest
@@ -23,15 +23,25 @@ import pytest
from litellm.caching.caching import DualCache
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
+ PROJECT_ITPM_DESCRIPTOR_KEY,
+ PROJECT_OTPM_DESCRIPTOR_KEY,
+ _AUDIO_BYTES_PER_TOKEN,
_PROXY_MaxParallelRequestsHandler_v3 as RateLimitHandler,
)
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
+ _call_id_from_callback_kwargs,
_request_stash,
get_or_create_request_stash,
get_request_stash,
)
from litellm.proxy.utils import InternalUsageCache, hash_token
-from litellm.types.utils import ModelResponse, Usage
+from litellm.types.llms.openai import (
+ InputTokensDetails,
+ ResponseAPIUsage,
+ ResponsesAPIResponse,
+)
+from litellm.types.rerank import RerankResponse
+from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage
@pytest.fixture
@@ -582,6 +592,47 @@ async def test_estimate_tokens_uses_max_tokens_when_explicit(rate_limiter):
assert estimate == 4 + 25
+@pytest.mark.asyncio
+async def test_estimate_tokens_honors_explicit_zero_max_tokens(rate_limiter):
+ """
+ Regression for a Greptile finding: explicit_max_tokens was resolved via
+ `data.get("max_tokens") or data.get("max_completion_tokens") or
+ data.get("max_output_tokens")`, so an explicit 0 in the first field was
+ falsy and fell through to the next field (or the no-max_tokens floor),
+ silently discarding a caller's explicit zero-output request.
+ """
+ handler, _cache = rate_limiter
+
+ estimate = handler._estimate_tokens_for_request(
+ data={
+ "messages": [
+ {"role": "user", "content": "abcd" * 4}
+ ], # 16 chars ~ 4 tokens
+ "max_tokens": 0,
+ }
+ )
+ assert estimate == 4, (
+ f"expected input-only reservation (4) for an explicit max_tokens=0, got {estimate}"
+ )
+
+
+@pytest.mark.asyncio
+async def test_estimate_tokens_honors_explicit_zero_max_output_tokens_for_responses(
+ rate_limiter,
+):
+ handler, _cache = rate_limiter
+
+ estimate = handler._estimate_tokens_for_request(
+ data={
+ "input": "describe this image in detail", # 29 chars ~ 7 tokens
+ "max_output_tokens": 0,
+ },
+ min_configured_tpm_limit=40,
+ call_type="aresponses",
+ )
+ assert estimate == 23
+
+
@pytest.mark.asyncio
async def test_estimate_tokens_zero_for_empty_embeddings(rate_limiter):
"""Embeddings have no output budget — reservation should equal input only."""
@@ -1197,5 +1248,2427 @@ async def test_small_tpm_cap_preserves_explicit_max_tokens(rate_limiter):
assert data["max_tokens"] == 500
+@pytest.mark.asyncio
+async def test_project_otpm_reservation_prevents_concurrent_bypass(rate_limiter):
+ """
+ Bedrock Mantle-style OTPM: with a 100 OTPM limit and 5 concurrent
+ requests each reserving 50+ output tokens, upfront reservation must
+ reject the late arrivals -- not let all 5 through. Exercises the
+ in-memory fallback in ``atomic_check_and_increment_by_n`` for the
+ project-scoped ITPM/OTPM descriptors specifically.
+ """
+ handler, cache = rate_limiter
+
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=hash_token("sk-otpm-bypass"),
+ project_id="proj-mantle-bypass",
+ project_metadata={
+ "model_otpm_limit": {"bedrock_mantle/claude-opus": 100},
+ },
+ )
+
+ request_data = {
+ "model": "bedrock_mantle/claude-opus",
+ "messages": [{"role": "user", "content": "hello"}],
+ "max_tokens": 50,
+ }
+
+ async def make_request(request_id: int) -> Dict[str, Any]:
+ data = request_data.copy()
+ try:
+ await handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=cache,
+ data=data,
+ call_type="",
+ )
+ return {"request_id": request_id, "success": True}
+ except Exception as e:
+ return {
+ "request_id": request_id,
+ "success": False,
+ "status_code": getattr(e, "status_code", None),
+ }
+
+ results = await asyncio.gather(*[make_request(i) for i in range(5)])
+
+ successful = [r for r in results if r["success"]]
+ rate_limited = [
+ r for r in results if not r["success"] and r.get("status_code") == 429
+ ]
+
+ assert len(rate_limited) > 0, (
+ f"Expected some OTPM-rate-limited requests but all {len(successful)} succeeded."
+ )
+
+
+@pytest.mark.asyncio
+async def test_project_otpm_rejects_multiple_completion_candidates(rate_limiter):
+ handler, cache = rate_limiter
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=hash_token("sk-otpm-multiple-candidates"),
+ project_id="proj-multiple-candidates",
+ project_metadata={
+ "model_otpm_limit": {"bedrock_mantle/claude-opus": 500},
+ },
+ )
+ data = {
+ "model": "bedrock_mantle/claude-opus",
+ "messages": [{"role": "user", "content": "hello"}],
+ "max_tokens": 100,
+ "n": 10,
+ }
+
+ with pytest.raises(Exception) as exc_info:
+ await handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=cache,
+ data=data,
+ call_type="acompletion",
+ )
+
+ assert getattr(exc_info.value, "status_code", None) == 429
+
+
+@pytest.mark.asyncio
+async def test_project_otpm_reserves_largest_conflicting_output_cap(rate_limiter):
+ handler, cache = rate_limiter
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=hash_token("sk-otpm-conflicting-caps"),
+ project_id="proj-conflicting-caps",
+ project_metadata={
+ "model_otpm_limit": {"bedrock_mantle/claude-opus": 50},
+ },
+ )
+ data = {
+ "model": "bedrock_mantle/claude-opus",
+ "messages": [{"role": "user", "content": "hello"}],
+ "max_tokens": 1,
+ "max_completion_tokens": 100,
+ }
+
+ with pytest.raises(Exception) as exc_info:
+ await handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=cache,
+ data=data,
+ call_type="acompletion",
+ )
+
+ assert getattr(exc_info.value, "status_code", None) == 429
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "call_type",
+ ["agenerate_content", "agenerate_content_stream"],
+)
+@pytest.mark.parametrize("config_field", ["config", "generationConfig"])
+async def test_project_otpm_rejects_google_genai_native_output_cap(
+ rate_limiter,
+ call_type,
+ config_field,
+):
+ handler, cache = rate_limiter
+ model = "gemini/gemini-3-flash-preview"
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=hash_token("sk-google-genai-native-otpm"),
+ project_id="project-google-genai-native-otpm",
+ project_metadata={"model_otpm_limit": {model: 50}},
+ )
+
+ with pytest.raises(Exception) as exc_info:
+ await handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=cache,
+ data={
+ "model": model,
+ "contents": [{"role": "user", "parts": [{"text": "Hello"}]}],
+ config_field: {"maxOutputTokens": 100},
+ },
+ call_type=call_type,
+ )
+
+ assert getattr(exc_info.value, "status_code", None) == 429
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "call_type",
+ ["agenerate_content", "agenerate_content_stream"],
+)
+@pytest.mark.parametrize("candidate_count_field", ["candidateCount", "candidate_count"])
+async def test_project_otpm_rejects_google_genai_native_candidate_count(
+ rate_limiter,
+ call_type,
+ candidate_count_field,
+):
+ handler, cache = rate_limiter
+ model = "gemini/gemini-3-flash-preview"
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=hash_token("sk-google-genai-native-candidate-count"),
+ project_id="project-google-genai-native-candidate-count",
+ project_metadata={"model_otpm_limit": {model: 150}},
+ )
+
+ with pytest.raises(Exception) as exc_info:
+ await handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=cache,
+ data={
+ "model": model,
+ "contents": [{"role": "user", "parts": [{"text": "Hello"}]}],
+ "config": {
+ "maxOutputTokens": 50,
+ candidate_count_field: 4,
+ },
+ },
+ call_type=call_type,
+ )
+
+ assert getattr(exc_info.value, "status_code", None) == 429
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "call_type",
+ ["agenerate_content", "agenerate_content_stream"],
+)
+@pytest.mark.parametrize("config_field", [None, "config", "generationConfig"])
+async def test_project_otpm_injects_google_genai_native_output_cap(
+ rate_limiter,
+ call_type,
+ config_field,
+):
+ handler, cache = rate_limiter
+ model = "gemini/gemini-3-flash-preview"
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=hash_token("sk-google-genai-native-implicit-otpm"),
+ project_id="project-google-genai-native-implicit-otpm",
+ project_metadata={"model_otpm_limit": {model: 40}},
+ )
+ data = {
+ "model": model,
+ "contents": [{"role": "user", "parts": [{"text": "Hello"}]}],
+ }
+ if config_field is not None:
+ data[config_field] = {"temperature": 0}
+
+ await handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=cache,
+ data=data,
+ call_type=call_type,
+ )
+
+ stash = get_request_stash()
+ assert stash is not None
+ assert stash.otpm_reserved_tokens == 10
+ expected_config_field = config_field or "config"
+ assert data[expected_config_field]["maxOutputTokens"] == 10
+ assert "max_tokens" not in data
+
+
+@pytest.mark.asyncio
+async def test_project_otpm_over_limit_rolls_back_itpm_reservation(rate_limiter):
+ """
+ When ITPM reserves fine but OTPM is then over limit, the ITPM
+ reservation this same pre-call already made must be rolled back --
+ otherwise it leaks until the window's TTL, silently shrinking the ITPM
+ budget for every other request in that minute.
+ """
+ handler, cache = rate_limiter
+
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=hash_token("sk-otpm-rollback"),
+ project_id="proj-mantle-rollback",
+ project_metadata={
+ "model_itpm_limit": {"bedrock_mantle/claude-opus": 1000000},
+ "model_otpm_limit": {"bedrock_mantle/claude-opus": 10},
+ },
+ )
+
+ itpm_counter_key = handler.create_rate_limit_keys(
+ key="model_per_project_itpm",
+ value="proj-mantle-rollback:bedrock_mantle/claude-opus",
+ rate_limit_type="tokens",
+ )
+
+ data = {
+ "model": "bedrock_mantle/claude-opus",
+ "messages": [{"role": "user", "content": "hello"}],
+ "max_tokens": 500, # blows past the 10-token OTPM limit
+ }
+
+ with pytest.raises(Exception) as exc_info:
+ await handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=cache,
+ data=data,
+ call_type="",
+ )
+ assert getattr(exc_info.value, "status_code", None) == 429
+
+ cached_value = await cache.async_get_cache(key=itpm_counter_key, local_only=True)
+ assert int(cached_value or 0) == 0, (
+ f"ITPM reservation leaked after OTPM rejection: counter={cached_value}"
+ )
+
+
+@pytest.mark.asyncio
+async def test_project_itpm_reconciled_on_success_excludes_cached_tokens(rate_limiter):
+ """
+ On success, ITPM reconciles to billable input tokens (prompt_tokens
+ minus cached_tokens) -- not raw prompt_tokens. Cached prompt-read tokens
+ are free under Bedrock Mantle and must not count against the ITPM quota,
+ even though they still appear in usage/cost logging elsewhere.
+ """
+ handler, _cache = rate_limiter
+
+ itpm_scope = ("model_per_project_itpm", "proj-mantle:model")
+ otpm_scope = ("model_per_project_otpm", "proj-mantle:model")
+
+ stash = get_or_create_request_stash()
+ stash.itpm_reserved_tokens = 100
+ stash.itpm_reserved_scopes = frozenset({itpm_scope})
+ stash.otpm_reserved_tokens = 60
+ stash.otpm_reserved_scopes = frozenset({otpm_scope})
+ mock_kwargs = {}
+
+ mock_response = ModelResponse(
+ id="test",
+ object="chat.completion",
+ created=int(datetime.now().timestamp()),
+ model="bedrock_mantle/claude-opus",
+ usage=Usage(
+ prompt_tokens=80,
+ completion_tokens=40,
+ total_tokens=120,
+ prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=30),
+ ),
+ choices=[],
+ )
+
+ increments = []
+
+ async def mock_increment(increment_list, **kwargs):
+ for op in increment_list:
+ increments.append({"key": op["key"], "increment": op["increment_value"]})
+
+ handler.internal_usage_cache.dual_cache.async_increment_cache_pipeline = (
+ mock_increment
+ )
+
+ await handler.async_log_success_event(
+ kwargs=mock_kwargs,
+ response_obj=mock_response,
+ start_time=datetime.now(),
+ end_time=datetime.now(),
+ )
+
+ itpm_adjustments = [i for i in increments if "model_per_project_itpm" in i["key"]]
+ otpm_adjustments = [i for i in increments if "model_per_project_otpm" in i["key"]]
+
+ # billable_input = 80 - 30 cached = 50; delta = 50 - 100 reserved = -50
+ assert any(i["increment"] == -50 for i in itpm_adjustments), (
+ f"Expected a -50 ITPM adjustment (50 billable - 100 reserved), got: {itpm_adjustments}"
+ )
+ # delta = 40 actual completion - 60 reserved = -20
+ assert any(i["increment"] == -20 for i in otpm_adjustments), (
+ f"Expected a -20 OTPM adjustment (40 actual - 60 reserved), got: {otpm_adjustments}"
+ )
+
+
+@pytest.mark.asyncio
+async def test_project_reconciliation_does_not_decrement_later_window():
+ current_time = datetime(2026, 8, 5, 12, 0, 0)
+ cache = DualCache()
+ handler = RateLimitHandler(
+ internal_usage_cache=InternalUsageCache(cache),
+ time_provider=lambda: current_time,
+ )
+ handler.window_size = 60
+ scope = (PROJECT_ITPM_DESCRIPTOR_KEY, "project:model")
+ descriptor = {
+ "key": scope[0],
+ "value": scope[1],
+ "rate_limit": {"tokens_per_unit": 1000, "window_size": 60},
+ }
+
+ reservation = await handler.atomic_check_and_increment_by_n(
+ descriptors=[descriptor],
+ increments=[{"tokens": 100}],
+ )
+ counter_key = handler.create_rate_limit_keys(*scope, rate_limit_type="tokens")
+ window_identity = next(
+ identity
+ for identity in reservation["reservation_windows"]
+ if identity[0] == counter_key
+ )
+ stash = get_or_create_request_stash()
+ stash.itpm_reserved_tokens = 100
+ stash.itpm_reserved_scopes = frozenset({scope})
+ stash.itpm_reserved_window_identities = frozenset(
+ {window_identity}
+ )
+
+ current_time += timedelta(seconds=61)
+ later_reservation = await handler.atomic_check_and_increment_by_n(
+ descriptors=[descriptor],
+ increments=[{"tokens": 20}],
+ )
+ assert window_identity not in later_reservation["reservation_windows"]
+
+ await handler.async_log_success_event(
+ kwargs={},
+ response_obj=ModelResponse(
+ usage=Usage(prompt_tokens=10, completion_tokens=0, total_tokens=10)
+ ),
+ start_time=current_time,
+ end_time=current_time,
+ )
+
+ assert float(await cache.async_get_cache(key=counter_key, local_only=True) or 0) == 20
+
+
+@pytest.mark.asyncio
+async def test_project_reconciliation_decrements_its_active_window(rate_limiter):
+ handler, cache = rate_limiter
+ scope = (PROJECT_ITPM_DESCRIPTOR_KEY, "project:model")
+ descriptor = {
+ "key": scope[0],
+ "value": scope[1],
+ "rate_limit": {"tokens_per_unit": 1000, "window_size": 60},
+ }
+ reservation = await handler.atomic_check_and_increment_by_n(
+ descriptors=[descriptor],
+ increments=[{"tokens": 100}],
+ )
+ counter_key = handler.create_rate_limit_keys(*scope, rate_limit_type="tokens")
+ window_identity = next(
+ identity
+ for identity in reservation["reservation_windows"]
+ if identity[0] == counter_key
+ )
+ stash = get_or_create_request_stash()
+ stash.itpm_reserved_tokens = 100
+ stash.itpm_reserved_scopes = frozenset({scope})
+ stash.itpm_reserved_window_identities = frozenset(
+ {window_identity}
+ )
+
+ await handler.async_log_success_event(
+ kwargs={},
+ response_obj=ModelResponse(
+ usage=Usage(prompt_tokens=10, completion_tokens=0, total_tokens=10)
+ ),
+ start_time=datetime.now(),
+ end_time=datetime.now(),
+ )
+
+ assert float(await cache.async_get_cache(key=counter_key, local_only=True) or 0) == 10
+
+
+@pytest.mark.asyncio
+async def test_redis_window_guard_uses_reservation_identity_and_never_falls_back_negative(
+ rate_limiter,
+):
+ handler, _cache = rate_limiter
+ calls = []
+
+ async def failing_guard(*, keys, args):
+ calls.append((keys, args))
+ raise RuntimeError("redis unavailable")
+
+ unguarded_calls = []
+
+ async def capture_unguarded(pipeline_operations, **_kwargs):
+ unguarded_calls.extend(pipeline_operations)
+
+ handler.window_guarded_token_increment_script = failing_guard
+ handler.async_increment_tokens_with_ttl_preservation = capture_unguarded
+ await handler.async_increment_reservation_aware_tokens(
+ pipeline_operations=[
+ {
+ "key": "{model_per_project_itpm:project:model}:tokens",
+ "increment_value": -90,
+ "ttl": 60,
+ "window_key": "{model_per_project_itpm:project:model}:window",
+ "expected_window_start": "1234",
+ "reservation_backend": "redis",
+ }
+ ]
+ )
+
+ assert calls == [
+ (
+ [
+ "{model_per_project_itpm:project:model}:window",
+ "{model_per_project_itpm:project:model}:tokens",
+ ],
+ ["1234", -90, 60],
+ )
+ ]
+ assert unguarded_calls == []
+
+
+@pytest.mark.asyncio
+async def test_atomic_lua_response_carries_redis_window_identity(rate_limiter):
+ handler, _cache = rate_limiter
+ counter_key = "{model_per_project_itpm:project:model}:tokens"
+ meta = [
+ {
+ "descriptor_key": PROJECT_ITPM_DESCRIPTOR_KEY,
+ "descriptor_value": "project:model",
+ "current_limit": 100,
+ "rate_limit_type": "tokens",
+ "counter_key": counter_key,
+ }
+ ]
+
+ async def successful_reservation(*, keys, args):
+ return [0, 25, 1234]
+
+ handler.check_and_increment_by_n_script = successful_reservation
+ assert await handler._atomic_lua_per_descriptor([]) == {
+ "overall_code": "OK",
+ "statuses": [],
+ }
+
+ response = await handler._atomic_lua_per_descriptor(
+ descriptor_groups=[
+ (
+ [
+ "{model_per_project_itpm:project:model}:window",
+ counter_key,
+ ],
+ [100, 25, 60, 60],
+ meta,
+ )
+ ]
+ )
+
+ assert response["statuses"][0]["limit_remaining"] == 75
+ assert response["reservation_windows"] == frozenset(
+ {(counter_key, "1234", "redis")}
+ )
+
+
+@pytest.mark.asyncio
+async def test_project_itpm_otpm_released_on_failure(rate_limiter):
+ """On failure, the full ITPM and OTPM reservations must be refunded."""
+ handler, _cache = rate_limiter
+
+ itpm_scope = ("model_per_project_itpm", "proj-mantle:model")
+ otpm_scope = ("model_per_project_otpm", "proj-mantle:model")
+
+ stash = get_or_create_request_stash()
+ stash.itpm_reserved_tokens = 100
+ stash.itpm_reserved_scopes = frozenset({itpm_scope})
+ stash.otpm_reserved_tokens = 60
+ stash.otpm_reserved_scopes = frozenset({otpm_scope})
+ mock_kwargs = {}
+
+ increments = []
+
+ async def mock_increment(increment_list, **kwargs):
+ for op in increment_list:
+ increments.append({"key": op["key"], "increment": op["increment_value"]})
+
+ handler.internal_usage_cache.dual_cache.async_increment_cache_pipeline = (
+ mock_increment
+ )
+
+ await handler.async_log_failure_event(
+ kwargs=mock_kwargs,
+ response_obj=None,
+ start_time=datetime.now(),
+ end_time=datetime.now(),
+ )
+
+ itpm_releases = [i for i in increments if "model_per_project_itpm" in i["key"]]
+ otpm_releases = [i for i in increments if "model_per_project_otpm" in i["key"]]
+
+ assert any(i["increment"] == -100 for i in itpm_releases), itpm_releases
+ assert any(i["increment"] == -60 for i in otpm_releases), otpm_releases
+
+
+@pytest.mark.asyncio
+async def test_proxy_rejection_refunds_itpm_otpm_by_their_own_amount_not_combined(
+ rate_limiter,
+):
+ """
+ Regression for a Greptile-flagged bug: when a project configures both a
+ combined model_tpm_limit and split model_itpm_limit/model_otpm_limit for
+ the same model, async_post_call_failure_hook's proxy-side refund path
+ used to decrement every token descriptor -- including the ITPM/OTPM
+ ones -- by the flat combined reservation amount, instead of each
+ bucket's own reserved amount. That drives the split counters negative
+ (or under-refunds them) instead of returning them to exactly zero.
+ """
+ handler, cache = rate_limiter
+
+ api_key = hash_token("sk-mixed-tpm-io")
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=api_key,
+ project_id="proj-mixed",
+ project_metadata={
+ "model_tpm_limit": {"bedrock_mantle/claude-opus": 100000},
+ "model_itpm_limit": {"bedrock_mantle/claude-opus": 100000},
+ "model_otpm_limit": {"bedrock_mantle/claude-opus": 100000},
+ },
+ )
+
+ data = {
+ "model": "bedrock_mantle/claude-opus",
+ "messages": [
+ {"role": "user", "content": "hello there, this is a test message"}
+ ],
+ "max_tokens": 60,
+ }
+
+ await handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=cache,
+ data=data,
+ call_type="",
+ )
+
+ tpm_counter_key = handler.create_rate_limit_keys(
+ key="model_per_project",
+ value="proj-mixed:bedrock_mantle/claude-opus",
+ rate_limit_type="tokens",
+ )
+ itpm_counter_key = handler.create_rate_limit_keys(
+ key="model_per_project_itpm",
+ value="proj-mixed:bedrock_mantle/claude-opus",
+ rate_limit_type="tokens",
+ )
+ otpm_counter_key = handler.create_rate_limit_keys(
+ key="model_per_project_otpm",
+ value="proj-mixed:bedrock_mantle/claude-opus",
+ rate_limit_type="tokens",
+ )
+
+ tpm_reserved = int(
+ await cache.async_get_cache(key=tpm_counter_key, local_only=True) or 0
+ )
+ itpm_reserved = int(
+ await cache.async_get_cache(key=itpm_counter_key, local_only=True) or 0
+ )
+ otpm_reserved = int(
+ await cache.async_get_cache(key=otpm_counter_key, local_only=True) or 0
+ )
+ assert tpm_reserved > 0 and itpm_reserved > 0 and otpm_reserved > 0
+
+ await handler.async_post_call_failure_hook(
+ request_data=data,
+ original_exception=Exception("guardrail rejected"),
+ user_api_key_dict=user_api_key_dict,
+ )
+
+ tpm_after = int(
+ await cache.async_get_cache(key=tpm_counter_key, local_only=True) or 0
+ )
+ itpm_after = int(
+ await cache.async_get_cache(key=itpm_counter_key, local_only=True) or 0
+ )
+ otpm_after = int(
+ await cache.async_get_cache(key=otpm_counter_key, local_only=True) or 0
+ )
+
+ assert tpm_after == 0, f"combined TPM counter leaked: {tpm_after}"
+ assert itpm_after == 0, (
+ f"ITPM counter corrupted by combined-amount refund: {itpm_after}"
+ )
+ assert otpm_after == 0, (
+ f"OTPM counter corrupted by combined-amount refund: {otpm_after}"
+ )
+
+
+@pytest.mark.asyncio
+async def test_proxy_rejection_refunds_itpm_otpm_only_reservation_with_no_combined_tpm(
+ rate_limiter,
+):
+ """
+ Regression for the second half of the same bug: with only
+ model_itpm_limit/model_otpm_limit configured (no model_tpm_limit), the
+ combined reserved_tokens is 0, and the proxy-side refund path used to
+ return immediately on that -- leaking the ITPM/OTPM reservations until
+ the rate-limit window's TTL expired.
+ """
+ handler, cache = rate_limiter
+
+ api_key = hash_token("sk-io-only")
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=api_key,
+ project_id="proj-io-only",
+ project_metadata={
+ "model_itpm_limit": {"bedrock_mantle/claude-opus": 100000},
+ "model_otpm_limit": {"bedrock_mantle/claude-opus": 100000},
+ },
+ )
+
+ data = {
+ "model": "bedrock_mantle/claude-opus",
+ "messages": [
+ {"role": "user", "content": "hello there, this is a test message"}
+ ],
+ "max_tokens": 60,
+ }
+
+ await handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=cache,
+ data=data,
+ call_type="",
+ )
+
+ itpm_counter_key = handler.create_rate_limit_keys(
+ key="model_per_project_itpm",
+ value="proj-io-only:bedrock_mantle/claude-opus",
+ rate_limit_type="tokens",
+ )
+ otpm_counter_key = handler.create_rate_limit_keys(
+ key="model_per_project_otpm",
+ value="proj-io-only:bedrock_mantle/claude-opus",
+ rate_limit_type="tokens",
+ )
+ assert (
+ int(await cache.async_get_cache(key=itpm_counter_key, local_only=True) or 0) > 0
+ )
+ assert (
+ int(await cache.async_get_cache(key=otpm_counter_key, local_only=True) or 0) > 0
+ )
+
+ await handler.async_post_call_failure_hook(
+ request_data=data,
+ original_exception=Exception("guardrail rejected"),
+ user_api_key_dict=user_api_key_dict,
+ )
+
+ itpm_after = int(
+ await cache.async_get_cache(key=itpm_counter_key, local_only=True) or 0
+ )
+ otpm_after = int(
+ await cache.async_get_cache(key=otpm_counter_key, local_only=True) or 0
+ )
+ assert itpm_after == 0, (
+ f"ITPM-only reservation leaked on proxy rejection: {itpm_after}"
+ )
+ assert otpm_after == 0, (
+ f"OTPM-only reservation leaked on proxy rejection: {otpm_after}"
+ )
+
+
+@pytest.mark.asyncio
+async def test_otpm_rejection_does_not_double_refund_combined_tpm(rate_limiter):
+ """
+ Regression for a High-severity review finding: when the project ITPM
+ reservation succeeds but OTPM is then over limit,
+ _reserve_project_io_tokens_or_raise rolls back the combined-TPM
+ reservation that already succeeded earlier in the same pre-call, then
+ raises. If it doesn't also mark that reservation released,
+ async_post_call_failure_hook -- which fires next in the real request
+ lifecycle, since raising from async_pre_call_hook triggers it -- sees
+ the same still-stashed reservation and refunds it a second time,
+ driving the combined TPM counter negative and letting a caller push
+ past the project's real TPM budget by repeatedly triggering OTPM
+ rejections.
+ """
+ handler, cache = rate_limiter
+
+ api_key = hash_token("sk-double-refund")
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=api_key,
+ project_id="proj-double-refund",
+ project_metadata={
+ "model_tpm_limit": {"bedrock_mantle/claude-opus": 100000},
+ "model_itpm_limit": {"bedrock_mantle/claude-opus": 100000},
+ "model_otpm_limit": {"bedrock_mantle/claude-opus": 5},
+ },
+ )
+
+ data = {
+ "model": "bedrock_mantle/claude-opus",
+ "messages": [
+ {"role": "user", "content": "hello there, this is a test message"}
+ ],
+ "max_tokens": 60, # blows past the 5-token OTPM limit
+ }
+
+ tpm_counter_key = handler.create_rate_limit_keys(
+ key="model_per_project",
+ value="proj-double-refund:bedrock_mantle/claude-opus",
+ rate_limit_type="tokens",
+ )
+
+ with pytest.raises(Exception) as exc_info:
+ await handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=cache,
+ data=data,
+ call_type="",
+ )
+ assert getattr(exc_info.value, "status_code", None) == 429
+
+ tpm_after_pre_call = int(
+ await cache.async_get_cache(key=tpm_counter_key, local_only=True) or 0
+ )
+ assert tpm_after_pre_call == 0, (
+ f"combined TPM reservation not rolled back: {tpm_after_pre_call}"
+ )
+
+ # In the real request lifecycle, async_post_call_failure_hook fires next
+ # for a pre-call rejection. It must not refund the same reservation again.
+ await handler.async_post_call_failure_hook(
+ request_data=data,
+ original_exception=exc_info.value,
+ user_api_key_dict=user_api_key_dict,
+ )
+
+ tpm_after_failure_hook = int(
+ await cache.async_get_cache(key=tpm_counter_key, local_only=True) or 0
+ )
+ assert tpm_after_failure_hook == 0, (
+ f"combined TPM counter went negative from a double refund: {tpm_after_failure_hook}"
+ )
+
+
+@pytest.mark.parametrize(
+ "embedding_input",
+ [
+ list(range(51)),
+ [list(range(25)), list(range(26))],
+ ],
+)
+@pytest.mark.asyncio
+async def test_project_itpm_rejects_pretokenized_embedding_input(
+ rate_limiter,
+ embedding_input,
+):
+ handler, cache = rate_limiter
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=hash_token("sk-pretokenized-embedding-itpm"),
+ project_id="proj-pretokenized-embedding",
+ project_metadata={
+ "model_itpm_limit": {"text-embedding-3-small": 50},
+ },
+ )
+ data = {
+ "model": "text-embedding-3-small",
+ "input": embedding_input,
+ }
+
+ with pytest.raises(Exception) as exc_info:
+ await handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=cache,
+ data=data,
+ call_type="aembedding",
+ )
+
+ assert getattr(exc_info.value, "status_code", None) == 429
+
+
+@pytest.mark.asyncio
+async def test_responses_api_not_misclassified_as_embedding_for_output_estimate(
+ rate_limiter,
+):
+ """
+ Regression for a High-severity review finding: the Responses API also
+ puts its prompt in data["input"], the same field embeddings use, so the
+ output-token estimate treated every Responses call as an embedding and
+ reserved zero output tokens. call_type now disambiguates the two: the
+ same input-only payload gets zero output tokens for an embedding call
+ but a real floor for a Responses API call.
+ """
+ handler, _cache = rate_limiter
+
+ data = {"input": "describe this image in detail"}
+
+ _, embedding_output_estimate = handler._estimate_input_and_output_tokens(
+ data=data, call_type="aembedding"
+ )
+ assert embedding_output_estimate == 0
+
+ _, responses_output_estimate = handler._estimate_input_and_output_tokens(
+ data=data, call_type="aresponses"
+ )
+ assert responses_output_estimate > 0, (
+ "Responses API call was misclassified as an embedding and reserved zero output tokens"
+ )
+
+
+@pytest.mark.parametrize(
+ ("data", "call_type", "expected_output_tokens"),
+ [
+ (
+ {
+ "messages": [{"role": "user", "content": "hello"}],
+ "max_tokens": 100,
+ "n": 10,
+ },
+ "acompletion",
+ 1000,
+ ),
+ (
+ {
+ "prompt": "hello",
+ "max_tokens": 100,
+ "n": 2,
+ "best_of": 5,
+ },
+ "text_completion",
+ 500,
+ ),
+ (
+ {
+ "messages": [{"role": "user", "content": "hello"}],
+ "max_tokens": 100,
+ "n": 0,
+ "best_of": "invalid",
+ },
+ "acompletion",
+ 100,
+ ),
+ ],
+)
+def test_output_estimate_accounts_for_completion_candidates(
+ rate_limiter,
+ data,
+ call_type,
+ expected_output_tokens,
+):
+ handler, _cache = rate_limiter
+
+ _, estimated_output_tokens = handler._estimate_input_and_output_tokens(
+ data=data,
+ call_type=call_type,
+ )
+
+ assert estimated_output_tokens == expected_output_tokens
+
+
+@pytest.mark.asyncio
+async def test_responses_api_usage_reconciles_using_input_output_tokens_fields(
+ rate_limiter,
+):
+ """
+ Regression for the other half of the same finding: ResponseAPIUsage
+ exposes input_tokens/output_tokens, not prompt_tokens/completion_tokens.
+ Before this fix, _resolve_io_token_reconcile_usage couldn't resolve
+ Responses API usage at all, so the reservation was silently kept as-is
+ instead of being trued up to the much larger actual usage.
+ """
+ handler, _cache = rate_limiter
+
+ itpm_scope = ("model_per_project_itpm", "proj-responses:model")
+ otpm_scope = ("model_per_project_otpm", "proj-responses:model")
+
+ stash = get_or_create_request_stash()
+ stash.itpm_reserved_tokens = 10
+ stash.itpm_reserved_scopes = frozenset({itpm_scope})
+ stash.otpm_reserved_tokens = 10
+ stash.otpm_reserved_scopes = frozenset({otpm_scope})
+ mock_kwargs = {}
+
+ mock_response = ResponsesAPIResponse(
+ id="resp_test",
+ created_at=int(datetime.now().timestamp()),
+ output=[],
+ usage=ResponseAPIUsage(input_tokens=80, output_tokens=400, total_tokens=480),
+ )
+
+ increments = []
+
+ async def mock_increment(increment_list, **kwargs):
+ for op in increment_list:
+ increments.append({"key": op["key"], "increment": op["increment_value"]})
+
+ handler.internal_usage_cache.dual_cache.async_increment_cache_pipeline = (
+ mock_increment
+ )
+
+ await handler.async_log_success_event(
+ kwargs=mock_kwargs,
+ response_obj=mock_response,
+ start_time=datetime.now(),
+ end_time=datetime.now(),
+ )
+
+ itpm_adjustments = [i for i in increments if "model_per_project_itpm" in i["key"]]
+ otpm_adjustments = [i for i in increments if "model_per_project_otpm" in i["key"]]
+
+ # delta = 80 actual input - 10 reserved = +70
+ assert any(i["increment"] == 70 for i in itpm_adjustments), (
+ f"ITPM reservation was never trued up to actual Responses API usage: {itpm_adjustments}"
+ )
+ # delta = 400 actual output - 10 reserved = +390
+ assert any(i["increment"] == 390 for i in otpm_adjustments), (
+ f"OTPM reservation was never trued up to actual Responses API usage: {otpm_adjustments}"
+ )
+
+
+@pytest.mark.asyncio
+async def test_itpm_reservation_accounts_for_audio_content_not_just_text(rate_limiter):
+ """
+ Regression for the audio half of a Medium-severity review finding:
+ litellm.token_counter has no per-type handling for `input_audio`
+ content blocks (unlike images, which it does count via
+ use_default_image_token_count), so it silently contributes 0 tokens for
+ them. Without DEFAULT_AUDIO_TOKEN_ESTIMATE, a burst of audio-heavy
+ requests with minimal text would each reserve only the one-token floor
+ and blow past the project ITPM limit before post-call reconciliation.
+ """
+ handler, cache = rate_limiter
+
+ api_key = hash_token("sk-audio-itpm")
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=api_key,
+ project_id="proj-audio",
+ project_metadata={
+ # Tighter than DEFAULT_AUDIO_TOKEN_ESTIMATE (300), but far bigger
+ # than the handful of tokens the bare text "hi" would cost.
+ "model_itpm_limit": {"bedrock_mantle/claude-opus": 50},
+ },
+ )
+
+ data = {
+ "model": "bedrock_mantle/claude-opus",
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "hi"},
+ {
+ "type": "input_audio",
+ "input_audio": {"data": "base64-audio-bytes", "format": "wav"},
+ },
+ ],
+ }
+ ],
+ }
+
+ with pytest.raises(Exception) as exc_info:
+ await handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=cache,
+ data=data,
+ call_type="",
+ )
+ assert getattr(exc_info.value, "status_code", None) == 429, (
+ "Expected the audio content to push the ITPM reservation over the "
+ "50-token limit; if this doesn't raise, audio content isn't being "
+ "counted again."
+ )
+
+
+def test_audio_token_estimate_scales_with_payload_size():
+ """
+ Regression for veria-ai Low finding: audio token reservation was flat
+ 300 per block regardless of duration. A short clip and a long clip both
+ reserved the same amount, letting a caller hide long audio in one block
+ to exhaust ITPM quota while reserving almost nothing.
+
+ The estimate must now grow proportionally with the base64 payload size
+ (len(b64) * 3 // 4 // _AUDIO_BYTES_PER_TOKEN), floored at
+ DEFAULT_AUDIO_TOKEN_ESTIMATE so reference-only blocks and genuinely
+ short clips still get a non-trivial reservation.
+
+ To exceed the floor the decoded payload must be > 300 * 1600 = 480 000
+ bytes. We synthesise a fake b64-length string of 650 000 chars
+ (decoded ≈ 487 500 bytes → 304 tokens) to avoid actually allocating
+ and encoding ~480 kB of audio in every test run.
+ """
+ large_b64 = "A" * 650_000
+ very_large_b64 = "A" * 12_900_000
+ small_b64 = "A" * 1_000
+
+ large_block = {
+ "type": "input_audio",
+ "input_audio": {"data": large_b64, "format": "wav"},
+ }
+ small_block = {
+ "type": "input_audio",
+ "input_audio": {"data": small_b64, "format": "wav"},
+ }
+ very_large_block = {
+ "type": "input_audio",
+ "input_audio": {"data": very_large_b64, "format": "wav"},
+ }
+ no_data_block = {"type": "input_audio", "input_audio": {"format": "wav"}}
+
+ large_estimate = RateLimitHandler._estimate_audio_block_tokens(large_block)
+ very_large_estimate = RateLimitHandler._estimate_audio_block_tokens(
+ very_large_block
+ )
+ small_estimate = RateLimitHandler._estimate_audio_block_tokens(small_block)
+ no_data_estimate = RateLimitHandler._estimate_audio_block_tokens(no_data_block)
+
+ assert large_estimate > small_estimate, (
+ f"Large payload ({large_estimate}) must reserve more than small payload "
+ f"({small_estimate}); flat-rate bug is back"
+ )
+ assert very_large_estimate == len(very_large_b64) * 3 // 4 // _AUDIO_BYTES_PER_TOKEN
+ assert very_large_estimate > 6_000
+ assert no_data_estimate >= 300, (
+ f"Reference-only block (no data) must use the DEFAULT_AUDIO_TOKEN_ESTIMATE floor; got {no_data_estimate}"
+ )
+ assert small_estimate >= 300, (
+ f"Small payload must be floored at DEFAULT_AUDIO_TOKEN_ESTIMATE=300; got {small_estimate}"
+ )
+
+
+@pytest.mark.asyncio
+async def test_itpm_rejects_large_audio_payload_that_would_pass_flat_estimate(
+ rate_limiter,
+):
+ """
+ Regression: a caller placing a long audio clip in one block previously
+ reserved only 300 tokens (the flat estimate). With the size-proportional
+ estimate, the same clip now reserves proportionally more and must trip
+ the ITPM limit when the limit is tuned to exactly expose the difference.
+
+ 1 100 000 b64 chars → decoded ≈ 825 000 bytes → 825 000 // 1600 ≈ 515
+ tokens > the 400-token limit. The flat estimate (300) would have passed.
+ """
+ handler, cache = rate_limiter
+
+ large_b64 = "A" * 1_100_000
+
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=hash_token("sk-large-audio"),
+ project_id="proj-large-audio",
+ project_metadata={
+ "model_itpm_limit": {"bedrock_mantle/claude-opus": 400},
+ },
+ )
+
+ data = {
+ "model": "bedrock_mantle/claude-opus",
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "transcribe this"},
+ {
+ "type": "input_audio",
+ "input_audio": {"data": large_b64, "format": "wav"},
+ },
+ ],
+ }
+ ],
+ }
+
+ with pytest.raises(Exception) as exc_info:
+ await handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=cache,
+ data=data,
+ call_type="",
+ )
+ assert getattr(exc_info.value, "status_code", None) == 429, (
+ "Large audio payload must exceed the 400-token ITPM limit under the "
+ "size-proportional estimate; the old flat-rate estimate (300 tokens) "
+ "would have passed this limit silently"
+ )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ ("call_type", "request_data"),
+ [
+ (
+ "acompletion",
+ {
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "describe this"},
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": "https://example.com/high-resolution.png",
+ "detail": "high",
+ },
+ },
+ ],
+ }
+ ]
+ },
+ ),
+ (
+ "acompletion",
+ {
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "summarize this"},
+ {
+ "type": "file",
+ "file": {
+ "filename": "document.pdf",
+ "file_data": "data:application/pdf;base64,dGVzdA==",
+ },
+ },
+ ],
+ }
+ ]
+ },
+ ),
+ (
+ "aresponses",
+ {
+ "input": [
+ {
+ "role": "user",
+ "content": [
+ {"type": "input_text", "text": "describe this"},
+ {
+ "type": "input_image",
+ "image_url": "https://example.com/high-resolution.png",
+ "detail": "high",
+ },
+ ],
+ }
+ ]
+ },
+ ),
+ (
+ "aresponses",
+ {"input": "continue", "previous_response_id": "resp-123"},
+ ),
+ ],
+)
+async def test_multimodal_requests_reserve_measured_project_itpm_not_full_limit(
+ rate_limiter,
+ call_type,
+ request_data,
+):
+ """
+ Regression: image, file, and previous_response_id requests used to
+ reserve the project's whole ITPM limit up front. Because the atomic
+ check is ``current + increment > limit``, that made every such request
+ 429 as soon as the window carried any usage at all and, while in
+ flight, blocked every other request for the same project + model. They
+ now reserve the token_counter estimate like everything else, so two
+ multimodal requests fit in the same window.
+ """
+ handler, cache = rate_limiter
+ model = "bedrock_mantle/claude-opus"
+ project_itpm_limit = 10_000
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=hash_token("sk-multimodal-measured"),
+ project_id="project-multimodal-measured",
+ project_metadata={"model_itpm_limit": {model: project_itpm_limit}},
+ )
+
+ await handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=cache,
+ data={"model": model, **request_data},
+ call_type=call_type,
+ )
+ first_stash = get_request_stash()
+ assert first_stash is not None
+ first_reservation = first_stash.itpm_reserved_tokens
+ assert 0 < first_reservation < project_itpm_limit // 2
+
+ _request_stash.set(None)
+ await handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=cache,
+ data={"model": model, **request_data},
+ call_type=call_type,
+ )
+ second_stash = get_request_stash()
+ assert second_stash is not None
+ assert second_stash is not first_stash
+ assert second_stash.itpm_reserved_tokens == first_reservation
+
+
+@pytest.mark.asyncio
+async def test_itpm_otpm_reservation_is_kept_on_stream_disconnect(rate_limiter):
+ handler, cache = rate_limiter
+
+ api_key = hash_token("sk-disconnect-test")
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=api_key,
+ project_id="proj-disconnect",
+ project_metadata={
+ "model_itpm_limit": {"bedrock_mantle/claude-opus": 1000},
+ "model_otpm_limit": {"bedrock_mantle/claude-opus": 500},
+ },
+ )
+
+ data: Dict[str, Any] = {
+ "model": "bedrock_mantle/claude-opus",
+ "messages": [{"role": "user", "content": "hello"}],
+ "max_tokens": 50,
+ }
+
+ await handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=cache,
+ data=data,
+ call_type="",
+ )
+
+ stash = get_request_stash()
+ assert stash is not None
+ assert stash.itpm_reserved_tokens > 0, (
+ "pre-call hook must stash an ITPM reservation"
+ )
+ assert stash.otpm_reserved_tokens > 0, (
+ "pre-call hook must stash an OTPM reservation"
+ )
+
+ increment_calls: list[dict] = []
+
+ async def mock_increment(increment_list, litellm_parent_otel_span=None):
+ for op in increment_list:
+ increment_calls.append(
+ {"key": op["key"], "increment": op["increment_value"]}
+ )
+
+ handler.internal_usage_cache.dual_cache.async_increment_cache_pipeline = (
+ mock_increment
+ )
+
+ await handler.async_release_max_parallel_requests_on_disconnect(
+ user_api_key_dict=user_api_key_dict
+ )
+
+ itpm_refunds = [
+ c
+ for c in increment_calls
+ if "model_per_project_itpm" in c["key"] and c["increment"] < 0
+ ]
+ otpm_refunds = [
+ c
+ for c in increment_calls
+ if "model_per_project_otpm" in c["key"] and c["increment"] < 0
+ ]
+
+ assert not itpm_refunds
+ assert not otpm_refunds
+ assert stash.reservation_released is False
+
+
+@pytest.mark.asyncio
+async def test_responses_api_otpm_output_cap_applied_not_skipped_as_embedding(
+ rate_limiter,
+):
+ """
+ Regression for a Greptile P1 finding: _reserve_project_io_tokens_or_raise
+ classified any request with data["input"] set as an embedding (no output
+ tokens), which also misclassifies the Responses API -- it puts its prompt
+ in "input" too, but does generate output. That skipped the output cap
+ applied whenever the configured OTPM limit is small enough to need it,
+ letting an unbounded Responses generation blow past OTPM before
+ post-call reconciliation catches up.
+
+ The cap must land on data["max_output_tokens"], not data["max_tokens"]:
+ the Responses-to-chat-completion transformation only reads
+ max_output_tokens, so a max_tokens cap is silently dropped before
+ provider dispatch (a second Greptile finding on the same code path).
+ """
+ handler, cache = rate_limiter
+
+ api_key = hash_token("sk-responses-otpm-cap")
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=api_key,
+ project_id="proj-responses-otpm",
+ project_metadata={
+ "model_otpm_limit": {"bedrock_mantle/claude-opus": 40},
+ },
+ )
+
+ data: Dict[str, Any] = {
+ "model": "bedrock_mantle/claude-opus",
+ "input": "describe this image in detail",
+ }
+
+ await handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=cache,
+ data=data,
+ call_type="aresponses",
+ )
+
+ assert data.get("max_output_tokens") is not None, (
+ "Responses call was misclassified as an embedding and skipped the OTPM output cap"
+ )
+ assert data["max_output_tokens"] == 16
+ assert data.get("max_tokens") is None, (
+ "OTPM output cap was written to max_tokens, which the Responses transformation ignores"
+ )
+
+
+@pytest.mark.asyncio
+async def test_explicit_zero_output_responses_call_reserves_effective_provider_minimum(
+ rate_limiter,
+):
+ handler, cache = rate_limiter
+
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=hash_token("sk-responses-zero-output"),
+ project_id="proj-responses-zero-output",
+ project_metadata={
+ "model_otpm_limit": {"bedrock_mantle/claude-opus": 5},
+ },
+ )
+
+ with pytest.raises(Exception) as exc_info:
+ await handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=cache,
+ data={
+ "model": "bedrock_mantle/claude-opus",
+ "input": "describe this image in detail",
+ "max_output_tokens": 0,
+ },
+ call_type="aresponses",
+ )
+
+ assert getattr(exc_info.value, "status_code", None) == 429
+
+
+@pytest.mark.asyncio
+async def test_responses_api_combined_tpm_output_cap_applied_not_skipped_as_embedding(
+ rate_limiter,
+):
+ """
+ Regression for the same misclassification bug in the combined-TPM
+ output-cap block of async_pre_call_hook (a second, independent
+ `is_embedding = data.get("input") is not None` check). A project with
+ only a combined model_tpm_limit (no split itpm/otpm) configured small
+ enough to need the output cap must still apply it to a Responses call,
+ and must write it to max_output_tokens for the same reason as the OTPM
+ case above.
+ """
+ handler, cache = rate_limiter
+
+ api_key = hash_token("sk-responses-tpm-cap")
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=api_key,
+ project_id="proj-responses-tpm",
+ project_metadata={
+ "model_tpm_limit": {"bedrock_mantle/claude-opus": 40},
+ },
+ )
+
+ data: Dict[str, Any] = {
+ "model": "bedrock_mantle/claude-opus",
+ "input": "describe this image in detail",
+ }
+
+ await handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=cache,
+ data=data,
+ call_type="aresponses",
+ )
+
+ assert data.get("max_output_tokens") is not None, (
+ "Responses call was misclassified as an embedding and skipped the combined-TPM output cap"
+ )
+ assert data["max_output_tokens"] == 16
+ assert data.get("max_tokens") is None, (
+ "combined-TPM output cap was written to max_tokens, which the Responses transformation ignores"
+ )
+
+
+@pytest.mark.asyncio
+async def test_responses_api_multimodal_input_counts_image_content(rate_limiter):
+ """
+ Regression for a Low-severity veria-ai finding: the Responses API's
+ `input` is commonly a list of message/content-block dicts, but
+ litellm.token_counter's `text` argument only joins plain string entries
+ in a list and silently drops everything else -- so an `input_image`
+ block contributed ~0 tokens to the ITPM estimate instead of the real
+ image token count. _estimate_precise_input_tokens now converts Responses
+ `input` to chat messages first (via the standard
+ transform_responses_api_input_to_messages helper) so image content is
+ counted the same way a chat completion's image content already is.
+ """
+ handler, _cache = rate_limiter
+
+ text_only_estimate = handler._estimate_precise_input_tokens(
+ data={"input": "hi"},
+ model="bedrock_mantle/claude-opus",
+ call_type="aresponses",
+ )
+
+ multimodal_estimate = handler._estimate_precise_input_tokens(
+ data={
+ "input": [
+ {
+ "role": "user",
+ "content": [
+ {"type": "input_text", "text": "hi"},
+ {
+ "type": "input_image",
+ "image_url": "https://example.com/some-image.png",
+ },
+ ],
+ }
+ ],
+ },
+ model="bedrock_mantle/claude-opus",
+ call_type="aresponses",
+ )
+
+ assert multimodal_estimate > text_only_estimate + 100, (
+ "Responses API input_image content block was not counted; got "
+ f"text_only={text_only_estimate}, multimodal={multimodal_estimate}"
+ )
+
+
+@pytest.mark.asyncio
+async def test_refund_reserved_tokens_noop_when_amount_zero(rate_limiter):
+ """_refund_reserved_tokens returns immediately without calling Redis when amount=0."""
+ handler, _cache = rate_limiter
+
+ calls = []
+
+ async def mock_increment(pipeline_operations, **kwargs):
+ calls.extend(pipeline_operations)
+
+ handler.async_increment_tokens_with_ttl_preservation = mock_increment
+
+ await handler._refund_reserved_tokens(
+ scopes=[("api_key", "sk-test")],
+ amount=0,
+ )
+
+ assert not calls, "No Redis ops expected when amount is zero"
+
+
+@pytest.mark.asyncio
+async def test_reserve_io_tokens_noop_when_no_itpm_otpm_descriptors(rate_limiter):
+ """reserve_io_tokens returns OK immediately when no ITPM/OTPM descriptors present."""
+ handler, _cache = rate_limiter
+
+ non_io_descriptor = {
+ "key": "api_key",
+ "value": "sk-test",
+ "rate_limit": {"tokens_per_unit": 1000, "window_size": 60},
+ }
+ response, itpm_reserved, otpm_reserved = await handler.reserve_io_tokens(
+ descriptors=[non_io_descriptor],
+ estimated_input_tokens=50,
+ estimated_output_tokens=50,
+ )
+
+ assert response["overall_code"] == "OK"
+ assert itpm_reserved == 0
+ assert otpm_reserved == 0
+
+
+@pytest.mark.asyncio
+async def test_reserve_io_tokens_itpm_only_no_otpm(rate_limiter):
+ """When only ITPM descriptors are present (no OTPM), returns itpm_reserved with otpm=0."""
+ handler, cache = rate_limiter
+
+ itpm_descriptor = {
+ "key": PROJECT_ITPM_DESCRIPTOR_KEY,
+ "value": "proj-a:model",
+ "rate_limit": {"tokens_per_unit": 10000, "window_size": 60},
+ }
+ response, itpm_reserved, otpm_reserved = await handler.reserve_io_tokens(
+ descriptors=[itpm_descriptor],
+ estimated_input_tokens=100,
+ estimated_output_tokens=50,
+ )
+
+ assert response["overall_code"] == "OK"
+ assert itpm_reserved == 100
+ assert otpm_reserved == 0
+
+
+def test_strip_audio_content_blocks_passthrough_non_list_messages():
+ """Non-list input is returned unchanged (early return on line 2605)."""
+ result = RateLimitHandler._strip_audio_content_blocks("not a list")
+ assert result == "not a list"
+
+
+def test_strip_audio_content_blocks_passthrough_non_dict_message():
+ """Non-dict entries in the message list are appended unchanged."""
+ messages = ["plain string message"]
+ result = RateLimitHandler._strip_audio_content_blocks(messages)
+ assert result == ["plain string message"]
+
+
+def test_strip_audio_content_blocks_passthrough_non_list_content():
+ """Messages with non-list content (e.g. plain string) pass through unchanged."""
+ messages = [{"role": "user", "content": "hello"}]
+ result = RateLimitHandler._strip_audio_content_blocks(messages)
+ assert result == [{"role": "user", "content": "hello"}]
+
+
+@pytest.mark.asyncio
+async def test_otpm_rejection_releases_stashed_parallel_slot(rate_limiter):
+ """
+ When OTPM is over limit and a parallel slot was already acquired, the
+ disconnect cleanup path in _reserve_project_io_tokens_or_raise must
+ release that slot. Exercises lines 2773-2777.
+ """
+ handler, cache = rate_limiter
+
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=hash_token("sk-otpm-slot"),
+ project_id="proj-slot",
+ project_metadata={"model_otpm_limit": {"m": 5}},
+ )
+
+ data: Dict[str, Any] = {
+ "model": "m",
+ "messages": [{"role": "user", "content": "hello"}],
+ "max_tokens": 50,
+ }
+
+ slot_released = []
+
+ async def mock_release(acquisition, parent_otel_span=None):
+ slot_released.append(acquisition)
+
+ handler._release_parallel_request_slots = mock_release
+
+ stash = get_or_create_request_stash()
+ stash.parallel_slot = {
+ "slot_id": "test-slot-id",
+ "counter_keys": ["some-key"],
+ }
+
+ otpm_descriptor = {
+ "key": PROJECT_OTPM_DESCRIPTOR_KEY,
+ "value": "proj-slot:m",
+ "rate_limit": {"tokens_per_unit": 5, "window_size": 60},
+ }
+
+ with pytest.raises(Exception) as exc_info:
+ await handler._reserve_project_io_tokens_or_raise(
+ descriptors=[otpm_descriptor],
+ data=data,
+ requested_model="m",
+ user_api_key_dict=user_api_key_dict,
+ tpm_reservation_scopes=[],
+ tpm_reservation_amount=0,
+ )
+ assert getattr(exc_info.value, "status_code", None) == 429
+ assert slot_released, "Parallel slot must be released when OTPM rejects"
+ assert stash.parallel_slot is None
+
+
+@pytest.mark.asyncio
+async def test_itpm_only_status_stored_when_no_prior_rate_limit_response(rate_limiter):
+ """
+ When only ITPM is configured (no combined TPM/RPM to pre-populate
+ the request stash), a successful ITPM reservation must store its status
+ there so post-call headers can read it.
+ """
+ handler, cache = rate_limiter
+
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=hash_token("sk-itpm-only-store"),
+ project_id="proj-store",
+ )
+
+ data: Dict[str, Any] = {"model": "m", "messages": []}
+
+ itpm_descriptor = {
+ "key": PROJECT_ITPM_DESCRIPTOR_KEY,
+ "value": "proj-store:m",
+ "rate_limit": {"tokens_per_unit": 100000, "window_size": 60},
+ }
+
+ await handler._reserve_project_io_tokens_or_raise(
+ descriptors=[itpm_descriptor],
+ data=data,
+ requested_model="m",
+ user_api_key_dict=user_api_key_dict,
+ tpm_reservation_scopes=[],
+ tpm_reservation_amount=0,
+ )
+
+ stash = get_request_stash()
+ assert stash is not None
+ stored = stash.rate_limit_response
+ assert stored is not None, (
+ "ITPM status must be stored in litellm_proxy_rate_limit_response"
+ )
+ assert stored.get("statuses"), "Stored response must contain statuses"
+
+
+def test_resolve_io_token_usage_responses_api_with_cached_tokens(rate_limiter):
+ """
+ ResponsesAPIResponse whose usage.input_tokens_details.cached_tokens is set
+ subtracts the cached portion from billable input. Covers line 3501.
+ """
+ handler, _cache = rate_limiter
+
+ response_obj = ResponsesAPIResponse(
+ id="resp_cached",
+ created_at=int(datetime.now().timestamp()),
+ output=[],
+ usage=ResponseAPIUsage(
+ input_tokens=100,
+ output_tokens=50,
+ total_tokens=150,
+ input_tokens_details=InputTokensDetails(cached_tokens=25),
+ ),
+ )
+ billable_input, completion_tokens, resolved = (
+ handler._resolve_io_token_reconcile_usage(response_obj)
+ )
+
+ assert resolved is True
+ assert billable_input == 75, f"Expected 100 - 25 cached = 75, got {billable_input}"
+ assert completion_tokens == 50
+
+
+def test_resolve_io_token_usage_dict_format(rate_limiter):
+ """
+ Dict-shaped usage on a ModelResponse (older SDK versions or raw dicts in
+ the usage field) is parsed correctly. Covers lines 3502-3506.
+ """
+ handler, _cache = rate_limiter
+
+ response_obj = ModelResponse.model_construct(
+ usage={
+ "prompt_tokens": 80,
+ "completion_tokens": 40,
+ "prompt_tokens_details": {"cached_tokens": 20},
+ }
+ )
+ billable_input, completion_tokens, resolved = (
+ handler._resolve_io_token_reconcile_usage(response_obj)
+ )
+
+ assert resolved is True
+ assert billable_input == 60, f"Expected 80 - 20 cached = 60, got {billable_input}"
+ assert completion_tokens == 40
+
+
+def test_resolve_io_token_usage_unknown_type_returns_unresolved(rate_limiter):
+ """
+ A ModelResponse whose usage attribute is not a Usage, ResponseAPIUsage,
+ or dict (e.g. a plain int) returns (0, 0, False) so the reservation is
+ kept rather than guessed. Covers lines 3507-3508.
+ """
+ handler, _cache = rate_limiter
+
+ response_obj = ModelResponse.model_construct(usage=42)
+ billable_input, completion_tokens, resolved = (
+ handler._resolve_io_token_reconcile_usage(response_obj)
+ )
+
+ assert resolved is False
+ assert billable_input == 0
+ assert completion_tokens == 0
+
+
+@pytest.mark.parametrize(
+ ("combined_usage", "expected_increments"),
+ [
+ (None, ()),
+ (
+ Usage(prompt_tokens=40, completion_tokens=15, total_tokens=55),
+ (-60, -45),
+ ),
+ ],
+)
+def test_zero_usage_keeps_reservations_unless_measured_fallback_exists(
+ rate_limiter,
+ combined_usage,
+ expected_increments,
+):
+ handler, _cache = rate_limiter
+ itpm_scope = (PROJECT_ITPM_DESCRIPTOR_KEY, "project:model")
+ otpm_scope = (PROJECT_OTPM_DESCRIPTOR_KEY, "project:model")
+ stash = get_or_create_request_stash()
+ stash.itpm_reserved_tokens = 100
+ stash.itpm_reserved_scopes = frozenset({itpm_scope})
+ stash.otpm_reserved_tokens = 60
+ stash.otpm_reserved_scopes = frozenset({otpm_scope})
+ kwargs = {} if combined_usage is None else {"combined_usage_object": combined_usage}
+ response_obj = ModelResponse(
+ usage=Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0)
+ )
+
+ operations = handler._build_io_token_reservation_ops(kwargs, response_obj)
+
+ assert tuple(operation["increment_value"] for operation in operations) == expected_increments
+
+
+@pytest.mark.parametrize(
+ ("usage", "expected_increments"),
+ [
+ (
+ Usage(prompt_tokens=40, completion_tokens=15, total_tokens=55),
+ (40, 15),
+ ),
+ (
+ Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0),
+ (100, 60),
+ ),
+ ],
+)
+def test_retry_success_charges_released_project_io_reservations(
+ rate_limiter,
+ usage,
+ expected_increments,
+):
+ handler, _cache = rate_limiter
+ itpm_scope = (PROJECT_ITPM_DESCRIPTOR_KEY, "project:model")
+ otpm_scope = (PROJECT_OTPM_DESCRIPTOR_KEY, "project:model")
+ stash = get_or_create_request_stash()
+ stash.itpm_reserved_tokens = 100
+ stash.itpm_reserved_scopes = frozenset({itpm_scope})
+ stash.otpm_reserved_tokens = 60
+ stash.otpm_reserved_scopes = frozenset({otpm_scope})
+ stash.reservation_released = True
+
+ operations = handler._build_io_token_reservation_ops(
+ {},
+ ModelResponse(usage=usage),
+ )
+
+ assert tuple(operation["increment_value"] for operation in operations) == expected_increments
+
+
+@pytest.mark.asyncio
+async def test_build_io_token_reservation_ops_skips_unresolvable_usage(rate_limiter):
+ """
+ When response_obj has no parseable usage, _build_io_token_reservation_ops
+ returns [] to keep the reservation as-is rather than zeroing it out on a
+ bad guess. Covers line 3538.
+ """
+ handler, _cache = rate_limiter
+
+ itpm_scope = ("model_per_project_itpm", "proj-b:model")
+ stash = get_or_create_request_stash()
+ stash.itpm_reserved_tokens = 50
+ stash.itpm_reserved_scopes = frozenset({itpm_scope})
+ mock_kwargs = {}
+
+ ops = handler._build_io_token_reservation_ops(
+ kwargs=mock_kwargs,
+ response_obj=object(),
+ )
+
+ assert not ops, f"Expected empty ops for unresolvable usage, got {ops}"
+
+
+@pytest.mark.asyncio
+async def test_post_call_failure_skips_rpm_only_descriptor_in_tpm_refund(rate_limiter):
+ """
+ async_post_call_failure_hook skips descriptors without tokens_per_unit
+ (e.g. an RPM-only api_key scope) when building the combined-TPM refund ops,
+ so a key with rpm_limit but no tpm_limit doesn't receive a spurious refund
+ that would drive its counter negative. Covers the continue guard at line 4250.
+ """
+ handler, cache = rate_limiter
+
+ api_key = hash_token("sk-rpm-only-desc")
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=api_key,
+ rpm_limit=100,
+ project_id="proj-rpm-only-desc",
+ project_metadata={"model_tpm_limit": {"gpt-3.5-turbo": 100000}},
+ )
+
+ data = {
+ "model": "gpt-3.5-turbo",
+ "messages": [{"role": "user", "content": "hello"}],
+ "max_tokens": 20,
+ }
+
+ await handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=cache,
+ data=data,
+ call_type="",
+ )
+
+ rpm_tokens_key = handler.create_rate_limit_keys(
+ key="api_key", value=api_key, rate_limit_type="tokens"
+ )
+
+ await handler.async_post_call_failure_hook(
+ request_data=data,
+ original_exception=Exception("rejected"),
+ user_api_key_dict=user_api_key_dict,
+ )
+
+ api_key_tokens_after = int(
+ await cache.async_get_cache(key=rpm_tokens_key, local_only=True) or 0
+ )
+ assert api_key_tokens_after >= 0, (
+ f"RPM-only api_key scope must not receive a negative TPM refund; got {api_key_tokens_after}"
+ )
+
+
+@pytest.mark.asyncio
+async def test_max_output_tokens_prevents_cap_injection(rate_limiter):
+ """
+ Regression for veria-ai comment: when a Responses API request supplies
+ max_output_tokens (the canonical Responses output bound) but not max_tokens
+ or max_completion_tokens, the has_explicit_max_tokens check was False, so
+ the code injected data["max_tokens"] = capped_floor and silently truncated
+ the response.
+
+ With the fix, max_output_tokens is included in the explicit-cap check and
+ data["max_tokens"] must NOT be injected when max_output_tokens is already
+ set.
+ """
+ handler, cache = rate_limiter
+
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=hash_token("sk-max-output-tokens"),
+ project_id="proj-responses-max-output",
+ project_metadata={
+ "model_otpm_limit": {"mock-model": 100},
+ },
+ )
+
+ data: dict = {
+ "model": "mock-model",
+ "input": "Summarise the document",
+ "max_output_tokens": 80,
+ "litellm_call_id": "test-max-output-tokens",
+ "metadata": {},
+ }
+
+ try:
+ await handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=cache,
+ data=data,
+ call_type="responses",
+ )
+ except Exception:
+ pass
+
+ assert "max_tokens" not in data, (
+ "data['max_tokens'] must not be injected when max_output_tokens is already "
+ "set; the cap injection was overriding the caller's explicit output bound"
+ )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ ("call_type", "request_data", "cap_field", "reserved_tokens"),
+ [
+ ("aresponses", {"input": "hello", "max_tokens": 1}, "max_output_tokens", 16),
+ (
+ "acompletion",
+ {
+ "messages": [{"role": "user", "content": "hello"}],
+ "max_output_tokens": 1,
+ },
+ "max_tokens",
+ 10,
+ ),
+ ],
+)
+async def test_output_reservation_ignores_cap_fields_from_other_endpoints(
+ rate_limiter,
+ call_type,
+ request_data,
+ cap_field,
+ reserved_tokens,
+):
+ handler, cache = rate_limiter
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=hash_token(f"sk-{call_type}"),
+ project_id=f"project-{call_type}",
+ project_metadata={"model_otpm_limit": {"model": 40}},
+ )
+ data = {"model": "model", **request_data}
+
+ await handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=cache,
+ data=data,
+ call_type=call_type,
+ )
+
+ assert data[cap_field] == reserved_tokens
+ stash = get_request_stash()
+ assert stash is not None
+ assert stash.otpm_reserved_tokens == reserved_tokens
+
+
+def test_responses_input_is_counted_even_when_messages_is_present(rate_limiter):
+ handler, _cache = rate_limiter
+ small_estimate = handler._estimate_precise_input_tokens(
+ data={"input": "short", "messages": [{"role": "user", "content": "ignored"}]},
+ model="",
+ call_type="aresponses",
+ )
+ large_estimate = handler._estimate_precise_input_tokens(
+ data={"input": "large input " * 500, "messages": []},
+ model="",
+ call_type="aresponses",
+ )
+
+ assert large_estimate > small_estimate
+
+
+def test_anthropic_messages_usage_reconciles_split_project_quota(rate_limiter):
+ handler, _cache = rate_limiter
+
+ billable_input, output_tokens, resolved = handler._resolve_io_token_reconcile_usage(
+ {
+ "usage": {
+ "input_tokens": 100,
+ "output_tokens": 25,
+ "cache_read_input_tokens": 30,
+ }
+ }
+ )
+
+ assert resolved is True
+ assert billable_input == 70
+ assert output_tokens == 25
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "call_type",
+ ["agenerate_content", "agenerate_content_stream"],
+)
+async def test_google_genai_native_contents_reserve_project_itpm(
+ rate_limiter,
+ call_type,
+):
+ handler, cache = rate_limiter
+ model = "gemini/gemini-2.5-flash"
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=hash_token("sk-google-genai-native-itpm"),
+ project_id="project-google-genai-native-itpm",
+ project_metadata={"model_itpm_limit": {model: 10_000}},
+ )
+
+ await handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=cache,
+ data={
+ "model": model,
+ "contents": [
+ {
+ "role": "user",
+ "parts": [{"text": "Gemini quota input " * 200}],
+ }
+ ],
+ },
+ call_type=call_type,
+ )
+
+ stash = get_request_stash()
+ assert stash is not None
+ assert stash.itpm_reserved_tokens > 100
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("call_type", ["rerank", "arerank"])
+async def test_rerank_query_and_documents_enforce_project_itpm(
+ rate_limiter,
+ monkeypatch,
+ call_type,
+):
+ handler, cache = rate_limiter
+ captured = {}
+
+ def token_counter(**kwargs):
+ captured.update(kwargs)
+ return 101
+
+ monkeypatch.setattr("litellm.token_counter", token_counter)
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=hash_token(f"sk-{call_type}-itpm"),
+ project_id=f"project-{call_type}-itpm",
+ project_metadata={"model_itpm_limit": {"rerank-model": 100}},
+ )
+
+ with pytest.raises(Exception) as exc_info:
+ await handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=cache,
+ data={
+ "model": "rerank-model",
+ "query": "Which document is most relevant?",
+ "documents": ["first document", {"text": "second document"}],
+ },
+ call_type=call_type,
+ )
+
+ assert getattr(exc_info.value, "status_code", None) == 429
+ assert captured["text"] == (
+ "Which document is most relevant?\n"
+ "first document\n"
+ "{'text': 'second document'}"
+ )
+
+
+def test_rerank_input_estimate_falls_back_to_character_count(
+ rate_limiter,
+ monkeypatch,
+):
+ handler, _cache = rate_limiter
+ data = {
+ "query": "query text",
+ "documents": ["first document", "second document"],
+ }
+
+ def token_counter(**_kwargs):
+ raise ValueError("tokenizer unavailable")
+
+ monkeypatch.setattr("litellm.token_counter", token_counter)
+ rerank_text = handler._rerank_input_to_text(data)
+
+ assert handler._estimate_precise_input_tokens(
+ data,
+ model="custom-rerank-model",
+ call_type="rerank",
+ ) == len(rerank_text) // 4
+
+
+@pytest.mark.parametrize(
+ ("response_obj", "expected"),
+ [
+ (
+ RerankResponse(
+ meta={"tokens": {"input_tokens": 42, "output_tokens": 3}}
+ ),
+ (42, 3, True),
+ ),
+ (
+ RerankResponse(
+ meta={
+ "tokens": {"input_tokens": 0, "output_tokens": 0},
+ "billed_units": {"total_tokens": 57},
+ }
+ ),
+ (57, 0, True),
+ ),
+ (
+ RerankResponse(
+ meta={
+ "tokens": {"input_tokens": 0, "output_tokens": 0},
+ "billed_units": {"total_tokens": 0},
+ }
+ ),
+ (0, 0, False),
+ ),
+ ],
+)
+def test_rerank_usage_reconciles_project_split_token_quota(
+ rate_limiter,
+ response_obj,
+ expected,
+):
+ handler, _cache = rate_limiter
+
+ assert handler._resolve_io_token_reconcile_usage(response_obj) == expected
+
+
+def test_split_quota_helpers_handle_non_mapping_inputs(rate_limiter):
+ handler, _cache = rate_limiter
+
+ assert _call_id_from_callback_kwargs(object()) is None
+ assert handler._is_embedding_request(object(), None) is False
+ assert handler._get_explicit_output_cap(object(), None) is None
+ assert handler.get_output_candidate_count(object()) == 1
+ assert handler.get_output_candidate_count({"n": 1e309}) == 1
+ assert (
+ handler._get_explicit_output_cap({"max_output_tokens": []}, "responses") is None
+ )
+ assert handler._apply_implicit_output_cap(object(), 100, "responses") is None
+ assert handler._estimate_input_and_output_tokens(object()) == (0, 0)
+ assert handler._build_io_token_reservation_ops(object(), object()) == ()
+
+
+@pytest.mark.parametrize(
+ ("data", "call_type", "expected"),
+ [
+ ({"max_tokens": "30.0"}, "", 30),
+ ({"max_tokens": "not-a-number"}, "", None),
+ ({"max_tokens": True}, "", None),
+ ({"max_output_tokens": "30.0"}, "responses", 30),
+ ({"max_output_tokens": "nan"}, "responses", None),
+ ({"generationConfig": {"maxOutputTokens": "12.5"}}, "agenerate_content", 12),
+ ({"generationConfig": {"maxOutputTokens": "oops"}}, "agenerate_content", None),
+ ],
+)
+def test_get_explicit_output_cap_tolerates_unparseable_values(
+ rate_limiter, data, call_type, expected
+):
+ """A client-supplied cap the proxy cannot parse must fall back to the
+ no-cap output estimate instead of raising ValueError and 500ing the
+ request before it ever reaches the provider."""
+ handler, _cache = rate_limiter
+
+ assert handler._get_explicit_output_cap(data, call_type) == expected
+
+
+@pytest.mark.asyncio
+async def test_project_io_counters_not_double_charged_when_reservation_disabled(
+ monkeypatch,
+):
+ """With LITELLM_TPM_TOKEN_RESERVATION_ENABLED=false the first
+ should_rate_limit pass used to +1 every ITPM/OTPM counter on top of the
+ full reservation _reserve_project_io_tokens_or_raise always makes,
+ permanently inflating each bucket by one token per request."""
+ monkeypatch.setenv("LITELLM_TPM_TOKEN_RESERVATION_ENABLED", "false")
+ cache = DualCache()
+ handler = RateLimitHandler(internal_usage_cache=InternalUsageCache(cache))
+ assert handler.tpm_reservation_enabled is False
+
+ user_api_key_dict = UserAPIKeyAuth(
+ api_key=hash_token("sk-io-no-reservation"),
+ project_id="proj-io-no-reservation",
+ project_metadata={
+ "model_itpm_limit": {"bedrock_mantle/claude-opus": 1000000},
+ "model_otpm_limit": {"bedrock_mantle/claude-opus": 1000000},
+ },
+ )
+ data = {
+ "model": "bedrock_mantle/claude-opus",
+ "messages": [{"role": "user", "content": "hello"}],
+ "max_tokens": 50,
+ }
+
+ await handler.async_pre_call_hook(
+ user_api_key_dict=user_api_key_dict,
+ cache=cache,
+ data=data,
+ call_type="",
+ )
+
+ stash = get_request_stash()
+ assert stash is not None
+ assert stash.itpm_reserved_tokens > 0
+ assert stash.otpm_reserved_tokens > 0
+
+ for descriptor_key, reserved in (
+ ("model_per_project_itpm", stash.itpm_reserved_tokens),
+ ("model_per_project_otpm", stash.otpm_reserved_tokens),
+ ):
+ counter_key = handler.create_rate_limit_keys(
+ key=descriptor_key,
+ value="proj-io-no-reservation:bedrock_mantle/claude-opus",
+ rate_limit_type="tokens",
+ )
+ cached = await cache.async_get_cache(key=counter_key, local_only=True)
+ assert int(cached or 0) == reserved, (
+ f"{descriptor_key} counter {cached} != reserved {reserved}: "
+ "first-pass should_rate_limit double-charged the bucket"
+ )
+
+
+@pytest.mark.parametrize(
+ ("call_type", "data"),
+ [
+ (
+ "text_completion",
+ {
+ "messages": [{"role": "user", "content": "ignored"}],
+ "prompt": "abcd",
+ "input": "ignored",
+ "max_tokens": 1,
+ },
+ ),
+ (None, {"prompt": "abcd", "max_tokens": 1}),
+ (None, {"prompt": ["abcd", "efgh"], "max_tokens": 1}),
+ ],
+)
+def test_split_token_estimate_selects_endpoint_input(rate_limiter, call_type, data):
+ handler, _cache = rate_limiter
+
+ estimated_input, estimated_output = handler._estimate_input_and_output_tokens(
+ data=data,
+ call_type=call_type,
+ )
+
+ assert estimated_input > 0
+ assert estimated_output == 1
+
+
+def test_split_quota_multimodal_guards_handle_non_mapping_inputs(rate_limiter):
+ handler, _cache = rate_limiter
+
+ assert handler._estimate_audio_block_tokens(
+ object()
+ ) == handler._estimate_audio_block_tokens({})
+ assert handler._responses_input_to_chat_messages(object()) == ()
+ assert handler._estimate_precise_input_tokens(object(), model=None) == 0
+
+
+@pytest.mark.parametrize(
+ ("call_type", "data", "expected_text"),
+ [
+ ("embedding", {"input": "embedding input"}, "embedding input"),
+ (
+ "embedding",
+ {"input": ["first embedding", "second embedding"]},
+ ["first embedding", "second embedding"],
+ ),
+ ("text_completion", {"prompt": "completion prompt"}, "completion prompt"),
+ ],
+)
+def test_precise_input_estimate_selects_endpoint_text(
+ rate_limiter,
+ monkeypatch,
+ call_type,
+ data,
+ expected_text,
+):
+ handler, _cache = rate_limiter
+ captured = {}
+
+ def token_counter(**kwargs):
+ captured.update(kwargs)
+ return 7
+
+ monkeypatch.setattr("litellm.token_counter", token_counter)
+
+ assert (
+ handler._estimate_precise_input_tokens(data, model="test", call_type=call_type)
+ == 7
+ )
+ assert captured["messages"] is None
+ assert captured["text"] == expected_text
+
+
+@pytest.mark.asyncio
+async def test_project_io_reservation_ignores_non_mapping_request_data(rate_limiter):
+ handler, _cache = rate_limiter
+
+ await handler._reserve_project_io_tokens_or_raise(
+ descriptors=[],
+ data=object(),
+ requested_model=None,
+ user_api_key_dict=UserAPIKeyAuth(),
+ tpm_reservation_scopes=(),
+ tpm_reservation_amount=0,
+ )
+
+
+@pytest.mark.asyncio
+async def test_streaming_combined_usage_reconciles_project_io_reservations(
+ rate_limiter,
+):
+ handler, _cache = rate_limiter
+ itpm_scope = (PROJECT_ITPM_DESCRIPTOR_KEY, "project:model")
+ otpm_scope = (PROJECT_OTPM_DESCRIPTOR_KEY, "project:model")
+ stash = get_or_create_request_stash()
+ stash.itpm_reserved_tokens = 100
+ stash.itpm_reserved_scopes = frozenset({itpm_scope})
+ stash.otpm_reserved_tokens = 60
+ stash.otpm_reserved_scopes = frozenset({otpm_scope})
+ kwargs = {
+ "combined_usage_object": Usage(
+ prompt_tokens=40,
+ completion_tokens=15,
+ total_tokens=55,
+ ),
+ }
+ increments = []
+
+ async def capture_increments(increment_list, **_kwargs):
+ increments.extend(increment_list)
+
+ handler.internal_usage_cache.dual_cache.async_increment_cache_pipeline = (
+ capture_increments
+ )
+
+ await handler.async_log_success_event(
+ kwargs=kwargs,
+ response_obj={"response": "stream body"},
+ start_time=datetime.now(),
+ end_time=datetime.now(),
+ )
+
+ itpm_adjustments = [
+ operation
+ for operation in increments
+ if PROJECT_ITPM_DESCRIPTOR_KEY in operation["key"]
+ ]
+ otpm_adjustments = [
+ operation
+ for operation in increments
+ if PROJECT_OTPM_DESCRIPTOR_KEY in operation["key"]
+ ]
+ assert [operation["increment_value"] for operation in itpm_adjustments] == [-60]
+ assert [operation["increment_value"] for operation in otpm_adjustments] == [-45]
+
+
+def test_aggregate_only_combined_usage_reconciles_project_io_reservations(rate_limiter):
+ handler, _cache = rate_limiter
+ stash = get_or_create_request_stash()
+ stash.itpm_reserved_tokens = 100
+ stash.itpm_reserved_scopes = frozenset(
+ {(PROJECT_ITPM_DESCRIPTOR_KEY, "project:model")}
+ )
+ stash.otpm_reserved_tokens = 80
+ stash.otpm_reserved_scopes = frozenset(
+ {(PROJECT_OTPM_DESCRIPTOR_KEY, "project:model")}
+ )
+ kwargs = {
+ "combined_usage_object": Usage(total_tokens=55),
+ }
+
+ operations = handler._build_io_token_reservation_ops(kwargs, object())
+
+ assert [operation["increment_value"] for operation in operations] == [-45, -25]
+
+
+def test_raw_split_usage_dict_reconciles_project_io_tokens(rate_limiter):
+ handler, _cache = rate_limiter
+
+ assert handler._resolve_io_token_reconcile_usage(
+ {
+ "input_tokens": 30,
+ "output_tokens": 12,
+ "input_tokens_details": {"cached_tokens": 5},
+ }
+ ) == (25, 12, True)
+
+
+@pytest.mark.asyncio
+async def test_post_call_success_hook_contains_header_merge_failures(
+ rate_limiter, monkeypatch
+):
+ handler, _cache = rate_limiter
+ response = ModelResponse()
+ response._hidden_params = {}
+
+ def raise_on_merge(**_kwargs):
+ raise RuntimeError("header merge failed")
+
+ monkeypatch.setattr(
+ handler,
+ "_merge_ratelimit_statuses_into_additional_headers",
+ raise_on_merge,
+ )
+
+ await handler.async_post_call_success_hook(
+ data={
+ "litellm_proxy_rate_limit_response": {
+ "overall_code": "OK",
+ "statuses": (),
+ }
+ },
+ user_api_key_dict=UserAPIKeyAuth(),
+ response=response,
+ )
+
+
if __name__ == "__main__":
pytest.main([__file__, "-v", "-s"])
diff --git a/tests/test_litellm/proxy/management_endpoints/test_access_group_management.py b/tests/test_litellm/proxy/management_endpoints/test_access_group_management.py
index 3240ad20edb..c973c6a8346 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_access_group_management.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_access_group_management.py
@@ -13,6 +13,9 @@ sys.path.insert(
) # Adds the parent directory to the system path
from litellm import Router
+from litellm.proxy.management_endpoints.model_management_endpoints import (
+ ReconcileOutcome,
+)
@pytest.mark.asyncio
@@ -121,7 +124,7 @@ async def test_create_access_group_with_model_ids_tags_only_specific_deployments
patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
patch(
"litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache",
- new_callable=AsyncMock,
+ new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)),
),
):
response = await create_model_group(
@@ -186,7 +189,7 @@ async def test_create_access_group_with_model_names_tags_all_deployments():
patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
patch(
"litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache",
- new_callable=AsyncMock,
+ new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)),
),
):
response = await create_model_group(
@@ -236,7 +239,7 @@ async def test_create_access_group_model_ids_takes_priority_over_model_names():
patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
patch(
"litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache",
- new_callable=AsyncMock,
+ new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)),
),
):
response = await create_model_group(
@@ -313,7 +316,7 @@ async def test_create_access_group_invalid_model_id_returns_400():
patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
patch(
"litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache",
- new_callable=AsyncMock,
+ new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)),
),
):
with pytest.raises(HTTPException) as exc_info:
@@ -352,7 +355,7 @@ async def test_create_access_group_surfaces_dropped_models():
patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
patch(
"litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache",
- new=AsyncMock(return_value=None),
+ new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)),
),
):
with pytest.raises(HTTPException) as exc_info:
@@ -365,6 +368,50 @@ async def test_create_access_group_surfaces_dropped_models():
assert "deploy-A" in str(exc_info.value.detail)
+
+@pytest.mark.asyncio
+async def test_create_access_group_trusts_reload_snapshot_over_post_lock_fresh_read():
+ """A concurrent reconcile sampled after the lock is released must not make this
+ write's reload look like it dropped the tagged model: the verdict has to judge from
+ the ReconcileOutcome the reload captured under the lock, not a fresh router read."""
+ from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
+ from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
+ create_model_group,
+ )
+ from litellm.types.proxy.management_endpoints.model_management_endpoints import (
+ NewModelGroupRequest,
+ )
+
+ deploy_a = MagicMock(model_id="deploy-A", model_name="gpt-4o", model_info={})
+
+ mock_prisma = MagicMock()
+ mock_prisma.db.litellm_proxymodeltable.find_many = AsyncMock(return_value=[])
+ mock_prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=deploy_a)
+ mock_prisma.db.litellm_proxymodeltable.update = AsyncMock()
+
+ concurrently_wiped_router = MagicMock()
+ concurrently_wiped_router.get_model_ids.side_effect = [["deploy-A"], []]
+ with (
+ patch("litellm.proxy.proxy_server.llm_router", concurrently_wiped_router),
+ patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
+ patch(
+ "litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache",
+ new=AsyncMock(
+ return_value=ReconcileOutcome(
+ still_desired=frozenset({"deploy-A"}), live_after=frozenset({"deploy-A"})
+ )
+ ),
+ ),
+ ):
+ response = await create_model_group(
+ data=NewModelGroupRequest(access_group="production-models", model_ids=["deploy-A"]),
+ user_api_key_dict=UserAPIKeyAuth(user_id="test_admin", user_role=LitellmUserRoles.PROXY_ADMIN),
+ )
+
+ assert response.models_updated == 1
+ assert concurrently_wiped_router.get_model_ids.call_count == 1
+
+
@pytest.mark.asyncio
async def test_tag_deployment_parses_string_model_info_and_refuses_corrupt():
"""The model_info column can arrive as its JSON string; tagging must parse it rather
@@ -420,7 +467,7 @@ async def test_delete_access_group_ignores_models_that_were_already_dead():
patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
patch(
"litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache",
- new=AsyncMock(return_value=None),
+ new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)),
),
):
response = await delete_access_group(
@@ -430,3 +477,99 @@ async def test_delete_access_group_ignores_models_that_were_already_dead():
assert response.models_updated == 1
mock_prisma.db.litellm_proxymodeltable.update.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+async def test_create_access_group_read_through_recovers_model_created_on_sibling_replica():
+ """Regression: an access group referencing a model that another replica just wrote
+ to the DB must be created instead of 400ing until the periodic config reload."""
+ from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
+ from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
+ create_model_group,
+ )
+ from litellm.types.proxy.management_endpoints.model_management_endpoints import (
+ NewModelGroupRequest,
+ )
+
+ from types import SimpleNamespace
+
+ model_name = "e2e-ag-sibling-replica-model"
+ db_row = SimpleNamespace(
+ model_id=f"{model_name}-id",
+ model_name=model_name,
+ litellm_params={"model": "openai/gpt-4o", "api_key": "fake", "mock_response": "hi"},
+ model_info={},
+ blocked=False,
+ )
+
+ mock_router = Router(
+ model_list=[
+ {
+ "model_name": "some-other-model",
+ "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"},
+ }
+ ]
+ )
+
+ mock_prisma = MagicMock()
+ mock_prisma.db.litellm_proxymodeltable.find_many = AsyncMock(side_effect=[[db_row], [], [db_row]])
+ mock_prisma.db.litellm_proxymodeltable.update = AsyncMock()
+
+ with (
+ patch("litellm.proxy.proxy_server.llm_router", mock_router),
+ patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
+ patch("litellm.proxy.proxy_server.store_model_in_db", True),
+ patch(
+ "litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache",
+ new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)),
+ ),
+ ):
+ response = await create_model_group(
+ data=NewModelGroupRequest(access_group="replica-lag-group", model_names=[model_name]),
+ user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN),
+ )
+
+ assert response.models_updated == 1
+ assert response.model_names == [model_name]
+ assert mock_prisma.db.litellm_proxymodeltable.find_many.await_args_list[0].kwargs["where"] == {
+ "model_name": model_name
+ }
+
+
+@pytest.mark.asyncio
+async def test_create_access_group_model_missing_everywhere_still_400s():
+ from fastapi import HTTPException
+
+ from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
+ from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
+ create_model_group,
+ )
+ from litellm.types.proxy.management_endpoints.model_management_endpoints import (
+ NewModelGroupRequest,
+ )
+
+ model_name = "e2e-ag-model-nobody-created"
+ mock_router = Router(
+ model_list=[
+ {
+ "model_name": "some-other-model",
+ "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"},
+ }
+ ]
+ )
+ mock_prisma = MagicMock()
+ mock_prisma.db.litellm_proxymodeltable.find_many = AsyncMock(return_value=[])
+
+ with (
+ patch("litellm.proxy.proxy_server.llm_router", mock_router),
+ patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
+ patch("litellm.proxy.proxy_server.store_model_in_db", True),
+ ):
+ with pytest.raises(HTTPException) as exc_info:
+ await create_model_group(
+ data=NewModelGroupRequest(access_group="ghost-group", model_names=[model_name]),
+ user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN),
+ )
+
+ assert exc_info.value.status_code == 400
+ assert model_name in str(exc_info.value.detail)
diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py
index 412e6f0f83c..62c05841197 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py
@@ -14,6 +14,7 @@ sys.path.insert(0, os.path.abspath("../../../..")) # Adds the parent directory
from litellm.proxy.management_endpoints.common_daily_activity import (
_adjust_dates_for_timezone,
_build_aggregated_sql_query,
+ _build_entity_rollup_sql_query,
_is_user_agent_tag,
_record_to_spend_metrics,
get_api_key_metadata,
@@ -982,6 +983,58 @@ class TestBuildAggregatedSqlQuery:
assert "COALESCE(model_group, model)" not in normalized
+class TestAggregatedEmptyEntityFilter:
+ _BUILDERS: Final = (_build_aggregated_sql_query, _build_entity_rollup_sql_query)
+
+ @pytest.mark.parametrize("build", _BUILDERS)
+ def test_empty_entity_list_emits_no_degenerate_in_clause(self, build):
+ sql, params = build(
+ table_name="litellm_dailyteamspend",
+ entity_id_field="team_id",
+ entity_id=[],
+ start_date="2026-08-01",
+ end_date="2026-08-19",
+ model=None,
+ api_key=None,
+ )
+
+ normalized = " ".join(sql.split())
+ assert "IN ()" not in normalized
+ assert '"team_id" IN' not in normalized
+ assert params == ["2026-08-01", "2026-08-19"]
+
+ @pytest.mark.parametrize("build", _BUILDERS)
+ def test_empty_entity_list_matches_nothing_rather_than_everything(self, build):
+ sql, _ = build(
+ table_name="litellm_dailyteamspend",
+ entity_id_field="team_id",
+ entity_id=[],
+ start_date="2026-08-01",
+ end_date="2026-08-19",
+ model=None,
+ api_key=None,
+ )
+
+ assert "FALSE" in " ".join(sql.split())
+
+ @pytest.mark.parametrize("build", _BUILDERS)
+ def test_populated_entity_list_still_filters_on_its_ids(self, build):
+ sql, params = build(
+ table_name="litellm_dailyteamspend",
+ entity_id_field="team_id",
+ entity_id=["team-alpha", "team-beta"],
+ start_date="2026-08-01",
+ end_date="2026-08-19",
+ model=None,
+ api_key=None,
+ )
+
+ normalized = " ".join(sql.split())
+ assert '"team_id" IN ($3, $4)' in normalized
+ assert "FALSE" not in normalized
+ assert params == ["2026-08-01", "2026-08-19", "team-alpha", "team-beta"]
+
+
@pytest.mark.asyncio
async def test_get_daily_activity_aggregated_empty_result_set():
"""Regression test for the empty-range 500.
diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py
index 664015003e4..69819318800 100644
--- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py
+++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py
@@ -1286,6 +1286,75 @@ class TestOpenAIPassthroughIntegration:
mock_chat_handler.assert_called_once()
assert result == {"result": None, "kwargs": {}}
+ def test_openai_passthrough_handler_embeddings_unmapped_model_logs_zero_cost(self):
+ response_body = {
+ "object": "list",
+ "model": "lit5787-unmapped-embeddings-deployment",
+ "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}],
+ "usage": {"prompt_tokens": 9, "total_tokens": 9},
+ }
+ mock_logging_obj = self._create_mock_logging_obj()
+ result = OpenAIPassthroughLoggingHandler.openai_passthrough_handler(
+ httpx_response=self._create_mock_httpx_response(response_body),
+ response_body=response_body,
+ logging_obj=mock_logging_obj,
+ url_route="https://my-resource.openai.azure.com/openai/v1/embeddings",
+ result="",
+ start_time=self.start_time,
+ end_time=self.end_time,
+ cache_hit=False,
+ request_body={
+ "model": "lit5787-unmapped-embeddings-deployment",
+ "input": "spend probe",
+ },
+ passthrough_logging_payload=PassthroughStandardLoggingPayload(
+ url="https://my-resource.openai.azure.com/openai/v1/embeddings",
+ request_body={
+ "model": "lit5787-unmapped-embeddings-deployment",
+ "input": "spend probe",
+ },
+ request_method="POST",
+ ),
+ litellm_params={},
+ )
+
+ assert result["result"] is not None
+ assert result["result"].usage.prompt_tokens == 9
+ assert result["kwargs"]["response_cost"] == 0.0
+ assert result["kwargs"]["model"] == "lit5787-unmapped-embeddings-deployment"
+ assert result["result"]._hidden_params["response_cost"] == 0.0
+ assert mock_logging_obj.model_call_details["response_cost"] == 0.0
+
+ def test_openai_passthrough_handler_embeddings_error_skips_chat_fallback(self):
+ response_body = {
+ "object": "list",
+ "model": "text-embedding-3-small",
+ "usage": {"prompt_tokens": 9, "total_tokens": 9},
+ }
+ kwargs_in = {
+ "passthrough_logging_payload": PassthroughStandardLoggingPayload(
+ url="https://api.openai.com/v1/embeddings",
+ request_body={"model": "text-embedding-3-small", "input": "spend probe"},
+ request_method="POST",
+ ),
+ "litellm_params": {},
+ }
+ result = OpenAIPassthroughLoggingHandler.openai_passthrough_handler(
+ httpx_response=self._create_mock_httpx_response(response_body),
+ response_body=response_body,
+ logging_obj=self._create_mock_logging_obj(),
+ url_route="https://api.openai.com/v1/embeddings",
+ result="",
+ start_time=self.start_time,
+ end_time=self.end_time,
+ cache_hit=False,
+ request_body={"model": "text-embedding-3-small", "input": "spend probe"},
+ **kwargs_in,
+ )
+
+ assert result["result"] is None
+ assert result["kwargs"]["passthrough_logging_payload"] == kwargs_in["passthrough_logging_payload"]
+
@patch(
"litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler.OpenAIPassthroughLoggingHandler.openai_passthrough_handler"
)
diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
index 050070e2fcf..b56a8da7c66 100644
--- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
+++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
@@ -1,7 +1,10 @@
+import contextlib
import json
import os
import sys
import traceback
+from collections.abc import Mapping
+from types import MappingProxyType, SimpleNamespace
from typing import Final
from unittest import mock
from unittest.mock import AsyncMock, MagicMock, Mock, patch
@@ -22,6 +25,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
_join_url_paths,
azure_proxy_route,
bedrock_llm_proxy_route,
+ bedrock_proxy_route,
create_pass_through_route,
cursor_proxy_route,
get_azure_ai_search_index_from_endpoint,
@@ -1730,6 +1734,116 @@ class TestBedrockLLMProxyRoute:
assert "Blocked by guardrail" in str(exc_info.value.detail)
+class TestBedrockAgentRuntimePassthroughToggle:
+ AGENT_RUNTIME_ENDPOINT: Final = "knowledgebases/KB1234567/retrieve"
+ MODEL_ENDPOINT: Final = "model/us.anthropic.claude-sonnet-4-5-20250929-v1:0/converse"
+ DISABLED: Final = MappingProxyType({"disable_bedrock_agent_runtime_passthrough": True})
+
+ @staticmethod
+ def _mock_request() -> Mock:
+ request: Final = Mock()
+ request.method = "POST"
+ request.state = SimpleNamespace()
+ request.json = AsyncMock(return_value={"retrievalQuery": {"text": "hi"}}) # mutable-ok: must be json.dumps-able
+ return request
+
+ @contextlib.contextmanager
+ def _patched_dispatch(self, general_settings: Mapping[str, object]):
+ from botocore.credentials import Credentials
+
+ bedrock_llm: Final = Mock()
+ bedrock_llm.get_credentials = Mock(return_value=Credentials("ak", "sk"))
+ forwarder: Final = AsyncMock(return_value="forwarded")
+
+ with (
+ patch("litellm.proxy.proxy_server.general_settings", general_settings),
+ patch("litellm.utils.get_secret", return_value="us-east-1"),
+ patch("litellm.llms.bedrock.chat.BedrockConverseLLM", return_value=bedrock_llm),
+ patch(
+ "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_request_copy",
+ Mock(),
+ ),
+ patch(
+ "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route",
+ return_value=forwarder,
+ ) as create_route,
+ ):
+ yield create_route, forwarder
+
+ @pytest.mark.asyncio
+ async def test_agent_runtime_dispatch_allowed_by_default(self):
+ with self._patched_dispatch(MappingProxyType({})) as (create_route, forwarder):
+ result: Final = await bedrock_proxy_route(
+ endpoint=self.AGENT_RUNTIME_ENDPOINT,
+ request=self._mock_request(),
+ fastapi_response=Mock(),
+ user_api_key_dict=UserAPIKeyAuth(),
+ )
+
+ assert result == "forwarded"
+ forwarder.assert_awaited_once()
+ assert "bedrock-agent-runtime.us-east-1.amazonaws.com" in create_route.call_args.kwargs["target"]
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("value", (True, "true", "True"))
+ async def test_agent_runtime_dispatch_rejected_when_disabled(self, value: bool | str):
+ settings: Final = MappingProxyType({"disable_bedrock_agent_runtime_passthrough": value})
+
+ with self._patched_dispatch(settings) as (create_route, forwarder):
+ with pytest.raises(HTTPException) as exc_info:
+ await bedrock_proxy_route(
+ endpoint=self.AGENT_RUNTIME_ENDPOINT,
+ request=self._mock_request(),
+ fastapi_response=Mock(),
+ user_api_key_dict=UserAPIKeyAuth(),
+ )
+
+ assert exc_info.value.status_code == 403
+ assert "bedrock-agent-runtime pass-through is disabled" in str(exc_info.value.detail)
+ create_route.assert_not_called()
+ forwarder.assert_not_awaited()
+
+ @pytest.mark.asyncio
+ async def test_model_invoke_still_routed_when_agent_runtime_disabled(self):
+ with (
+ patch("litellm.proxy.proxy_server.general_settings", self.DISABLED),
+ patch("litellm.utils.get_secret", return_value="us-east-1"),
+ patch(
+ "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_request_copy",
+ Mock(),
+ ),
+ patch(
+ "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.bedrock_llm_proxy_route",
+ new=AsyncMock(return_value="llm-route"),
+ ) as llm_route,
+ ):
+ result: Final = await bedrock_proxy_route(
+ endpoint=self.MODEL_ENDPOINT,
+ request=self._mock_request(),
+ fastapi_response=Mock(),
+ user_api_key_dict=UserAPIKeyAuth(),
+ )
+
+ assert result == "llm-route"
+ llm_route.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("value", (False, "false", None, "", "yes"))
+ async def test_agent_runtime_dispatch_allowed_for_non_true_values(self, value: object):
+ settings: Final = MappingProxyType({"disable_bedrock_agent_runtime_passthrough": value})
+
+ with self._patched_dispatch(settings) as (create_route, forwarder):
+ result: Final = await bedrock_proxy_route(
+ endpoint=self.AGENT_RUNTIME_ENDPOINT,
+ request=self._mock_request(),
+ fastapi_response=Mock(),
+ user_api_key_dict=UserAPIKeyAuth(),
+ )
+
+ assert result == "forwarded"
+ create_route.assert_called_once()
+
+
class TestLLMPassthroughFactoryProxyRoute:
@pytest.mark.asyncio
async def test_llm_passthrough_factory_proxy_route_success(self):
diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
index 33d835652cd..e5add059260 100644
--- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
+++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
@@ -124,6 +124,30 @@ def test_get_logging_payload_maps_openai_cache_write_tokens_to_cache_creation_in
assert additional_usage_values["prompt_tokens_details"]["cache_write_tokens"] == 800
+def test_get_logging_payload_maps_nested_cache_creation_input_tokens():
+ """
+ Regression (LIT-5757): DashScope nests cache_creation_input_tokens inside
+ prompt_tokens_details; SpendLogs must record it as cache_creation_input_tokens.
+ """
+ additional_usage_values: Final = _get_additional_usage_values_for_usage(
+ litellm.Usage(
+ prompt_tokens=2059,
+ completion_tokens=31,
+ total_tokens=2090,
+ prompt_tokens_details={
+ "cached_tokens": 0,
+ "text_tokens": 2059,
+ "cache_type": "ephemeral",
+ "cache_creation_input_tokens": 2048,
+ "cache_creation": {"ephemeral_5m_input_tokens": 2048},
+ },
+ )
+ )
+
+ assert additional_usage_values["cache_creation_input_tokens"] == 2048
+ assert additional_usage_values["prompt_tokens_details"]["cache_write_tokens"] == 2048
+
+
def test_get_logging_payload_preserves_anthropic_cache_creation_input_tokens():
additional_usage_values = _get_additional_usage_values_for_usage(
litellm.Usage(
diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py
index 5545ee92e84..75aa716bb85 100644
--- a/tests/test_litellm/proxy/test_proxy_server.py
+++ b/tests/test_litellm/proxy/test_proxy_server.py
@@ -11098,3 +11098,100 @@ async def test_moderations_reraises_proxy_exception_unwrapped():
assert exc_info.value.code == "400"
assert exc_info.value.param == "metadata"
mock_logging.post_call_failure_hook.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+async def test_init_agents_in_db_rebuilds_registry_under_agent_reconcile_lock(monkeypatch):
+ from litellm.proxy.agent_endpoints.agent_registry import (
+ AGENT_RECONCILE_LOCK,
+ global_agent_registry,
+ )
+ from litellm.proxy.proxy_server import ProxyConfig
+
+ lock_states: list[bool] = []
+
+ async def fake_get_all_agents_from_db(prisma_client) -> list:
+ lock_states.append(AGENT_RECONCILE_LOCK.locked())
+ return []
+
+ def fake_load_agents_from_db_and_config(db_agents) -> None:
+ lock_states.append(AGENT_RECONCILE_LOCK.locked())
+
+ monkeypatch.setattr(global_agent_registry, "get_all_agents_from_db", fake_get_all_agents_from_db)
+ monkeypatch.setattr(global_agent_registry, "load_agents_from_db_and_config", fake_load_agents_from_db_and_config)
+
+ await ProxyConfig()._init_agents_in_db(prisma_client=MagicMock())
+
+ assert lock_states == [True, True]
+ assert not AGENT_RECONCILE_LOCK.locked()
+
+
+@pytest.mark.asyncio
+async def test_init_guardrails_in_db_snapshots_and_reconciles_under_guardrail_reconcile_lock(monkeypatch):
+ from litellm.proxy.guardrails.guardrail_registry import (
+ GUARDRAIL_RECONCILE_LOCK,
+ IN_MEMORY_GUARDRAIL_HANDLER,
+ GuardrailRegistry,
+ )
+ from litellm.proxy.proxy_server import ProxyConfig
+
+ lock_states: list[bool] = []
+
+ async def fake_get_all_guardrails_from_db(prisma_client) -> list:
+ lock_states.append(GUARDRAIL_RECONCILE_LOCK.locked())
+ return []
+
+ def fake_reconcile_db_guardrails(db_guardrail_ids) -> list:
+ lock_states.append(GUARDRAIL_RECONCILE_LOCK.locked())
+ return []
+
+ monkeypatch.setattr(GuardrailRegistry, "get_all_guardrails_from_db", fake_get_all_guardrails_from_db)
+ monkeypatch.setattr(IN_MEMORY_GUARDRAIL_HANDLER, "reconcile_db_guardrails", fake_reconcile_db_guardrails)
+
+ await ProxyConfig()._init_guardrails_in_db(prisma_client=MagicMock())
+
+ assert lock_states == [True, True]
+ assert not GUARDRAIL_RECONCILE_LOCK.locked()
+
+
+class TestEmbeddingsFailureHookRequestData:
+ @pytest.mark.asyncio
+ async def test_failure_hook_gets_post_setup_data_with_logging_obj(self):
+ """Request setup replaces the processor's data dict (adding the logging
+ object the failure hook needs to lift token usage from); the embeddings
+ exception handler must pass that replaced dict, not the raw request body
+ dict it was rebuilt from."""
+ from litellm.proxy._types import ProxyException
+
+ captured = {}
+ logging_obj_sentinel = MagicMock()
+
+ async def fake_process(self, **kwargs):
+ self.data = {**self.data, "litellm_logging_obj": logging_obj_sentinel}
+ captured["processor_data"] = self.data
+ raise RuntimeError("provider timeout")
+
+ with (
+ patch.object(
+ proxy_server_module,
+ "_read_request_body",
+ new=AsyncMock(return_value={"model": "my-embed", "input": "hello"}),
+ ),
+ patch.object(
+ proxy_server_module.ProxyBaseLLMRequestProcessing,
+ "base_process_llm_request",
+ new=fake_process,
+ ),
+ patch.object(proxy_server_module, "proxy_logging_obj") as mock_logging,
+ ):
+ mock_logging.post_call_failure_hook = AsyncMock(return_value=None)
+ with pytest.raises(ProxyException):
+ await proxy_server_module.embeddings(
+ request=MagicMock(),
+ fastapi_response=MagicMock(),
+ user_api_key_dict=UserAPIKeyAuth(),
+ )
+
+ hook_request_data = mock_logging.post_call_failure_hook.await_args.kwargs["request_data"]
+ assert hook_request_data is captured["processor_data"]
+ assert hook_request_data["litellm_logging_obj"] is logging_obj_sentinel
diff --git a/tests/test_litellm/proxy/test_proxy_types.py b/tests/test_litellm/proxy/test_proxy_types.py
index 5354de182a0..4a93e9ac7ba 100644
--- a/tests/test_litellm/proxy/test_proxy_types.py
+++ b/tests/test_litellm/proxy/test_proxy_types.py
@@ -159,3 +159,21 @@ def test_update_key_request_requires_key_or_key_alias():
by_alias = UpdateKeyRequest(key_alias="my-alias")
assert by_alias.key is None
assert by_alias.key_alias == "my-alias"
+
+
+@pytest.mark.parametrize("request_type", ["new", "update"])
+def test_project_io_token_limits_are_stored_in_metadata(request_type):
+ from litellm.proxy._types import NewProjectRequest, UpdateProjectRequest
+
+ limits = {
+ "model_itpm_limit": {"bedrock_mantle/openai.gpt-oss-120b": 20_000_000},
+ "model_otpm_limit": {"bedrock_mantle/openai.gpt-oss-120b": 4_000_000},
+ }
+ request = (
+ NewProjectRequest(team_id="team-1", **limits)
+ if request_type == "new"
+ else UpdateProjectRequest(project_id="project-1", **limits)
+ )
+
+ assert request.metadata == limits
+ assert request.model_dump(exclude_none=True)["metadata"] == limits
diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py
index 1504c3c3103..d6cf0e30139 100644
--- a/tests/test_litellm/proxy/test_proxy_utils.py
+++ b/tests/test_litellm/proxy/test_proxy_utils.py
@@ -478,6 +478,307 @@ class TestPostCallFailureHookLiftsRecoveredPartialSpend:
assert "response_cost" not in request_data
+class TestPostCallFailureHookEstimatesDispatchedInputTokens:
+ """A non-stream request that failed after dispatch (timeout, provider
+ error) consumed provider-billed input tokens but recovered no usage.
+ post_call_failure_hook must estimate the input side onto request_data so
+ the spend log's failure row records what was sent instead of zero, while
+ never charging spend for the failure (LIT-5690).
+ """
+
+ async def _run(self, request_data):
+ from unittest.mock import AsyncMock, patch
+
+ from litellm.proxy._types import UserAPIKeyAuth
+
+ proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache())
+ proxy_logging_obj.alert_types = []
+ with patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()):
+ await proxy_logging_obj.post_call_failure_hook(
+ request_data=request_data,
+ original_exception=Exception("boom"),
+ user_api_key_dict=UserAPIKeyAuth(),
+ )
+
+ def _logging_obj(self, model_call_details):
+ logging_obj = MagicMock()
+ logging_obj.model_call_details = model_call_details
+ return logging_obj
+
+ @pytest.mark.asyncio
+ async def test_dispatched_failure_estimates_input_tokens_with_zero_cost(self):
+ from datetime import datetime
+
+ from litellm.types.utils import Usage
+
+ request_data = {
+ "litellm_logging_obj": self._logging_obj(
+ {
+ "first_api_call_start_time": datetime.now(),
+ "model": "gpt-3.5-turbo",
+ "messages": [{"role": "user", "content": "count these input tokens please"}],
+ "call_type": "acompletion",
+ }
+ ),
+ "metadata": {},
+ "response_cost": 123.0,
+ }
+ await self._run(request_data)
+
+ estimated = request_data["combined_usage_object"]
+ assert isinstance(estimated, Usage)
+ assert estimated.prompt_tokens > 0
+ assert estimated.completion_tokens == 0
+ assert estimated.total_tokens == estimated.prompt_tokens
+ assert request_data["response_cost"] == 0.0
+
+ @pytest.mark.asyncio
+ async def test_failure_before_dispatch_stays_zero(self):
+ request_data = {
+ "litellm_logging_obj": self._logging_obj(
+ {
+ "model": "gpt-3.5-turbo",
+ "messages": [{"role": "user", "content": "never dispatched"}],
+ }
+ ),
+ "metadata": {},
+ }
+ await self._run(request_data)
+
+ assert "combined_usage_object" not in request_data
+ assert "response_cost" not in request_data
+
+ @pytest.mark.asyncio
+ async def test_proxy_only_error_never_dispatched_stays_zero(self):
+ from datetime import datetime
+
+ from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL
+
+ request_data = {
+ "litellm_logging_obj": self._logging_obj(
+ {
+ "first_api_call_start_time": datetime.now(),
+ "model": "no-such-model",
+ "messages": [{"role": "user", "content": "hi"}],
+ LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL: True,
+ }
+ ),
+ "metadata": {},
+ }
+ await self._run(request_data)
+
+ assert "combined_usage_object" not in request_data
+ assert "response_cost" not in request_data
+
+ @pytest.mark.asyncio
+ async def test_recovered_partial_usage_wins_over_estimate(self):
+ from datetime import datetime
+
+ from litellm.types.utils import Usage
+
+ recovered_usage = Usage(prompt_tokens=30, completion_tokens=7, total_tokens=37)
+ request_data = {
+ "litellm_logging_obj": self._logging_obj(
+ {
+ "first_api_call_start_time": datetime.now(),
+ "model": "gpt-3.5-turbo",
+ "messages": [{"role": "user", "content": "mid-stream failure"}],
+ "call_type": "acompletion",
+ "combined_usage_object": recovered_usage,
+ "response_cost": 3.5e-05,
+ }
+ ),
+ "metadata": {},
+ }
+ await self._run(request_data)
+
+ assert request_data["combined_usage_object"] is recovered_usage
+ assert request_data["response_cost"] == 3.5e-05
+
+ @pytest.mark.asyncio
+ async def test_dispatched_failure_with_text_completion_prompt(self):
+ from datetime import datetime
+
+ from litellm.types.utils import Usage
+
+ request_data = {
+ "litellm_logging_obj": self._logging_obj(
+ {
+ "first_api_call_start_time": datetime.now(),
+ "model": "gpt-3.5-turbo",
+ "messages": "a plain text-completion prompt string",
+ "call_type": "atext_completion",
+ }
+ ),
+ "metadata": {},
+ }
+ await self._run(request_data)
+
+ estimated = request_data["combined_usage_object"]
+ assert isinstance(estimated, Usage)
+ assert estimated.prompt_tokens > 0
+ assert estimated.completion_tokens == 0
+
+ def _dispatched_request_data(self, messages, optional_params, call_type="acompletion"):
+ from datetime import datetime
+
+ return {
+ "litellm_logging_obj": self._logging_obj(
+ {
+ "first_api_call_start_time": datetime.now(),
+ "model": "gpt-3.5-turbo",
+ "messages": messages,
+ "optional_params": optional_params,
+ "call_type": call_type,
+ }
+ ),
+ "metadata": {},
+ }
+
+ @pytest.mark.asyncio
+ async def test_image_message_estimated_without_fetching_image(self):
+ import litellm as litellm_module
+ from litellm.types.utils import Usage
+
+ messages = [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "describe this image"},
+ {
+ "type": "image_url",
+ "image_url": {"url": "http://127.0.0.1:1/unreachable.png", "detail": "high"},
+ },
+ ],
+ }
+ ]
+ request_data = self._dispatched_request_data(messages, {})
+ await self._run(request_data)
+
+ estimated = request_data["combined_usage_object"]
+ assert isinstance(estimated, Usage)
+ expected = litellm_module.token_counter(
+ model="gpt-3.5-turbo", messages=messages, use_default_image_token_count=True
+ )
+ assert estimated.prompt_tokens == expected
+ assert estimated.prompt_tokens > 0
+
+ @pytest.mark.asyncio
+ async def test_embedding_string_list_input_counted_in_estimate(self):
+ import litellm as litellm_module
+ from litellm.types.utils import Usage
+
+ embedding_input = ["first embedding text", "second embedding text"]
+ request_data = self._dispatched_request_data(embedding_input, {}, call_type="aembedding")
+ await self._run(request_data)
+
+ estimated = request_data["combined_usage_object"]
+ assert isinstance(estimated, Usage)
+ expected = litellm_module.token_counter(model="gpt-3.5-turbo", text="".join(embedding_input))
+ assert estimated.prompt_tokens == expected
+
+ @pytest.mark.asyncio
+ async def test_transcription_checksum_not_estimated(self):
+ request_data = self._dispatched_request_data("a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6", {}, call_type="atranscription")
+ await self._run(request_data)
+
+ assert "combined_usage_object" not in request_data
+ assert "response_cost" not in request_data
+
+ @pytest.mark.asyncio
+ async def test_anthropic_system_prompt_counted_in_estimate(self):
+ import litellm as litellm_module
+ from litellm.types.utils import Usage
+
+ system_prompt = "You are a verbose historian who narrates every fact in exhaustive detail."
+ messages = [{"role": "user", "content": "write a short essay"}]
+ request_data = self._dispatched_request_data(messages, {"system": system_prompt, "max_tokens": 100})
+ await self._run(request_data)
+
+ estimated = request_data["combined_usage_object"]
+ assert isinstance(estimated, Usage)
+ expected = litellm_module.token_counter(model="gpt-3.5-turbo", messages=messages) + litellm_module.token_counter(
+ model="gpt-3.5-turbo", text=system_prompt
+ )
+ assert estimated.prompt_tokens == expected
+
+ @pytest.mark.asyncio
+ async def test_anthropic_system_text_blocks_counted_in_estimate(self):
+ import litellm as litellm_module
+ from litellm.types.utils import Usage
+
+ system_blocks = [
+ {"type": "text", "text": "part one of the system prompt. "},
+ {"type": "text", "text": "part two of the system prompt."},
+ ]
+ messages = [{"role": "user", "content": "write a short essay"}]
+ request_data = self._dispatched_request_data(messages, {"system": system_blocks})
+ await self._run(request_data)
+
+ estimated = request_data["combined_usage_object"]
+ assert isinstance(estimated, Usage)
+ expected = litellm_module.token_counter(model="gpt-3.5-turbo", messages=messages) + litellm_module.token_counter(
+ model="gpt-3.5-turbo", text="part one of the system prompt. part two of the system prompt."
+ )
+ assert estimated.prompt_tokens == expected
+
+ @pytest.mark.asyncio
+ async def test_responses_instructions_counted_in_estimate(self):
+ import litellm as litellm_module
+ from litellm.types.utils import Usage
+
+ instructions = "Answer every question as a meticulous archivist."
+ request_data = self._dispatched_request_data("summarize the archive", {"instructions": instructions})
+ await self._run(request_data)
+
+ estimated = request_data["combined_usage_object"]
+ assert isinstance(estimated, Usage)
+ expected = litellm_module.token_counter(
+ model="gpt-3.5-turbo", text="summarize the archive"
+ ) + litellm_module.token_counter(model="gpt-3.5-turbo", text=instructions)
+ assert estimated.prompt_tokens == expected
+
+ @pytest.mark.asyncio
+ async def test_request_body_system_counted_when_optional_params_empty(self):
+ import litellm as litellm_module
+ from litellm.types.utils import Usage
+
+ system_prompt = "You are a meticulous cartographer who labels every landmark."
+ messages = [{"role": "user", "content": "draw me a map"}]
+ request_data = {
+ **self._dispatched_request_data(messages, {}, call_type="aanthropic_messages"),
+ "system": system_prompt,
+ }
+ await self._run(request_data)
+
+ estimated = request_data["combined_usage_object"]
+ assert isinstance(estimated, Usage)
+ expected = litellm_module.token_counter(model="gpt-3.5-turbo", messages=messages) + litellm_module.token_counter(
+ model="gpt-3.5-turbo", text=system_prompt
+ )
+ assert estimated.prompt_tokens == expected
+
+ @pytest.mark.asyncio
+ async def test_optional_params_system_wins_over_request_body_system(self):
+ import litellm as litellm_module
+ from litellm.types.utils import Usage
+
+ dispatched_system = "short dispatched system prompt"
+ messages = [{"role": "user", "content": "hello"}]
+ request_data = {
+ **self._dispatched_request_data(messages, {"system": dispatched_system}),
+ "system": "a much longer request body system prompt that must not be double counted here",
+ }
+ await self._run(request_data)
+
+ estimated = request_data["combined_usage_object"]
+ assert isinstance(estimated, Usage)
+ expected = litellm_module.token_counter(model="gpt-3.5-turbo", messages=messages) + litellm_module.token_counter(
+ model="gpt-3.5-turbo", text=dispatched_system
+ )
+ assert estimated.prompt_tokens == expected
+
+
from typing import cast
import litellm
diff --git a/tests/test_litellm/proxy/test_route_a2a_models.py b/tests/test_litellm/proxy/test_route_a2a_models.py
index 02e4bddcee0..0523e796543 100644
--- a/tests/test_litellm/proxy/test_route_a2a_models.py
+++ b/tests/test_litellm/proxy/test_route_a2a_models.py
@@ -50,6 +50,7 @@ async def test_route_a2a_model_bypasses_router():
)
mock_registry = Mock()
+ mock_registry.get_agent_by_id = Mock(return_value=None)
mock_registry.get_agent_by_name = Mock(return_value=mock_agent)
# Mock litellm.acompletion to verify it's called
@@ -106,3 +107,79 @@ async def test_route_non_a2a_model_raises_error_if_not_in_router():
user_model=None,
route_type="acompletion",
)
+
+
+class _DbAgentRow:
+ def __init__(self, agent_id: str, agent_name: str) -> None:
+ self.agent_id = agent_id
+ self.agent_name = agent_name
+ self.object_permission = None
+ self.spend = 0.0
+
+ def model_dump(self):
+ return {
+ "agent_id": self.agent_id,
+ "agent_name": self.agent_name,
+ "agent_card_params": {"name": self.agent_name, "url": "http://sibling-db-agent.example.com"},
+ "litellm_params": {},
+ "object_permission": None,
+ "spend": self.spend,
+ }
+
+
+def _router_without_models():
+ mock_router = Mock()
+ mock_router.model_names = []
+ mock_router.deployment_names = []
+ mock_router.has_model_id = Mock(return_value=False)
+ mock_router.model_group_alias = None
+ mock_router.router_general_settings = Mock(pass_through_all_models=False)
+ mock_router.default_deployment = None
+ mock_router.pattern_router = Mock(patterns=[])
+ mock_router.map_team_model = Mock(return_value=None)
+ mock_router.is_recognized_model = Mock(return_value=False)
+ mock_router.team_public_model_names = []
+ return mock_router
+
+
+@pytest.mark.asyncio
+async def test_route_a2a_model_read_through_recovers_agent_created_on_sibling_replica(monkeypatch):
+ import litellm.proxy.proxy_server as proxy_server
+ from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
+
+ agent_name = "a2a-sibling-replica-agent"
+ prisma_client = Mock()
+ prisma_client.db.litellm_agentstable.find_unique = AsyncMock(
+ side_effect=[None, _DbAgentRow("a2a-sibling-replica-agent-id", agent_name)]
+ )
+ monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+ monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+
+ original_agents = list(global_agent_registry.agent_list)
+ original_config_agents = getattr(global_agent_registry, "config_agents", ())
+ global_agent_registry.agent_list = []
+ global_agent_registry.config_agents = ()
+
+ data = {
+ "model": f"a2a/{agent_name}",
+ "messages": [{"role": "user", "content": "Hello"}],
+ }
+ mock_acompletion = AsyncMock(return_value={"id": "read-through-response"})
+
+ try:
+ with patch("litellm.acompletion", mock_acompletion):
+ await route_request(
+ data=data,
+ llm_router=_router_without_models(),
+ user_model=None,
+ route_type="acompletion",
+ )
+ finally:
+ global_agent_registry.agent_list = original_agents
+ global_agent_registry.config_agents = original_config_agents
+
+ mock_acompletion.assert_called_once()
+ call_kwargs = mock_acompletion.call_args.kwargs
+ assert call_kwargs["model"] == f"a2a/{agent_name}"
+ assert call_kwargs["api_base"] == "http://sibling-db-agent.example.com"
+ prisma_client.db.litellm_agentstable.find_unique.assert_awaited()
diff --git a/tests/test_litellm/proxy/test_route_llm_request.py b/tests/test_litellm/proxy/test_route_llm_request.py
index fc3b14592cd..1e716f7c148 100644
--- a/tests/test_litellm/proxy/test_route_llm_request.py
+++ b/tests/test_litellm/proxy/test_route_llm_request.py
@@ -1119,6 +1119,126 @@ async def test_route_request_rejects_chat_completion_without_messages():
llm_router.acompletion.assert_not_called()
+class FakeProxyModelTable:
+ def __init__(self, rows):
+ self.rows = rows
+ self.find_many_wheres = []
+
+ async def find_many(self, where=None, **kwargs):
+ self.find_many_wheres.append(where)
+ return list(self.rows)
+
+
+def _fake_prisma_client_with_models(rows):
+ from types import SimpleNamespace
+
+ table = FakeProxyModelTable(rows)
+ return SimpleNamespace(db=SimpleNamespace(litellm_proxymodeltable=table)), table
+
+
+def _db_model_row(model_name: str, mock_response: str):
+ from types import SimpleNamespace
+
+ return SimpleNamespace(
+ model_id=f"{model_name}-id",
+ model_name=model_name,
+ litellm_params={"model": "openai/gpt-4o", "api_key": "fake", "mock_response": mock_response},
+ model_info={},
+ blocked=False,
+ )
+
+
+@pytest.mark.asyncio
+async def test_route_request_read_through_recovers_model_created_on_sibling_replica(monkeypatch):
+ """Regression: a model written to the DB by another replica must be served on
+ first request instead of 400ing until the periodic config reload."""
+ import litellm
+ import litellm.proxy.proxy_server as proxy_server
+
+ model_name = "e2e-sibling-replica-model"
+ router = litellm.Router(
+ model_list=[
+ {
+ "model_name": "some-other-model",
+ "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"},
+ }
+ ]
+ )
+ fake_prisma, table = _fake_prisma_client_with_models([_db_model_row(model_name, "hello-from-db")])
+ monkeypatch.setattr(proxy_server, "prisma_client", fake_prisma)
+ monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+ monkeypatch.setattr(proxy_server, "llm_router", router)
+
+ llm_call = await route_request(
+ data={"model": model_name, "messages": [{"role": "user", "content": "hi"}]},
+ llm_router=router,
+ user_model=None,
+ route_type="acompletion",
+ )
+ response = await llm_call
+
+ assert response.choices[0].message.content == "hello-from-db"
+ assert len(table.find_many_wheres) == 1
+ assert table.find_many_wheres[0] == {"model_name": model_name}
+
+
+@pytest.mark.asyncio
+async def test_route_request_unknown_model_raises_and_hits_db_once_within_ttl(monkeypatch):
+ import litellm
+ import litellm.proxy.proxy_server as proxy_server
+
+ model_name = "e2e-model-nobody-created"
+ router = litellm.Router(
+ model_list=[
+ {
+ "model_name": "some-other-model",
+ "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"},
+ }
+ ]
+ )
+ fake_prisma, table = _fake_prisma_client_with_models([])
+ monkeypatch.setattr(proxy_server, "prisma_client", fake_prisma)
+ monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+ monkeypatch.setattr(proxy_server, "llm_router", router)
+
+ data = {"model": model_name, "messages": [{"role": "user", "content": "hi"}]}
+ with pytest.raises(ProxyModelNotFoundError):
+ await route_request(data=data, llm_router=router, user_model=None, route_type="acompletion")
+ with pytest.raises(ProxyModelNotFoundError):
+ await route_request(data=data, llm_router=router, user_model=None, route_type="acompletion")
+
+ assert table.find_many_wheres == [{"model_name": model_name}, {"model_id": model_name}]
+
+
+@pytest.mark.asyncio
+async def test_route_request_read_through_disabled_without_store_model_in_db(monkeypatch):
+ import litellm
+ import litellm.proxy.proxy_server as proxy_server
+
+ model_name = "e2e-config-only-proxy-model"
+ router = litellm.Router(
+ model_list=[
+ {
+ "model_name": "some-other-model",
+ "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"},
+ }
+ ]
+ )
+ fake_prisma, table = _fake_prisma_client_with_models([_db_model_row(model_name, "should-not-load")])
+ monkeypatch.setattr(proxy_server, "prisma_client", fake_prisma)
+ monkeypatch.setattr(proxy_server, "store_model_in_db", False)
+ monkeypatch.setattr(proxy_server, "llm_router", router)
+
+ with pytest.raises(ProxyModelNotFoundError):
+ await route_request(
+ data={"model": model_name, "messages": [{"role": "user", "content": "hi"}]},
+ llm_router=router,
+ user_model=None,
+ route_type="acompletion",
+ )
+
+ assert table.find_many_wheres == []
+
@pytest.mark.asyncio
async def test_route_request_routing_group_name_passes_model_gate():
from unittest.mock import AsyncMock, patch
@@ -1141,3 +1261,39 @@ async def test_route_request_routing_group_name_passes_model_gate():
assert response == "group_response"
spy.assert_called_once_with(**data)
+
+
+@pytest.mark.asyncio
+async def test_route_request_a2a_agent_miss_does_not_consume_model_read_through(monkeypatch):
+ from types import SimpleNamespace
+ from unittest.mock import AsyncMock
+
+ import litellm
+ import litellm.proxy.proxy_server as proxy_server
+
+ model_name = "a2a/agent-nobody-created"
+ router = litellm.Router(
+ model_list=[
+ {
+ "model_name": "some-other-model",
+ "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"},
+ }
+ ]
+ )
+ fake_prisma, model_table = _fake_prisma_client_with_models([])
+ agents_find_unique = AsyncMock(return_value=None)
+ fake_prisma.db.litellm_agentstable = SimpleNamespace(find_unique=agents_find_unique)
+ monkeypatch.setattr(proxy_server, "prisma_client", fake_prisma)
+ monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+ monkeypatch.setattr(proxy_server, "llm_router", router)
+
+ with pytest.raises(ProxyModelNotFoundError):
+ await route_request(
+ data={"model": model_name, "messages": [{"role": "user", "content": "hi"}]},
+ llm_router=router,
+ user_model=None,
+ route_type="acompletion",
+ )
+
+ assert agents_find_unique.await_count == 2
+ assert model_table.find_many_wheres == []
diff --git a/tests/test_litellm/responses/test_responses_websocket_all_providers.py b/tests/test_litellm/responses/test_responses_websocket_all_providers.py
index 4509abc7749..2d523bfdeb3 100644
--- a/tests/test_litellm/responses/test_responses_websocket_all_providers.py
+++ b/tests/test_litellm/responses/test_responses_websocket_all_providers.py
@@ -1030,6 +1030,171 @@ class TestWebSocketErrorHandling:
assert "Invalid JSON" in error_event
+class TestWebSocketProjectQuotaEnforcement:
+ """VERIA regression: the connection-level pre-call hook only runs once,
+ but a WebSocket connection accepts many response.create frames. Every
+ frame must be checked against any registered project ITPM/OTPM quota
+ callback, not just the first one."""
+
+ @pytest.mark.asyncio
+ async def test_managed_handler_blocks_frame_rejected_by_quota_callback(self, monkeypatch):
+ from unittest.mock import AsyncMock, MagicMock
+
+ import litellm
+ from litellm.exceptions import RateLimitError
+ from litellm.litellm_core_utils.litellm_logging import Logging
+ from litellm.responses.streaming_iterator import (
+ ManagedResponsesWebSocketHandler,
+ )
+
+ aresponses_called = False
+
+ async def fake_aresponses(*args, **kwargs):
+ nonlocal aresponses_called
+ aresponses_called = True
+
+ monkeypatch.setattr(litellm, "aresponses", fake_aresponses)
+
+ quota_callback = MagicMock()
+ quota_callback.enforce_project_io_token_quota_for_frame = AsyncMock(
+ side_effect=RateLimitError(message="project OTPM exceeded", llm_provider="", model="")
+ )
+
+ mock_websocket = MagicMock()
+ mock_websocket.send_text = AsyncMock()
+ mock_logging_obj = Logging(
+ model="test-model",
+ messages=[],
+ stream=True,
+ call_type="aresponses",
+ start_time=0,
+ litellm_call_id="test-id",
+ function_id="test-func",
+ )
+ handler = ManagedResponsesWebSocketHandler(
+ websocket=mock_websocket,
+ model="test-model",
+ logging_obj=mock_logging_obj,
+ quota_callbacks=[quota_callback],
+ )
+
+ await handler._process_response_create(json.dumps({"type": "response.create", "input": "hi"}))
+
+ quota_callback.enforce_project_io_token_quota_for_frame.assert_awaited_once()
+ assert aresponses_called is False
+ mock_websocket.send_text.assert_called_once()
+ error_event = mock_websocket.send_text.call_args[0][0]
+ assert "rate_limit_exceeded" in error_event
+
+ @pytest.mark.asyncio
+ async def test_managed_handler_forwards_frame_allowed_by_quota_callback(self, monkeypatch):
+ from unittest.mock import AsyncMock, MagicMock
+
+ import litellm
+ from litellm.litellm_core_utils.litellm_logging import Logging
+ from litellm.responses.streaming_iterator import (
+ ManagedResponsesWebSocketHandler,
+ )
+
+ aresponses_called = False
+
+ async def fake_aresponses(*args, **kwargs):
+ nonlocal aresponses_called
+ aresponses_called = True
+
+ async def _empty():
+ return
+ yield
+
+ return _empty()
+
+ monkeypatch.setattr(litellm, "aresponses", fake_aresponses)
+
+ quota_callback = MagicMock()
+ quota_callback.enforce_project_io_token_quota_for_frame = AsyncMock(return_value=None)
+
+ mock_websocket = MagicMock()
+ mock_websocket.send_text = AsyncMock()
+ mock_logging_obj = Logging(
+ model="test-model",
+ messages=[],
+ stream=True,
+ call_type="aresponses",
+ start_time=0,
+ litellm_call_id="test-id",
+ function_id="test-func",
+ )
+ handler = ManagedResponsesWebSocketHandler(
+ websocket=mock_websocket,
+ model="test-model",
+ logging_obj=mock_logging_obj,
+ quota_callbacks=[quota_callback],
+ )
+
+ await handler._process_response_create(json.dumps({"type": "response.create", "input": "hi"}))
+
+ quota_callback.enforce_project_io_token_quota_for_frame.assert_awaited_once()
+ assert aresponses_called is True
+
+ @pytest.mark.asyncio
+ async def test_native_handler_blocks_frame_rejected_by_quota_callback(self):
+ from unittest.mock import AsyncMock, MagicMock
+
+ from litellm.exceptions import RateLimitError
+ from litellm.responses.streaming_iterator import ResponsesWebSocketStreaming
+
+ quota_callback = MagicMock()
+ quota_callback.enforce_project_io_token_quota_for_frame = AsyncMock(
+ side_effect=RateLimitError(message="project OTPM exceeded", llm_provider="", model="")
+ )
+
+ mock_backend_ws = MagicMock()
+ mock_backend_ws.send = AsyncMock()
+ mock_websocket = MagicMock()
+ mock_websocket.send_text = AsyncMock()
+
+ handler = ResponsesWebSocketStreaming(
+ websocket=mock_websocket,
+ backend_ws=mock_backend_ws,
+ logging_obj=MagicMock(),
+ authorized_model="gpt-4o",
+ quota_callbacks=[quota_callback],
+ )
+
+ allowed = await handler._enforce_or_reject_frame(
+ json.dumps({"type": "response.create", "input": "hi"})
+ )
+
+ assert allowed is False
+ mock_backend_ws.send.assert_not_called()
+ mock_websocket.send_text.assert_called_once()
+ assert "rate_limit_exceeded" in mock_websocket.send_text.call_args[0][0]
+
+ @pytest.mark.asyncio
+ async def test_native_handler_forwards_frame_allowed_by_quota_callback(self):
+ from unittest.mock import AsyncMock, MagicMock
+
+ from litellm.responses.streaming_iterator import ResponsesWebSocketStreaming
+
+ quota_callback = MagicMock()
+ quota_callback.enforce_project_io_token_quota_for_frame = AsyncMock(return_value=None)
+
+ handler = ResponsesWebSocketStreaming(
+ websocket=MagicMock(),
+ backend_ws=MagicMock(),
+ logging_obj=MagicMock(),
+ authorized_model="gpt-4o",
+ quota_callbacks=[quota_callback],
+ )
+
+ allowed = await handler._enforce_or_reject_frame(
+ json.dumps({"type": "response.create", "input": "hi"})
+ )
+
+ assert allowed is True
+ quota_callback.enforce_project_io_token_quota_for_frame.assert_awaited_once()
+
+
class TestNativeWebSocketGuardrails:
@pytest.mark.asyncio
async def test_response_create_injects_authorized_model(self):
diff --git a/tests/test_litellm/router_utils/test_fallback_event_handlers.py b/tests/test_litellm/router_utils/test_fallback_event_handlers.py
index 68395737469..24477248a8a 100644
--- a/tests/test_litellm/router_utils/test_fallback_event_handlers.py
+++ b/tests/test_litellm/router_utils/test_fallback_event_handlers.py
@@ -1,4 +1,5 @@
import json
+from typing import NoReturn
from unittest.mock import MagicMock, patch
import httpx
@@ -167,6 +168,10 @@ async def _acreate_batch(*args, **kwargs):
raise AssertionError("only used for its __name__")
+async def _acreate_file(*args: object, **kwargs: object) -> NoReturn:
+ raise AssertionError("only used for its __name__")
+
+
@pytest.mark.asyncio
async def test_run_async_fallback_keeps_uploaded_file_requests_in_their_model_group():
"""An input_file_id only exists under the credentials of the group it was uploaded
@@ -229,6 +234,46 @@ async def test_run_async_fallback_allows_same_model_group_retry_for_uploaded_fil
assert router.attempted_model_groups == ["openai-group"]
+@pytest.mark.asyncio
+async def test_run_async_fallback_keeps_file_creation_in_its_model_group():
+ """A file created for batches lands in the account of the deployment that stored it,
+ and its id is only usable against the model group the caller named. A cross-group
+ fallback silently stores the file with the wrong provider."""
+ router = AttemptRecordingRouter()
+
+ with pytest.raises(RuntimeError, match="azure connection error"):
+ await run_async_fallback(
+ litellm_router=router,
+ fallback_model_group=["openai-group"],
+ original_model_group="azure-group",
+ original_exception=RuntimeError("azure connection error"),
+ max_fallbacks=3,
+ fallback_depth=0,
+ model="azure-group",
+ original_function=_acreate_file,
+ )
+
+ assert router.attempted_model_groups == []
+
+
+@pytest.mark.asyncio
+async def test_run_async_fallback_allows_same_model_group_retry_for_file_creation():
+ router = AttemptRecordingRouter()
+
+ await run_async_fallback(
+ litellm_router=router,
+ fallback_model_group=[{"model": "azure-group", "_target_order": 2}],
+ original_model_group="azure-group",
+ original_exception=RuntimeError("first deployment failed"),
+ max_fallbacks=3,
+ fallback_depth=0,
+ model="azure-group",
+ original_function=_acreate_file,
+ )
+
+ assert router.attempted_model_groups == ["azure-group"]
+
+
@pytest.mark.asyncio
async def test_run_async_fallback_still_crosses_model_groups_without_an_uploaded_file():
router = AttemptRecordingRouter()
diff --git a/tests/test_litellm/test_conftest.py b/tests/test_litellm/test_conftest.py
new file mode 100644
index 00000000000..cca4f7c3ef2
--- /dev/null
+++ b/tests/test_litellm/test_conftest.py
@@ -0,0 +1,43 @@
+import os
+import subprocess
+import sys
+from pathlib import Path
+from typing import Final
+
+REPO_ROOT: Final = Path(__file__).resolve().parents[2]
+
+PROXY_BASE_URL_SENSITIVE_NODE: Final = (
+ "tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py"
+ "::TestTemporaryMCPSessionEndpoints"
+ "::test_mcp_token_opens_sealed_passthrough_code_and_exchanges_with_minted_client"
+)
+
+COVERAGE_SUBPROCESS_VARS: Final = frozenset(
+ {"COV_CORE_SOURCE", "COV_CORE_CONFIG", "COV_CORE_DATAFILE", "COV_CORE_CONTEXT", "COVERAGE_PROCESS_START"}
+)
+
+
+def test_host_proxy_base_url_cannot_reach_request_derived_url_tests():
+ child_env: Final = {
+ key: value for key, value in os.environ.items() if key not in COVERAGE_SUBPROCESS_VARS
+ } | {"PROXY_BASE_URL": "https://leaked-host-origin.example.com"}
+
+ completed: Final = subprocess.run(
+ [
+ sys.executable,
+ "-m",
+ "pytest",
+ PROXY_BASE_URL_SENSITIVE_NODE,
+ "-q",
+ "--no-header",
+ "-p",
+ "no:cacheprovider",
+ ],
+ cwd=REPO_ROOT,
+ env=child_env,
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+
+ assert completed.returncode == 0, completed.stdout + completed.stderr
diff --git a/tests/test_litellm/test_logging.py b/tests/test_litellm/test_logging.py
index 9ab362f6cd5..784ec5b6cf4 100644
--- a/tests/test_litellm/test_logging.py
+++ b/tests/test_litellm/test_logging.py
@@ -177,7 +177,10 @@ def test_json_formatter_parses_embedded_python_dict_repr():
# Python dict parsed and promoted to first-class properties
assert obj["model_name"] == "text-embedding-3-large"
assert "litellm_params" in obj
- assert obj["litellm_params"]["api_key"] == "sk**********"
+ # Redacted, not passed through: SecretRedactionFilter already collapses this
+ # pair in the plain path before any formatter sees it, so the JSON path matching
+ # it is production parity. The key survives because redaction is per-value here.
+ assert obj["litellm_params"]["api_key"] == "REDACTED"
assert obj["litellm_params"]["tpm"] == 1000000
assert obj["litellm_params"]["use_in_pass_through"] is False
assert "model_info" in obj
@@ -185,6 +188,38 @@ def test_json_formatter_parses_embedded_python_dict_repr():
assert obj["model_info"]["db_model"] is False
+def test_json_formatter_output_stays_parseable_when_a_secret_is_redacted():
+ """Redaction must collapse the value only, never the surrounding JSON member.
+
+ Redacting the serialized document turned '"api_key": "sk-..."' into a bare
+ REDACTED token, so the line stopped being valid JSON entirely.
+ """
+ formatter = JsonFormatter()
+ record = logging.LogRecord(
+ name="LiteLLM",
+ level=logging.INFO,
+ pathname="",
+ lineno=0,
+ msg="calling deployment",
+ args=(),
+ exc_info=None,
+ )
+ record.deployment = {
+ "api_key": "sk-abcdefghijklmnopqrstuvwxyz0123456789",
+ "aws_secret_access_key": "wJalrXUtnFEMIQAfakeKEYbPxRfiCYEXAMPLEKEY",
+ "aws_region_name": "us-east-1",
+ "nested": {"tokens": ["Bearer abcdefghijklmnop", "keep-me"]},
+ }
+
+ obj = json.loads(formatter.format(record))
+
+ assert obj["deployment"]["api_key"] == "REDACTED"
+ assert obj["deployment"]["aws_secret_access_key"] == "REDACTED"
+ # Non-secret siblings stay legible so the logs remain useful
+ assert obj["deployment"]["aws_region_name"] == "us-east-1"
+ assert obj["deployment"]["nested"]["tokens"] == ["REDACTED", "keep-me"]
+
+
def test_json_formatter_includes_component_field():
"""
Test that JsonFormatter always emits a 'component' field equal to the logger name.
diff --git a/tests/test_litellm/test_responses_api_bridge_non_stream.py b/tests/test_litellm/test_responses_api_bridge_non_stream.py
index 25a3bc2dbba..c272b151865 100644
--- a/tests/test_litellm/test_responses_api_bridge_non_stream.py
+++ b/tests/test_litellm/test_responses_api_bridge_non_stream.py
@@ -1,6 +1,6 @@
import os
import sys
-from typing import Optional
+from typing import Final, Optional
from unittest.mock import Mock
import pytest
@@ -192,6 +192,32 @@ def test_transform_usage_with_cached_tokens_only():
print("✓ Transformation works with cached_tokens only")
+def test_transform_usage_maps_nested_cache_creation_input_tokens():
+ """
+ Regression (LIT-5757): DashScope nests cache_creation_input_tokens inside
+ prompt_tokens_details; the bridge must surface it as cache_write_tokens.
+ """
+ usage: Final = Usage(
+ prompt_tokens=2059,
+ completion_tokens=31,
+ total_tokens=2090,
+ prompt_tokens_details={
+ "cached_tokens": 0,
+ "text_tokens": 2059,
+ "cache_type": "ephemeral",
+ "cache_creation_input_tokens": 2048,
+ "cache_creation": {"ephemeral_5m_input_tokens": 2048},
+ },
+ )
+
+ responses_usage: Final = LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage(
+ usage
+ )
+
+ assert responses_usage.input_tokens_details is not None
+ assert responses_usage.input_tokens_details.cache_write_tokens == 2048
+
+
def test_transform_usage_with_reasoning_tokens_only():
"""
Test transformation when only reasoning_tokens is provided (no cached_tokens).
diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py
index 49ed236356c..65debae9a16 100644
--- a/tests/test_litellm/test_router.py
+++ b/tests/test_litellm/test_router.py
@@ -492,6 +492,54 @@ async def test_async_router_acreate_file_with_jsonl():
assert first_call_content == non_jsonl_content
+@pytest.mark.asyncio
+async def test_async_router_acreate_file_does_not_fall_back_across_model_groups():
+ """A file created for batches only exists under the credentials of the model group
+ the caller named. A cross-group fallback silently stores it with the wrong provider
+ and the later batch create against the named group permanently fails."""
+ from unittest.mock import MagicMock, patch
+
+ router = litellm.Router(
+ model_list=[
+ {
+ "model_name": "azure-gpt",
+ "litellm_params": {
+ "model": "azure/my-azure-deployment",
+ "api_base": "http://127.0.0.1:9",
+ "api_key": "dummy-key",
+ "api_version": "2024-06-01",
+ },
+ },
+ {
+ "model_name": "openai-gpt",
+ "litellm_params": {"model": "gpt-4o-mini"},
+ },
+ ],
+ fallbacks=[{"azure-gpt": ["openai-gpt"]}],
+ )
+
+ def fail_azure(*args: object, **kwargs: object) -> MagicMock:
+ if kwargs.get("model") == "azure/my-azure-deployment":
+ raise litellm.APIConnectionError(
+ message="Connection error.",
+ llm_provider="azure",
+ model="azure/my-azure-deployment",
+ )
+ return MagicMock()
+
+ with patch("litellm.acreate_file", side_effect=fail_azure) as mock_acreate_file:
+ with pytest.raises(litellm.APIConnectionError):
+ await router.acreate_file(
+ model="azure-gpt",
+ purpose="batch",
+ file=MagicMock(),
+ )
+
+ called_models = [call.kwargs.get("model") for call in mock_acreate_file.call_args_list]
+ assert "azure/my-azure-deployment" in called_models
+ assert "gpt-4o-mini" not in called_models
+
+
@pytest.mark.asyncio
async def test_async_router_acreate_file_uses_deployment_custom_llm_provider():
"""
@@ -7595,6 +7643,95 @@ def test_pre_call_checks_keeps_deployment_when_provider_is_unresolvable(monkeypa
assert len(result) == 1
+class TestUpsertDeploymentRollback:
+ """
+ Regression tests: `upsert_deployment` pops the previous deployment before
+ re-adding the edited one. When the re-add raises under
+ `ignore_invalid_deployments=True`, the pop must be rolled back so this pod
+ keeps serving the previous configuration instead of silently dropping a live
+ deployment (the "Error upserting deployment" drop behind the access-group
+ reload 500 in the 2-replica e2e suite).
+ """
+
+ def test_failed_upsert_keeps_previous_deployment_serving(self):
+ from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
+
+ router = litellm.Router(
+ model_list=[
+ {
+ "model_name": "prod-model",
+ "litellm_params": {"model": "gpt-4o", "api_key": "sk-old"},
+ "model_info": {"id": "prod-1", "db_model": True},
+ }
+ ],
+ ignore_invalid_deployments=True,
+ )
+
+ result = router.upsert_deployment(
+ deployment=Deployment(
+ model_name="prod-model",
+ litellm_params=LiteLLM_Params(model="auto_router/broken"),
+ model_info=ModelInfo(id="prod-1", db_model=True),
+ )
+ )
+
+ assert result is None
+ restored = router.get_deployment(model_id="prod-1")
+ assert restored is not None
+ assert restored.litellm_params.model == "gpt-4o"
+ assert [model["model_name"] for model in router.model_list] == ["prod-model"]
+
+ def test_failed_fresh_add_returns_none_without_restore(self):
+ from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
+
+ router = litellm.Router(model_list=[], ignore_invalid_deployments=True)
+
+ result = router.upsert_deployment(
+ deployment=Deployment(
+ model_name="fresh-router",
+ litellm_params=LiteLLM_Params(model="auto_router/broken"),
+ model_info=ModelInfo(id="fresh-1", db_model=True),
+ )
+ )
+
+ assert result is None
+ assert router.get_deployment(model_id="fresh-1") is None
+ assert router.model_list == []
+
+ def test_restore_re_adds_popped_deployment(self):
+ router = litellm.Router(
+ model_list=[
+ {
+ "model_name": "prod-model",
+ "litellm_params": {"model": "gpt-4o", "api_key": "sk-old"},
+ "model_info": {"id": "prod-1", "db_model": True},
+ }
+ ],
+ ignore_invalid_deployments=True,
+ )
+ previous = router.get_deployment(model_id="prod-1")
+ router.delete_deployment(id="prod-1")
+ assert router.has_model_id("prod-1") is False
+
+ router._restore_deployment_after_failed_upsert(
+ previous_deployment=previous, model_id="prod-1"
+ )
+
+ restored = router.get_deployment(model_id="prod-1")
+ assert restored is not None
+ assert restored.litellm_params.model == "gpt-4o"
+
+ router._restore_deployment_after_failed_upsert(
+ previous_deployment=previous, model_id="prod-1"
+ )
+ assert len(router.model_list) == 1
+
+ router._restore_deployment_after_failed_upsert(
+ previous_deployment=None, model_id="prod-1"
+ )
+ assert len(router.model_list) == 1
+
+
class TestConsumedRequestTagsStamp:
"""Issue #36621: when a request's tags select a tagged pre-routing strategy, those
tags are consumed by the selection; the hook must stamp the rewritten model group so
@@ -7794,6 +7931,21 @@ class TestTaggedAutoRouterOnSharedModelName:
assert response is not None
assert response.model == "gemini-flash"
+ @pytest.mark.asyncio
+ async def test_request_level_tag_filtering_from_key_settings_bypasses_the_marker(self):
+ router = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=False)
+
+ assert await self._hook_response(router, {"enable_tag_filtering": True}) is None
+
+ @pytest.mark.asyncio
+ async def test_globally_disabled_filtering_still_lets_the_marker_capture_untagged_requests(self):
+ router = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=False)
+
+ response = await self._hook_response(router, {})
+
+ assert response is not None
+ assert response.model == "gemini-flash"
+
@pytest.mark.asyncio
async def test_marker_only_alias_still_captures_untagged_requests(self):
router = self._router(marker_tags=["route"], include_plain_sibling=False, enable_tag_filtering=True)
diff --git a/tests/test_litellm/test_secret_redaction.py b/tests/test_litellm/test_secret_redaction.py
index a5f3f912339..0188d87dfdb 100644
--- a/tests/test_litellm/test_secret_redaction.py
+++ b/tests/test_litellm/test_secret_redaction.py
@@ -495,3 +495,54 @@ def test_redaction_survives_uvicorn_logging_reconfiguration():
lg.handlers[:] = handlers
lg.setLevel(level)
lg.propagate = True
+
+
+def test_aws_credential_redaction_catches_quoted_values():
+ """AWS creds appear as quoted dict-repr values, not just bare key=value."""
+ cases = (
+ "{'aws_secret_access_key': 'wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY'}",
+ '{"aws_session_token": "IQoJb3JpZ2luX2VjEaCXVzLWVhc3QtMSJHMEUCIQ"}',
+ "aws_session_token: 'FwoGZXIvYXdzEBYaDHh4eHh4eHh4eHh4eCLLAe'",
+ "aws_secret_access_key=wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY",
+ "{'aws_access_key_id': 'not-an-akia-shaped-value'}",
+ )
+ for secret_line in cases:
+ result = redact_string(secret_line)
+ assert "REDACTED" in result, f"AWS redaction missed: {secret_line!r}"
+ assert "wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY" not in result
+ assert "IQoJb3JpZ2luX2VjEaCXVzLWVhc3QtMSJHMEUCIQ" not in result
+
+ safe = "'aws_region_name': 'us-east-1'"
+ assert redact_string(safe) == safe
+
+
+@pytest.mark.parametrize(
+ "extra",
+ (
+ {"api_base": {f"https://host/v1?key={SECRET}"}},
+ {"blob": {"authorization": f"Bearer {SECRET}"}},
+ {"blob": [f"Bearer {SECRET}"]},
+ {"blob": ({"nested": {"deep": SECRET}},)},
+ ),
+ ids=("set", "dict", "list", "nested"),
+)
+def test_json_formatter_redacts_non_string_extra_values(extra):
+ """SecretRedactionFilter only scrubs str attrs, so containers must be caught on render."""
+ buf = StringIO()
+ handler = logging.StreamHandler(buf)
+ handler.setFormatter(JsonFormatter())
+ handler.addFilter(_secret_filter)
+
+ logger = logging.getLogger("test_json_extra_redaction")
+ logger.handlers = [handler]
+ logger.setLevel(logging.DEBUG)
+ logger.propagate = False
+ try:
+ logger.warning("request sent", extra=extra)
+ finally:
+ logger.handlers = []
+
+ output = buf.getvalue()
+ assert output.strip(), "no record captured"
+ assert SECRET not in output, f"non-string extra leaked a secret: {output}"
+ assert "REDACTED" in output
diff --git a/tests/test_litellm/test_thinking_enabled.py b/tests/test_litellm/test_thinking_enabled.py
index 8ba406c395a..744b258e617 100644
--- a/tests/test_litellm/test_thinking_enabled.py
+++ b/tests/test_litellm/test_thinking_enabled.py
@@ -60,6 +60,7 @@ class TestIsThinkingEnabled:
({"reasoning_effort": "medium"}, True),
# both thinking enabled and reasoning_effort returns True
({"thinking": {"type": "enabled"}, "reasoning_effort": "high"}, True),
+ ({"thinking": True}, True),
# falsy thinking values should not crash
({"thinking": False}, False),
({"thinking": 0}, False),
diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py
index afdfdf170ac..efccdc4a986 100644
--- a/tests/test_litellm/test_utils.py
+++ b/tests/test_litellm/test_utils.py
@@ -3766,6 +3766,20 @@ class TestValidateAndFixThinkingParam:
assert "budgetTokens" in thinking
assert "budget_tokens" not in thinking
+ def test_bool_true_maps_to_enabled_with_default_budget(self):
+ from litellm.constants import DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET
+ from litellm.utils import validate_and_fix_thinking_param
+
+ assert validate_and_fix_thinking_param(thinking=True) == {
+ "type": "enabled",
+ "budget_tokens": DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
+ }
+
+ def test_bool_false_returns_none(self):
+ from litellm.utils import validate_and_fix_thinking_param
+
+ assert validate_and_fix_thinking_param(thinking=False) is None
+
def test_deepseek_v4_models_in_cost_map():
"""
diff --git a/tests/test_litellm/types/test_types_utils.py b/tests/test_litellm/types/test_types_utils.py
index cd5e8dda012..672aa84cc73 100644
--- a/tests/test_litellm/types/test_types_utils.py
+++ b/tests/test_litellm/types/test_types_utils.py
@@ -1,5 +1,6 @@
import os
import sys
+from typing import Final
import pytest
@@ -75,6 +76,29 @@ def test_usage_dump():
assert new_usage.prompt_tokens_details.web_search_requests == 1
+def test_prompt_tokens_details_maps_nested_cache_creation_input_tokens():
+ """Regression (LIT-5757): DashScope nests the Anthropic-spelled
+ cache_creation_input_tokens inside prompt_tokens_details. It must populate
+ the canonical cache_write_tokens/cache_creation_tokens pair, without
+ overriding an explicitly provided canonical value."""
+ from litellm.types.utils import PromptTokensDetailsWrapper
+
+ nested: Final = PromptTokensDetailsWrapper(
+ cached_tokens=0, text_tokens=2059, cache_creation_input_tokens=2048
+ )
+ assert nested.cache_write_tokens == 2048
+ assert nested.cache_creation_tokens == 2048
+
+ explicit: Final = PromptTokensDetailsWrapper(
+ cache_write_tokens=100, cache_creation_input_tokens=2048
+ )
+ assert explicit.cache_write_tokens == 100
+ assert explicit.cache_creation_tokens == 100
+
+ non_int: Final = PromptTokensDetailsWrapper(cache_creation_input_tokens=None)
+ assert not hasattr(non_int, "cache_write_tokens")
+
+
def test_usage_server_tool_use_dict_is_coerced_and_round_trips():
from litellm.types.utils import ServerToolUse, Usage
diff --git a/type-discipline-budget.json b/type-discipline-budget.json
index 94c1f9b86f9..e7cfff93aa4 100644
--- a/type-discipline-budget.json
+++ b/type-discipline-budget.json
@@ -1,9 +1,9 @@
{
"LIT001": {
- "limit": 22897
+ "limit": 22892
},
"LIT002": {
- "limit": 26888
+ "limit": 26886
},
"LIT003": {
"limit": 269
@@ -27,7 +27,7 @@
"limit": 0
},
"LIT010": {
- "limit": 16713
+ "limit": 16699
},
"LIT011": {
"limit": 5590
diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json
index ed72e784a35..c471355db68 100644
--- a/ui/litellm-dashboard/eslint-suppressions.json
+++ b/ui/litellm-dashboard/eslint-suppressions.json
@@ -793,16 +793,6 @@
"count": 1
}
},
- "src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx": {
- "no-restricted-imports": {
- "count": 1
- }
- },
- "src/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel.tsx": {
- "no-restricted-imports": {
- "count": 1
- }
- },
"src/app/(dashboard)/models-and-endpoints/utils/modelDataTransformer.ts": {
"prefer-const": {
"count": 6
@@ -1055,11 +1045,6 @@
"count": 1
}
},
- "src/app/(dashboard)/policies/_components/impact_popover.test.tsx": {
- "react/display-name": {
- "count": 1
- }
- },
"src/app/(dashboard)/policies/_components/impact_popover.tsx": {
"local/filename-pascal-case": {
"count": 1
@@ -1073,11 +1058,6 @@
"count": 1
}
},
- "src/app/(dashboard)/policies/_components/index.test.tsx": {
- "react/display-name": {
- "count": 1
- }
- },
"src/app/(dashboard)/policies/_components/index.tsx": {
"local/filename-pascal-case": {
"count": 1
@@ -1750,7 +1730,7 @@
},
"src/components/add_model/AddModelForm.test.tsx": {
"no-restricted-imports": {
- "count": 2
+ "count": 1
}
},
"src/components/add_model/AddModelForm.tsx": {
@@ -1761,7 +1741,7 @@
"count": 1
},
"no-restricted-imports": {
- "count": 3
+ "count": 2
}
},
"src/components/add_model/ClassificationMethodConfig.tsx": {
@@ -1813,9 +1793,6 @@
},
"no-restricted-imports": {
"count": 3
- },
- "prefer-const": {
- "count": 2
}
},
"src/components/add_model/auto_router_connection_test.tsx": {
@@ -1826,17 +1803,6 @@
"src/components/add_model/cache_control_settings.tsx": {
"local/filename-pascal-case": {
"count": 1
- },
- "no-restricted-imports": {
- "count": 1
- },
- "prefer-const": {
- "count": 1
- }
- },
- "src/components/add_model/conditional_public_model_name.test.tsx": {
- "no-restricted-imports": {
- "count": 1
}
},
"src/components/add_model/conditional_public_model_name.tsx": {
@@ -1863,11 +1829,6 @@
"count": 1
}
},
- "src/components/add_model/litellm_model_name.test.tsx": {
- "no-restricted-imports": {
- "count": 1
- }
- },
"src/components/add_model/litellm_model_name.tsx": {
"local/filename-pascal-case": {
"count": 1
@@ -1887,11 +1848,6 @@
"count": 2
}
},
- "src/components/add_model/provider_specific_fields.test.tsx": {
- "no-restricted-imports": {
- "count": 1
- }
- },
"src/components/add_model/provider_specific_fields.tsx": {
"local/filename-pascal-case": {
"count": 1
@@ -2063,14 +2019,6 @@
"count": 1
}
},
- "src/components/common_components/chartUtils.tsx": {
- "local/filename-pascal-case": {
- "count": 1
- },
- "no-nested-ternary": {
- "count": 1
- }
- },
"src/components/common_components/check_openapi_schema.tsx": {
"local/filename-pascal-case": {
"count": 1
@@ -2294,15 +2242,6 @@
"local/filename-pascal-case": {
"count": 1
},
- "local/no-complex-jsx-arrow": {
- "count": 1
- },
- "max-lines": {
- "count": 1
- },
- "no-nested-ternary": {
- "count": 14
- },
"no-restricted-imports": {
"count": 1
},
@@ -2361,21 +2300,10 @@
"count": 1
}
},
- "src/components/organisms/create_key_button.test.tsx": {
- "@typescript-eslint/no-require-imports": {
- "count": 1
- },
- "react/display-name": {
- "count": 8
- }
- },
"src/components/organisms/create_key_button.tsx": {
"local/filename-pascal-case": {
"count": 1
},
- "local/no-complex-jsx-arrow": {
- "count": 2
- },
"max-lines": {
"count": 1
},
@@ -2669,15 +2597,6 @@
"src/components/templates/key_edit_view.tsx": {
"local/filename-pascal-case": {
"count": 1
- },
- "local/no-complex-jsx-arrow": {
- "count": 2
- },
- "no-nested-ternary": {
- "count": 2
- },
- "no-restricted-imports": {
- "count": 1
}
},
"src/components/templates/key_info_view.tsx": {
@@ -3062,9 +2981,6 @@
"tests/setupTests.ts": {
"@typescript-eslint/no-this-alias": {
"count": 1
- },
- "react/display-name": {
- "count": 1
}
}
}
\ No newline at end of file
diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json
index 93ee979f37d..186d38234d5 100644
--- a/ui/litellm-dashboard/package-lock.json
+++ b/ui/litellm-dashboard/package-lock.json
@@ -18,7 +18,6 @@
"@tanstack/react-pacer": "0.22.1",
"@tanstack/react-query": "5.100.7",
"@tanstack/react-table": "8.21.3",
- "@tremor/react": "3.18.7",
"@types/papaparse": "5.5.2",
"antd": "5.29.3",
"cva": "1.0.0-beta.4",
@@ -1460,87 +1459,12 @@
"@floating-ui/utils": "^0.2.11"
}
},
- "node_modules/@floating-ui/react": {
- "version": "0.19.2",
- "resolved": "https://registry.npmjs.org/@floating-ui/react/-/react-0.19.2.tgz",
- "integrity": "sha512-JyNk4A0Ezirq8FlXECvRtQOX/iBe5Ize0W/pLkrZjfHW9GUV7Xnq6zm6fyZuQzaHHqEnVizmvlA96e1/CkZv+w==",
- "license": "MIT",
- "dependencies": {
- "@floating-ui/react-dom": "^1.3.0",
- "aria-hidden": "^1.1.3",
- "tabbable": "^6.0.1"
- },
- "peerDependencies": {
- "react": ">=16.8.0",
- "react-dom": ">=16.8.0"
- }
- },
- "node_modules/@floating-ui/react-dom": {
- "version": "1.3.0",
- "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-1.3.0.tgz",
- "integrity": "sha512-htwHm67Ji5E/pROEAr7f8IKFShuiCKHwUC/UY4vC3I5jiSvGFAYnSYiZO5MlGmads+QqvUkR9ANHEguGrDv72g==",
- "license": "MIT",
- "dependencies": {
- "@floating-ui/dom": "^1.2.1"
- },
- "peerDependencies": {
- "react": ">=16.8.0",
- "react-dom": ">=16.8.0"
- }
- },
"node_modules/@floating-ui/utils": {
"version": "0.2.11",
"resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.11.tgz",
"integrity": "sha512-RiB/yIh78pcIxl6lLMG0CgBXAZ2Y0eVHqMPYugu+9U0AeT6YBeiJpf7lbdJNIugFP5SIjwNRgo4DhR1Qxi26Gg==",
"license": "MIT"
},
- "node_modules/@headlessui/react": {
- "version": "2.2.0",
- "resolved": "https://registry.npmjs.org/@headlessui/react/-/react-2.2.0.tgz",
- "integrity": "sha512-RzCEg+LXsuI7mHiSomsu/gBJSjpupm6A1qIZ5sWjd7JhARNlMiSA4kKfJpCKwU9tE+zMRterhhrP74PvfJrpXQ==",
- "license": "MIT",
- "dependencies": {
- "@floating-ui/react": "^0.26.16",
- "@react-aria/focus": "^3.17.1",
- "@react-aria/interactions": "^3.21.3",
- "@tanstack/react-virtual": "^3.8.1"
- },
- "engines": {
- "node": ">=10"
- },
- "peerDependencies": {
- "react": "^18 || ^19 || ^19.0.0-rc",
- "react-dom": "^18 || ^19 || ^19.0.0-rc"
- }
- },
- "node_modules/@headlessui/react/node_modules/@floating-ui/react": {
- "version": "0.26.28",
- "resolved": "https://registry.npmjs.org/@floating-ui/react/-/react-0.26.28.tgz",
- "integrity": "sha512-yORQuuAtVpiRjpMhdc0wJj06b9JFjrYF4qp96j++v2NBpbi6SEGF7donUJ3TMieerQ6qVkAv1tgr7L4r5roTqw==",
- "license": "MIT",
- "dependencies": {
- "@floating-ui/react-dom": "^2.1.2",
- "@floating-ui/utils": "^0.2.8",
- "tabbable": "^6.0.0"
- },
- "peerDependencies": {
- "react": ">=16.8.0",
- "react-dom": ">=16.8.0"
- }
- },
- "node_modules/@headlessui/react/node_modules/@floating-ui/react-dom": {
- "version": "2.1.8",
- "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.1.8.tgz",
- "integrity": "sha512-cC52bHwM/n/CxS87FH0yWdngEZrjdtLW/qVruo68qg+prK7ZQ4YGdut2GyDVpoGeAYe/h899rVeOVm6Oi40k2A==",
- "license": "MIT",
- "dependencies": {
- "@floating-ui/dom": "^1.7.6"
- },
- "peerDependencies": {
- "react": ">=16.8.0",
- "react-dom": ">=16.8.0"
- }
- },
"node_modules/@headlessui/tailwindcss": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/@headlessui/tailwindcss/-/tailwindcss-0.2.2.tgz",
@@ -2141,33 +2065,6 @@
"url": "https://opencollective.com/libvips"
}
},
- "node_modules/@internationalized/date": {
- "version": "3.12.1",
- "resolved": "https://registry.npmjs.org/@internationalized/date/-/date-3.12.1.tgz",
- "integrity": "sha512-6IedsVWXyq4P9Tj+TxuU8WGWM70hYLl12nbYU8jkikVpa6WXapFazPUcHUMDMoWftIDE2ILDkFFte6W2nFCkRQ==",
- "license": "Apache-2.0",
- "dependencies": {
- "@swc/helpers": "^0.5.0"
- }
- },
- "node_modules/@internationalized/number": {
- "version": "3.6.6",
- "resolved": "https://registry.npmjs.org/@internationalized/number/-/number-3.6.6.tgz",
- "integrity": "sha512-iFgmQaXHE0vytNfpLZWOC2mEJCBRzcUxt53Xf/yCXG93lRvqas237i3r7X4RKMwO3txiyZD4mQjKAByFv6UGSQ==",
- "license": "Apache-2.0",
- "dependencies": {
- "@swc/helpers": "^0.5.0"
- }
- },
- "node_modules/@internationalized/string": {
- "version": "3.2.8",
- "resolved": "https://registry.npmjs.org/@internationalized/string/-/string-3.2.8.tgz",
- "integrity": "sha512-NdbMQUSfXLYIQol5VyMtinm9pZDciiMfN7RtmSuSB78io1hqwJ0naYfxyW6vgxWBkzWymQa/3uLDlbfmshtCaA==",
- "license": "Apache-2.0",
- "dependencies": {
- "@swc/helpers": "^0.5.0"
- }
- },
"node_modules/@istanbuljs/schema": {
"version": "0.1.6",
"resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.6.tgz",
@@ -2876,44 +2773,6 @@
"react-dom": ">=16.9.0"
}
},
- "node_modules/@react-aria/focus": {
- "version": "3.22.0",
- "resolved": "https://registry.npmjs.org/@react-aria/focus/-/focus-3.22.0.tgz",
- "integrity": "sha512-ZfDOVuVhqDsM9mkNji3QUZ/d40JhlVgXrDkrfXylM1035QCrcTHN7m2DpbE95sU2A8EQb4wikvt5jM6K/73BPg==",
- "license": "Apache-2.0",
- "dependencies": {
- "@swc/helpers": "^0.5.0",
- "react-aria": "3.48.0"
- },
- "peerDependencies": {
- "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1",
- "react-dom": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1"
- }
- },
- "node_modules/@react-aria/interactions": {
- "version": "3.28.0",
- "resolved": "https://registry.npmjs.org/@react-aria/interactions/-/interactions-3.28.0.tgz",
- "integrity": "sha512-OXwdU1EWFdMxmr/K1CXNGJzmNlCClByb+PuCaqUyzBymHPCGVhawirLIon/CrIN5psh3AiWpHSh4H0WeJdVpng==",
- "license": "Apache-2.0",
- "dependencies": {
- "@react-types/shared": "^3.34.0",
- "@swc/helpers": "^0.5.0",
- "react-aria": "3.48.0"
- },
- "peerDependencies": {
- "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1",
- "react-dom": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1"
- }
- },
- "node_modules/@react-types/shared": {
- "version": "3.34.0",
- "resolved": "https://registry.npmjs.org/@react-types/shared/-/shared-3.34.0.tgz",
- "integrity": "sha512-gp6xo/s2lX54AlTjOiqwDnxA7UW79BNvI9dB9pr3LZTzRKCd1ZA+ZbgKw/ReIiWuvvVw/8QFJpnqeeFyLocMcQ==",
- "license": "Apache-2.0",
- "peerDependencies": {
- "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1"
- }
- },
"node_modules/@redocly/ajv": {
"version": "8.11.2",
"resolved": "https://registry.npmjs.org/@redocly/ajv/-/ajv-8.11.2.tgz",
@@ -3839,23 +3698,6 @@
"react-dom": ">=16.8"
}
},
- "node_modules/@tanstack/react-virtual": {
- "version": "3.13.24",
- "resolved": "https://registry.npmjs.org/@tanstack/react-virtual/-/react-virtual-3.13.24.tgz",
- "integrity": "sha512-aIJvz5OSkhNIhZIpYivrxrPTKYsjW9Uzy+sP/mx0S3sev2HyvPb7xmjbYvokzEpfgYHy/HjzJ2zFAETuUfgCpg==",
- "license": "MIT",
- "dependencies": {
- "@tanstack/virtual-core": "3.14.0"
- },
- "funding": {
- "type": "github",
- "url": "https://github.com/sponsors/tannerlinsley"
- },
- "peerDependencies": {
- "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0",
- "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
- }
- },
"node_modules/@tanstack/store": {
"version": "0.11.0",
"resolved": "https://registry.npmjs.org/@tanstack/store/-/store-0.11.0.tgz",
@@ -3879,16 +3721,6 @@
"url": "https://github.com/sponsors/tannerlinsley"
}
},
- "node_modules/@tanstack/virtual-core": {
- "version": "3.14.0",
- "resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.14.0.tgz",
- "integrity": "sha512-JLANqGy/D6k4Ujmh8Tr25lGimuOXNiaVyXaCAZS0W+1390sADdGnyUdSWNIfd49gebtIxGMij4IktRVzrdr12Q==",
- "license": "MIT",
- "funding": {
- "type": "github",
- "url": "https://github.com/sponsors/tannerlinsley"
- }
- },
"node_modules/@testing-library/dom": {
"version": "10.4.1",
"resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.1.tgz",
@@ -3978,93 +3810,6 @@
"@testing-library/dom": ">=7.21.4"
}
},
- "node_modules/@tremor/react": {
- "version": "3.18.7",
- "resolved": "https://registry.npmjs.org/@tremor/react/-/react-3.18.7.tgz",
- "integrity": "sha512-nmqvf/1m0GB4LXc7v2ftdfSLoZhy5WLrhV6HNf0SOriE6/l8WkYeWuhQq8QsBjRi94mUIKLJ/VC3/Y/pj6VubQ==",
- "license": "Apache 2.0",
- "dependencies": {
- "@floating-ui/react": "^0.19.2",
- "@headlessui/react": "2.2.0",
- "date-fns": "^3.6.0",
- "react-day-picker": "^8.10.1",
- "react-transition-state": "^2.1.2",
- "recharts": "^2.13.3",
- "tailwind-merge": "^2.5.2"
- },
- "peerDependencies": {
- "react": "^18.0.0",
- "react-dom": ">=16.6.0"
- }
- },
- "node_modules/@tremor/react/node_modules/eventemitter3": {
- "version": "4.0.7",
- "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz",
- "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==",
- "license": "MIT"
- },
- "node_modules/@tremor/react/node_modules/react-is": {
- "version": "18.3.1",
- "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz",
- "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==",
- "license": "MIT"
- },
- "node_modules/@tremor/react/node_modules/recharts": {
- "version": "2.15.4",
- "resolved": "https://registry.npmjs.org/recharts/-/recharts-2.15.4.tgz",
- "integrity": "sha512-UT/q6fwS3c1dHbXv2uFgYJ9BMFHu3fwnd7AYZaEQhXuYQ4hgsxLvsUXzGdKeZrW5xopzDCvuA2N41WJ88I7zIw==",
- "deprecated": "1.x and 2.x branches are no longer active. Bump to Recharts v3 to receive latest features and bugfixes. See https://github.com/recharts/recharts/wiki/3.0-migration-guide",
- "license": "MIT",
- "dependencies": {
- "clsx": "^2.0.0",
- "eventemitter3": "^4.0.1",
- "lodash": "^4.17.21",
- "react-is": "^18.3.1",
- "react-smooth": "^4.0.4",
- "recharts-scale": "^0.4.4",
- "tiny-invariant": "^1.3.1",
- "victory-vendor": "^36.6.8"
- },
- "engines": {
- "node": ">=14"
- },
- "peerDependencies": {
- "react": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0",
- "react-dom": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
- }
- },
- "node_modules/@tremor/react/node_modules/tailwind-merge": {
- "version": "2.6.1",
- "resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-2.6.1.tgz",
- "integrity": "sha512-Oo6tHdpZsGpkKG88HJ8RR1rg/RdnEkQEfMoEk2x1XRI3F1AxeU+ijRXpiVUF4UbLfcxxRGw6TbUINKYdWVsQTQ==",
- "license": "MIT",
- "funding": {
- "type": "github",
- "url": "https://github.com/sponsors/dcastil"
- }
- },
- "node_modules/@tremor/react/node_modules/victory-vendor": {
- "version": "36.9.2",
- "resolved": "https://registry.npmjs.org/victory-vendor/-/victory-vendor-36.9.2.tgz",
- "integrity": "sha512-PnpQQMuxlwYdocC8fIJqVXvkeViHYzotI+NJrCuav0ZYFoq912ZHBk3mCeuj+5/VpodOjPe1z0Fk2ihgzlXqjQ==",
- "license": "MIT AND ISC",
- "dependencies": {
- "@types/d3-array": "^3.0.3",
- "@types/d3-ease": "^3.0.0",
- "@types/d3-interpolate": "^3.0.1",
- "@types/d3-scale": "^4.0.2",
- "@types/d3-shape": "^3.1.0",
- "@types/d3-time": "^3.0.0",
- "@types/d3-timer": "^3.0.0",
- "d3-array": "^3.1.6",
- "d3-ease": "^3.0.1",
- "d3-interpolate": "^3.0.1",
- "d3-scale": "^4.0.2",
- "d3-shape": "^3.1.0",
- "d3-time": "^3.0.0",
- "d3-timer": "^3.0.1"
- }
- },
"node_modules/@tybys/wasm-util": {
"version": "0.10.3",
"resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz",
@@ -5203,18 +4948,6 @@
"dev": true,
"license": "Python-2.0"
},
- "node_modules/aria-hidden": {
- "version": "1.2.6",
- "resolved": "https://registry.npmjs.org/aria-hidden/-/aria-hidden-1.2.6.tgz",
- "integrity": "sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA==",
- "license": "MIT",
- "dependencies": {
- "tslib": "^2.0.0"
- },
- "engines": {
- "node": ">=10"
- }
- },
"node_modules/aria-query": {
"version": "5.3.0",
"resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz",
@@ -6314,16 +6047,6 @@
"dev": true,
"license": "MIT"
},
- "node_modules/dom-helpers": {
- "version": "5.2.1",
- "resolved": "https://registry.npmjs.org/dom-helpers/-/dom-helpers-5.2.1.tgz",
- "integrity": "sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA==",
- "license": "MIT",
- "dependencies": {
- "@babel/runtime": "^7.8.7",
- "csstype": "^3.0.2"
- }
- },
"node_modules/dunder-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
@@ -7201,15 +6924,6 @@
"dev": true,
"license": "MIT"
},
- "node_modules/fast-equals": {
- "version": "5.4.1",
- "resolved": "https://registry.npmjs.org/fast-equals/-/fast-equals-5.4.1.tgz",
- "integrity": "sha512-DjlFSM5Pk9cGcL0q5QXl66eGzx0N6szNgaswwc5ZphlBohjTVJSnGgI+rJVOgOi65qUoQnDZN4nDqi33udtydQ==",
- "license": "MIT",
- "engines": {
- "node": ">=6.0.0"
- }
- },
"node_modules/fast-glob": {
"version": "3.3.1",
"resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.1.tgz",
@@ -9229,12 +8943,6 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
- "node_modules/lodash": {
- "version": "4.18.1",
- "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz",
- "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==",
- "license": "MIT"
- },
"node_modules/lodash.merge": {
"version": "4.6.2",
"resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
@@ -11868,27 +11576,6 @@
"node": ">=0.10.0"
}
},
- "node_modules/react-aria": {
- "version": "3.48.0",
- "resolved": "https://registry.npmjs.org/react-aria/-/react-aria-3.48.0.tgz",
- "integrity": "sha512-jQjd4rBEIMqecBaAKYJbVGK6EqIHLa5znVQ7jwFyK5vCyljoj6KhgtiahmcIPsG5vG5vEDLw+ba+bEWn6A2P4w==",
- "license": "Apache-2.0",
- "dependencies": {
- "@internationalized/date": "^3.12.1",
- "@internationalized/number": "^3.6.6",
- "@internationalized/string": "^3.2.8",
- "@react-types/shared": "^3.34.0",
- "@swc/helpers": "^0.5.0",
- "aria-hidden": "^1.2.3",
- "clsx": "^2.0.0",
- "react-stately": "3.46.0",
- "use-sync-external-store": "^1.6.0"
- },
- "peerDependencies": {
- "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1",
- "react-dom": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1"
- }
- },
"node_modules/react-copy-to-clipboard": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/react-copy-to-clipboard/-/react-copy-to-clipboard-5.1.1.tgz",
@@ -11902,20 +11589,6 @@
"react": ">=15.3.0"
}
},
- "node_modules/react-day-picker": {
- "version": "8.10.2",
- "resolved": "https://registry.npmjs.org/react-day-picker/-/react-day-picker-8.10.2.tgz",
- "integrity": "sha512-LK68OTbHB3oJNhl9cA0qVizzp3o26w61YSjAFkYi67N86iro32wx86kSNeFU/hq+gI8m1yzWhnomMLfZ041RzQ==",
- "license": "MIT",
- "funding": {
- "type": "individual",
- "url": "https://github.com/sponsors/gpbl"
- },
- "peerDependencies": {
- "date-fns": "^2.28.0 || ^3.0.0",
- "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
- }
- },
"node_modules/react-dom": {
"version": "18.3.1",
"resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz",
@@ -12013,38 +11686,6 @@
}
}
},
- "node_modules/react-smooth": {
- "version": "4.0.4",
- "resolved": "https://registry.npmjs.org/react-smooth/-/react-smooth-4.0.4.tgz",
- "integrity": "sha512-gnGKTpYwqL0Iii09gHobNolvX4Kiq4PKx6eWBCYYix+8cdw+cGo3do906l1NBPKkSWx1DghC1dlWG9L2uGd61Q==",
- "license": "MIT",
- "dependencies": {
- "fast-equals": "^5.0.1",
- "prop-types": "^15.8.1",
- "react-transition-group": "^4.4.5"
- },
- "peerDependencies": {
- "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0",
- "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
- }
- },
- "node_modules/react-stately": {
- "version": "3.46.0",
- "resolved": "https://registry.npmjs.org/react-stately/-/react-stately-3.46.0.tgz",
- "integrity": "sha512-OdxhWvHgs2L4OJGIs7hnuTr5WjjMM6enhNEAMRqiekhF8+ITvA2LRwNftOZwcogaoCslGYq5S2VQTQwnm0GbCA==",
- "license": "Apache-2.0",
- "dependencies": {
- "@internationalized/date": "^3.12.1",
- "@internationalized/number": "^3.6.6",
- "@internationalized/string": "^3.2.8",
- "@react-types/shared": "^3.34.0",
- "@swc/helpers": "^0.5.0",
- "use-sync-external-store": "^1.6.0"
- },
- "peerDependencies": {
- "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1"
- }
- },
"node_modules/react-syntax-highlighter": {
"version": "15.6.6",
"resolved": "https://registry.npmjs.org/react-syntax-highlighter/-/react-syntax-highlighter-15.6.6.tgz",
@@ -12062,32 +11703,6 @@
"react": ">= 0.14.0"
}
},
- "node_modules/react-transition-group": {
- "version": "4.4.5",
- "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.5.tgz",
- "integrity": "sha512-pZcd1MCJoiKiBR2NRxeCRg13uCXbydPnmB4EOeRrY7480qNWO8IIgQG6zlDkm6uRMsURXPuKq0GWtiM59a5Q6g==",
- "license": "BSD-3-Clause",
- "dependencies": {
- "@babel/runtime": "^7.5.5",
- "dom-helpers": "^5.0.1",
- "loose-envify": "^1.4.0",
- "prop-types": "^15.6.2"
- },
- "peerDependencies": {
- "react": ">=16.6.0",
- "react-dom": ">=16.6.0"
- }
- },
- "node_modules/react-transition-state": {
- "version": "2.3.3",
- "resolved": "https://registry.npmjs.org/react-transition-state/-/react-transition-state-2.3.3.tgz",
- "integrity": "sha512-wsIyg07ohlWEAYDZHvuXh/DY7mxlcLb0iqVv2aMXJ0gwgPVKNWKhOyNyzuJy/tt/6urSq0WT6BBZ/tdpybaAsQ==",
- "license": "MIT",
- "peerDependencies": {
- "react": ">=16.8.0",
- "react-dom": ">=16.8.0"
- }
- },
"node_modules/recharts": {
"version": "3.9.2",
"resolved": "https://registry.npmjs.org/recharts/-/recharts-3.9.2.tgz",
@@ -12118,15 +11733,6 @@
"react-is": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
}
},
- "node_modules/recharts-scale": {
- "version": "0.4.5",
- "resolved": "https://registry.npmjs.org/recharts-scale/-/recharts-scale-0.4.5.tgz",
- "integrity": "sha512-kivNFO+0OcUNu7jQquLXAxz1FIwZj8nrj+YkOKc5694NbjCvcT6aSZiIzNzd2Kul4o4rTto8QVR9lMNtxD4G1w==",
- "license": "MIT",
- "dependencies": {
- "decimal.js-light": "^2.4.1"
- }
- },
"node_modules/redent": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/redent/-/redent-3.0.0.tgz",
@@ -13200,12 +12806,6 @@
"dev": true,
"license": "MIT"
},
- "node_modules/tabbable": {
- "version": "6.4.0",
- "resolved": "https://registry.npmjs.org/tabbable/-/tabbable-6.4.0.tgz",
- "integrity": "sha512-05PUHKSNE8ou2dwIxTngl4EzcnsCDZGJ/iCLtDflR/SHB/ny14rXc+qU5P4mG9JkusiV7EivzY9Mhm55AzAvCg==",
- "license": "MIT"
- },
"node_modules/tailwind-merge": {
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-3.4.0.tgz",
diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json
index 483ab39c336..9407ff25a1f 100644
--- a/ui/litellm-dashboard/package.json
+++ b/ui/litellm-dashboard/package.json
@@ -31,7 +31,6 @@
"@tanstack/react-pacer": "0.22.1",
"@tanstack/react-query": "5.100.7",
"@tanstack/react-table": "8.21.3",
- "@tremor/react": "3.18.7",
"@types/papaparse": "5.5.2",
"antd": "5.29.3",
"cva": "1.0.0-beta.4",
@@ -103,7 +102,6 @@
"axios": "1.13.6",
"postcss": "8.5.23",
"esbuild": "0.28.1",
- "date-fns": "^4.4.0",
"sharp": "^0.35.0"
},
"engines": {
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/TagsInput.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/TagsInput.tsx
index 0d799474399..6fb1508497e 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/TagsInput.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/TagsInput.tsx
@@ -28,6 +28,7 @@ interface TagsInputProps {
emptyText?: string;
tokenSeparators?: string[];
loading?: boolean;
+ disabled?: boolean;
id?: string;
}
@@ -48,6 +49,7 @@ export const TagsInput = ({
emptyText = "No matching options",
tokenSeparators = [],
loading = false,
+ disabled = false,
id,
}: TagsInputProps) => {
const anchor = useComboboxAnchor();
@@ -103,6 +105,7 @@ export const TagsInput = ({
itemToStringLabel={(option: TagsInputOption) => option.label}
filter={matchesQuery}
openOnInputClick
+ disabled={disabled || loading}
>
} className="min-h-8 py-1 text-sm">
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx
index 320e0b5baef..b6ee130d50a 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx
@@ -1,5 +1,6 @@
import * as networking from "@/components/networking";
import { fireEvent, render, waitFor, within } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
import { afterEach, describe, expect, it, vi } from "vitest";
import GuardrailInfoView from "./guardrail_info";
@@ -136,7 +137,7 @@ describe("Guardrail Info", () => {
vi.mocked(networking.getGuardrailProviderSpecificParams).mockResolvedValue({});
- const { getByText, container } = render(
+ const { getByText, findByText, container } = render(
{}} accessToken="123" isAdmin={true} />,
);
@@ -152,18 +153,9 @@ describe("Guardrail Info", () => {
expect(getByText("Guardrail Settings")).toBeInTheDocument();
});
- // Find the info icon and hover over it
- const infoIcon = within(container).getByRole("img", { name: "info-circle" });
- expect(infoIcon).toBeInTheDocument();
+ await userEvent.hover(within(container).getByRole("img", { name: "info-circle" }));
- if (infoIcon) {
- fireEvent.mouseEnter(infoIcon);
-
- // Wait for the tooltip to appear
- await waitFor(() => {
- expect(getByText("Guardrail is defined in the config file and cannot be edited.")).toBeInTheDocument();
- });
- }
+ expect(await findByText("Guardrail is defined in the config file and cannot be edited.")).toBeInTheDocument();
});
it("should render the guardrail info", async () => {
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx
index caac09a1c83..e80ddac932f 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx
@@ -6,7 +6,8 @@ import {
} from "@/components/networking";
import { copyToClipboard as utilCopyToClipboard } from "@/utils/dataUtils";
import { CodeOutlined, EyeInvisibleOutlined, InfoCircleOutlined, StopOutlined } from "@ant-design/icons";
-import { Button as AntdButton, Tooltip } from "antd";
+
+import { Button as AntdButton } from "antd";
import { ArrowLeft, CheckIcon, CopyIcon } from "lucide-react";
import { Badge } from "@/components/ui/badge";
import { Card } from "@/components/ui/card";
@@ -21,7 +22,7 @@ import { Input } from "@/components/ui/input";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { Separator } from "@/components/ui/separator";
import { Textarea } from "@/components/ui/textarea";
-import { TooltipProvider } from "@/components/ui/tooltip";
+import { SimpleTooltip, TooltipProvider } from "@/components/ui/tooltip";
import {
asText,
GuardrailField,
@@ -663,9 +664,9 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose,
Guardrail Settings
{isConfigGuardrail && (
-
+
-
+
)}
{!isEditing &&
!isConfigGuardrail &&
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/AwsSigV4Fields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/AwsSigV4Fields.tsx
index d4ae537bffa..09016bda266 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/AwsSigV4Fields.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/AwsSigV4Fields.tsx
@@ -1,7 +1,25 @@
import React from "react";
-import { Form, Input, Tooltip } from "antd";
+import { Input, Tooltip } from "antd";
import { InfoCircleOutlined } from "@ant-design/icons";
+import { MountedFormField } from "@/components/common_components/MountedFormField";
+import { antdRequired } from "@/components/common_components/antdFormRules";
+import { requiredWhenSiblingSet, textControl } from "./mcpFieldRules";
+
+const fieldClassName = "rounded-lg border-gray-300 focus:border-blue-500 focus:ring-blue-500";
+
+const FieldLabel: React.FC<{ label: string; tooltip: string }> = ({ label, tooltip }) => (
+
+ {label}
+
+
+
+
+);
+
+const ACCESS_KEY_PATH = ["credentials", "aws_access_key_id"] as const;
+const SECRET_KEY_PATH = ["credentials", "aws_secret_access_key"] as const;
+
const AwsSigV4Fields: React.FC = () => (
<>
@@ -15,140 +33,120 @@ const AwsSigV4Fields: React.FC = () => (
View docs →
-
- AWS Region
-
-
-
-
- }
+ }
name={["credentials", "aws_region_name"]}
- rules={[{ required: true, message: "AWS region is required for SigV4 auth" }]}
+ required
+ rules={{ validate: { required: antdRequired("AWS region is required for SigV4 auth") } }}
>
-
-
-
}
+
+
- AWS Service Name
-
-
-
-
+
}
name={["credentials", "aws_service_name"]}
>
-
-
-
}
+
+
- AWS Access Key ID
-
-
-
-
+
}
- name={["credentials", "aws_access_key_id"]}
- dependencies={[["credentials", "aws_secret_access_key"]]}
- rules={[
- ({ getFieldValue }) => ({
- validator(_, value) {
- const secretKey = getFieldValue(["credentials", "aws_secret_access_key"]);
- if (secretKey && !value) {
- return Promise.reject(new Error("Access Key ID is required when Secret Access Key is provided"));
- }
- return Promise.resolve();
- },
- }),
- ]}
+ name={ACCESS_KEY_PATH}
+ rules={{
+ deps: ["credentials.aws_secret_access_key"],
+ validate: {
+ pairedWithSecret: requiredWhenSiblingSet(
+ SECRET_KEY_PATH,
+ "Access Key ID is required when Secret Access Key is provided",
+ ),
+ },
+ }}
>
-
-
-
(
+
+ )}
+
+
- AWS Secret Access Key
-
-
-
-
+
}
- name={["credentials", "aws_secret_access_key"]}
- dependencies={[["credentials", "aws_access_key_id"]]}
- rules={[
- ({ getFieldValue }) => ({
- validator(_, value) {
- const accessKeyId = getFieldValue(["credentials", "aws_access_key_id"]);
- if (accessKeyId && !value) {
- return Promise.reject(new Error("Secret Access Key is required when Access Key ID is provided"));
- }
- return Promise.resolve();
- },
- }),
- ]}
+ name={SECRET_KEY_PATH}
+ rules={{
+ deps: ["credentials.aws_access_key_id"],
+ validate: {
+ pairedWithAccessKey: requiredWhenSiblingSet(
+ ACCESS_KEY_PATH,
+ "Secret Access Key is required when Access Key ID is provided",
+ ),
+ },
+ }}
>
-
-
-
- AWS Session Token
-
-
-
-
- }
+ {(control) => (
+
+ )}
+
+ }
name={["credentials", "aws_session_token"]}
>
-
-
-
(
+
+ )}
+
+
- AWS Role ARN
-
-
-
-
+
}
name={["credentials", "aws_role_name"]}
>
-
-
-
(
+
+ )}
+
+
- AWS Session Name
-
-
-
-
+
}
name={["credentials", "aws_session_name"]}
>
-
-
+ {(control) => (
+
+ )}
+
>
);
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.integration.test.tsx
index f08aacb5522..78d89b7e19e 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.integration.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.integration.test.tsx
@@ -2147,7 +2147,7 @@ describe("CreateMCPServer dcr_bridge toggle", () => {
});
// Forcing dcr_bridge false for every non-client-forwarded auth type is covered in
- // createServerPayload.test.ts. The two form-state cases below stay: they prove the Form.Item
+ // createServerPayload.test.ts. The two form-state cases below stay: they prove the field
// unmounts on a switch away, and that the live value survives a client-forwarded swap.
it("forces dcr_bridge: false when the auth type is switched away after toggling", async () => {
@@ -2179,7 +2179,7 @@ describe("CreateMCPServer dcr_bridge toggle", () => {
});
expect(getDcrToggle()).toHaveAttribute("aria-checked", "true");
- // The Form.Item is mounted in both client-forwarded modes, so switching between them keeps the
+ // The field is mounted in both client-forwarded modes, so switching between them keeps the
// live toggle value rather than forcing it back to the default or to false.
await selectAntOption("Authentication", "OAuth Delegate (client-supplied upstream token)");
await waitFor(() => {
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.permissions.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.permissions.integration.test.tsx
new file mode 100644
index 00000000000..788be106d8a
--- /dev/null
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.permissions.integration.test.tsx
@@ -0,0 +1,160 @@
+import { act, fireEvent, render, screen, waitFor } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import * as networking from "@/components/networking";
+import CreateMCPServer from "./CreateMCPServer";
+import { selectAntOption } from "./testUtils";
+
+vi.mock("@/components/networking", () => ({
+ createMCPServer: vi.fn(),
+ fetchOpenAPIRegistry: vi.fn().mockResolvedValue({ apis: [] }),
+ registerMCPServer: vi.fn(),
+ storeMCPOAuthUserCredential: vi.fn().mockResolvedValue({}),
+ testMCPToolsListRequest: vi.fn().mockResolvedValue({ tools: [], error: null }),
+}));
+
+vi.mock("@/utils/mcpTokenStore", () => ({
+ setToken: vi.fn(),
+}));
+
+vi.mock("./OpenAPIQuickPicker", () => ({
+ default: () => null,
+}));
+
+vi.mock("@/hooks/useMcpOAuthFlow", () => ({
+ useMcpOAuthFlow: () => ({
+ startOAuthFlow: vi.fn(),
+ status: "idle",
+ error: null,
+ tokenResponse: null,
+ reset: vi.fn(),
+ }),
+}));
+
+vi.mock("./mcp_server_cost_config", () => ({
+ default: () =>
,
+}));
+
+vi.mock("./mcp_tool_configuration", () => ({
+ default: () =>
,
+}));
+
+vi.mock("./mcp_connection_status", () => ({
+ default: () =>
,
+}));
+
+vi.mock("./StdioConfiguration", () => ({
+ default: () =>
,
+}));
+
+const defaultProps = {
+ userRole: "Admin",
+ accessToken: "test-token",
+ onCreateSuccess: vi.fn(),
+ isModalVisible: true,
+ setModalVisible: vi.fn(),
+ availableAccessGroups: ["group-a", "group-b"],
+};
+
+const getServerNameInput = () => document.getElementById("server_name") as HTMLInputElement;
+
+const switchFor = (labelText: string): HTMLElement => {
+ const label = screen.getByText(labelText);
+ const row = label.closest(".flex.items-start.justify-between");
+ const control = row?.querySelector("button[role='switch']");
+ if (control === null || control === undefined) {
+ throw new Error(`no switch found for "${labelText}"`);
+ }
+ return control as HTMLElement;
+};
+
+const fillMinimalHttpServer = async (name: string) => {
+ await selectAntOption("Transport Type", "Streamable HTTP");
+ await waitFor(() => {
+ expect(screen.getByPlaceholderText("https://your-mcp-server.com")).toBeInTheDocument();
+ });
+ const user = userEvent.setup({ delay: null });
+ await user.type(getServerNameInput(), name);
+ await user.type(screen.getByPlaceholderText("https://your-mcp-server.com"), "https://example.com/mcp");
+ await selectAntOption("Authentication", "None");
+};
+
+const submitAndReadPayload = async () => {
+ await act(async () => {
+ fireEvent.click(screen.getByRole("button", { name: "Add MCP Server" }));
+ });
+ await waitFor(() => expect(networking.createMCPServer).toHaveBeenCalledTimes(1));
+ return vi.mocked(networking.createMCPServer).mock.calls[0][1];
+};
+
+const createdServer = {
+ server_id: "new-server-1",
+ server_name: "Perm_Server",
+ alias: "Perm_Server",
+ url: "https://example.com/mcp",
+ transport: "http",
+ auth_type: "none",
+ created_at: "2024-01-01T00:00:00Z",
+ created_by: "user-1",
+ updated_at: "2024-01-01T00:00:00Z",
+ updated_by: "user-1",
+};
+
+describe("CreateMCPServer permission toggles reaching the payload", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ vi.mocked(networking.createMCPServer).mockResolvedValue(createdServer);
+ });
+
+ it("sends the panel's untouched defaults rather than dropping the keys the panel owns", async () => {
+ render(
);
+ await fillMinimalHttpServer("Perm_Server");
+
+ const payload = await submitAndReadPayload();
+
+ expect(payload.allow_all_keys).toBe(false);
+ expect(payload.available_on_public_internet).toBe(true);
+ });
+
+ it("sends allow_all_keys true once the operator turns the public-to-all-keys switch on", async () => {
+ render(
);
+ await fillMinimalHttpServer("Perm_Server");
+
+ await act(async () => {
+ fireEvent.click(switchFor("Allow All LiteLLM Keys"));
+ });
+
+ const payload = await submitAndReadPayload();
+
+ expect(payload.allow_all_keys).toBe(true);
+ });
+
+ it("sends available_on_public_internet false when the operator restricts the server to the internal network", async () => {
+ render(
);
+ await fillMinimalHttpServer("Perm_Server");
+
+ const internalOnly = switchFor("Internal network only");
+ expect(internalOnly).toHaveAttribute("aria-checked", "false");
+
+ await act(async () => {
+ fireEvent.click(internalOnly);
+ });
+ expect(internalOnly).toHaveAttribute("aria-checked", "true");
+
+ const payload = await submitAndReadPayload();
+
+ expect(payload.available_on_public_internet).toBe(false);
+ });
+
+ it("omits delegate_auth_to_upstream's true value on a none-auth server, whose gate never mounts that switch", async () => {
+ render(
);
+ await fillMinimalHttpServer("Perm_Server");
+
+ expect(screen.getByText("Allow All LiteLLM Keys")).toBeInTheDocument();
+ expect(screen.queryByText("Delegate auth to upstream (PKCE passthrough)")).not.toBeInTheDocument();
+
+ const payload = await submitAndReadPayload();
+
+ expect(payload.delegate_auth_to_upstream).toBe(false);
+ });
+});
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx
index ed8d3bddc98..92bdd14754c 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx
@@ -1,5 +1,6 @@
import React, { useState } from "react";
-import { Modal, Tooltip, Form, Select, Input as AntdInput, InputNumber, Collapse } from "antd";
+import { Modal, Tooltip, Select, Input as AntdInput, InputNumber, Collapse } from "antd";
+import { FormProvider, useForm, useWatch } from "react-hook-form";
import { InfoCircleOutlined } from "@ant-design/icons";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
@@ -49,6 +50,16 @@ import { validateMCPServerUrl, validateMCPServerName } from "./utils";
import { toast } from "@/lib/toast";
import { useMcpOAuthFlow } from "@/hooks/useMcpOAuthFlow";
import { useTestMCPConnection } from "@/hooks/useTestMCPConnection";
+import {
+ MountedFormField,
+ MountedFormProvider,
+ projectMountedValues,
+ useMountRegistry,
+ type MountedFormValues,
+} from "@/components/common_components/MountedFormField";
+import { antdRequired, antdRules } from "@/components/common_components/antdFormRules";
+import { allFieldsValue, mountedPaths, resetFields, setFieldsValue, singleBranchChange } from "./mcpFormStore";
+import { numberControl, notOnlyWhitespace, selectControl, textControl } from "./mcpFieldRules";
import mcpLogo from "../../../../../public/assets/logos/mcp_logo.png";
export const mcpLogoImg = mcpLogo.src;
@@ -76,6 +87,13 @@ const payloadErrorMessage = (result: Exclude
= ({
userID,
userRole,
@@ -87,7 +105,8 @@ const CreateMCPServer: React.FC = ({
prefillData,
onBackToDiscovery,
}) => {
- const [form] = Form.useForm();
+ const form = useForm({ mode: "onChange", defaultValues: CREATE_DEFAULTS });
+ const registry = useMountRegistry();
const [isLoading, setIsLoading] = useState(false);
const [costConfig, setCostConfig] = useState({});
const [formValues, setFormValues] = useState>({});
@@ -136,6 +155,8 @@ const CreateMCPServer: React.FC = ({
enabled: true,
});
+ const authSectionMounted = transportType !== "stdio" && transportType !== "";
+ const watchedAuthType = useWatch({ control: form.control, name: "auth_type" }) as string | undefined;
const authType = formValues.auth_type as string | undefined;
const shouldShowAuthValueField = authType ? AUTH_TYPES_REQUIRING_AUTH_VALUE.includes(authType) : false;
const isOAuthAuthType = authType === AUTH_TYPE.OAUTH2;
@@ -147,7 +168,7 @@ const CreateMCPServer: React.FC = ({
const persistCreateUiState = () => {
writeCreateUiSnapshot({
modalVisible: isModalVisible,
- formValues: form.getFieldsValue(true),
+ formValues: allFieldsValue(form),
transportType,
costConfig,
allowedTools,
@@ -170,11 +191,11 @@ const CreateMCPServer: React.FC = ({
// Merge the ref-held DCR client so a re-authorize reuses the registered client instead of
// re-registering; the form store itself never holds the DCR client (see onTokenReceived).
getCredentials: () => ({
- ...((form.getFieldValue("credentials") as Record | undefined) ?? {}),
+ ...((allFieldsValue(form).credentials as Record | undefined) ?? {}),
...(dcrClientRef.current ?? {}),
}),
getTemporaryPayload: () => {
- const values = form.getFieldsValue(true);
+ const values = allFieldsValue(form);
const transport = values.transport || transportType;
// For OpenAPI transport the form has spec_path instead of url.
// We pass the spec_path as url so the temp-session endpoint has something
@@ -218,12 +239,12 @@ const CreateMCPServer: React.FC = ({
return;
}
- if (isClientForwardedTokenMode(form.getFieldValue("auth_type"))) {
+ if (isClientForwardedTokenMode(allFieldsValue(form).auth_type)) {
// Browser-only modes: the token is held in local state (oauthAccessToken) for tool preview
// and committed to sessionStorage on submit; it must never be written into form.credentials,
// which would persist it as server-level credentials on the created server row. Mirrors the
// edit form's onTokenReceived early return.
- setAuthorizedIdentity(getOAuthAuthorizationIdentity(form.getFieldsValue(true)));
+ setAuthorizedIdentity(getOAuthAuthorizationIdentity(allFieldsValue(form)));
toast.success(
"Token held for this browser session. Tools can now be previewed and configured; the token is not saved to LiteLLM.",
);
@@ -240,7 +261,7 @@ const CreateMCPServer: React.FC = ({
}
: null;
- const current = (form.getFieldValue("credentials") as Record | undefined) ?? {};
+ const current = (allFieldsValue(form).credentials as Record | undefined) ?? {};
const nextCredentials = {
...(preservedAdminCredentials(current) ?? {}),
...(current.scopes !== undefined && { scopes: current.scopes }),
@@ -252,10 +273,10 @@ const CreateMCPServer: React.FC = ({
// Path-replace (not deep-merge) so a re-authorize with fewer token fields does not leave stale
// siblings from the previous token behind; the admin-typed client keys and scopes are carried
// explicitly above.
- form.setFieldValue("credentials", nextCredentials);
+ form.setValue("credentials", nextCredentials);
// Capture the identity AFTER writing the token so the held token is not spuriously invalidated by
// its own credential write.
- setAuthorizedIdentity(getOAuthAuthorizationIdentity(form.getFieldsValue(true)));
+ setAuthorizedIdentity(getOAuthAuthorizationIdentity(allFieldsValue(form)));
toast.success("OAuth authorization successful! Please click 'Create MCP Server' to save the configuration.");
},
@@ -277,10 +298,10 @@ const CreateMCPServer: React.FC = ({
// Capture the admin-typed app before resetFields destroys it, then re-apply it: the app is
// upstream-scoped config, not minted material, so it survives every invalidation (the token is
// what gets discarded). Token-shaped keys are excluded by the helper's key filter.
- const keptAdminCredentials = preservedAdminCredentials(form.getFieldValue("credentials"));
- form.resetFields([...CLEARED_ON_INVALIDATION]);
+ const keptAdminCredentials = preservedAdminCredentials(allFieldsValue(form).credentials);
+ resetFields(form, [...CLEARED_ON_INVALIDATION]);
if (keptAdminCredentials) {
- form.setFieldsValue({ credentials: keptAdminCredentials });
+ setFieldsValue(form, { credentials: keptAdminCredentials });
}
// Re-apply the in-flight edit last; rc-field-form deep-merges nested objects, so a changed
// credentials sub-field composes with the preserved sibling instead of replacing the object.
@@ -288,7 +309,7 @@ const CreateMCPServer: React.FC = ({
CLEARED_ON_INVALIDATION.filter((key) => key in changedValues).map((key) => [key, changedValues[key]]),
);
if (Object.keys(preserved).length > 0) {
- form.setFieldsValue(preserved);
+ setFieldsValue(form, preserved);
}
};
@@ -337,7 +358,7 @@ const CreateMCPServer: React.FC = ({
// wait until transportType state catches up so the URL field is mounted
return;
}
- form.setFieldsValue(pendingRestoredValues.values);
+ setFieldsValue(form, pendingRestoredValues.values);
setFormValues(pendingRestoredValues.values);
setPendingRestoredValues(null);
}, [pendingRestoredValues, form, transportType]);
@@ -381,11 +402,20 @@ const CreateMCPServer: React.FC = ({
prefillValues.url = prefillData.url;
}
- form.setFieldsValue(prefillValues);
+ setFieldsValue(form, prefillValues);
setFormValues(prefillValues);
setAliasManuallyEdited(false);
}, [isModalVisible, prefillData, form]);
+ const handleSubmit = async (event: React.FormEvent) => {
+ event.preventDefault();
+ const isValid = await form.trigger(mountedPaths(registry) as string[]);
+ if (!isValid) {
+ return;
+ }
+ await handleCreate(projectMountedValues(registry, form.getValues));
+ };
+
const handleCreate = async (values: Record) => {
const built = buildCreateServerPayload(values, {
transportType,
@@ -446,7 +476,7 @@ const CreateMCPServer: React.FC = ({
description: "Once an admin approves it, the server will appear in your MCP Servers list.",
});
}
- form.resetFields();
+ form.reset(CREATE_DEFAULTS);
setCostConfig({});
clearTools();
setAllowedTools([]);
@@ -466,7 +496,7 @@ const CreateMCPServer: React.FC = ({
// state
const handleCancel = () => {
- form.resetFields();
+ form.reset(CREATE_DEFAULTS);
setCostConfig({});
clearTools();
setAllowedTools([]);
@@ -489,11 +519,11 @@ const CreateMCPServer: React.FC = ({
? { url: undefined, command: undefined, args: undefined, env: undefined }
: { spec_path: undefined, command: undefined, args: undefined, env: undefined };
- form.setFieldsValue(transportValues);
- if (isHeldOAuthTokenStale(form.getFieldsValue(true), authorizedIdentity)) {
+ setFieldsValue(form, transportValues);
+ if (isHeldOAuthTokenStale(allFieldsValue(form), authorizedIdentity)) {
clearHeldOAuthToken();
}
- setFormValues(form.getFieldsValue(true));
+ setFormValues(allFieldsValue(form));
};
// Generate options with existing groups and potential new group
@@ -532,7 +562,7 @@ const CreateMCPServer: React.FC = ({
React.useEffect(() => {
if (!aliasManuallyEdited && formValues.server_name) {
const normalized = formValues.server_name.replace(/\s+/g, "_");
- form.setFieldsValue({ alias: normalized });
+ setFieldsValue(form, { alias: normalized });
setFormValues((prev) => ({ ...prev, alias: normalized }));
}
}, [formValues.server_name]);
@@ -549,7 +579,7 @@ const CreateMCPServer: React.FC = ({
const wasVisible = wasModalVisibleRef.current;
wasModalVisibleRef.current = isModalVisible;
if (!isModalVisible && wasVisible) {
- form.resetFields();
+ form.reset(CREATE_DEFAULTS);
setFormValues({});
setOauthAccessToken(null);
clearTools();
@@ -582,19 +612,35 @@ const CreateMCPServer: React.FC = ({
const upstreamChanged = ["url", "spec_path", "issuer", "authorization_url", "token_url", "registration_url"].some(
(key) => key in changedValues,
);
- const hasDeclaredApp = preservedDeclaredAppCredentials(form.getFieldValue("credentials")) !== undefined;
+ const hasDeclaredApp = preservedDeclaredAppCredentials(allFieldsValue(form).credentials) !== undefined;
if (upstreamChanged && hasDeclaredApp) {
setAppMayNotMatchUpstream(true);
}
}
- if (isHeldOAuthTokenStale(form.getFieldsValue(true), authorizedIdentity)) {
+ if (isHeldOAuthTokenStale(allFieldsValue(form), authorizedIdentity)) {
clearHeldOAuthToken(changedValues);
- setFormValues(form.getFieldsValue(true));
+ setFormValues(allFieldsValue(form));
return;
}
setFormValues(allValues);
};
+ const valuesChangeRef = React.useRef(handleFormValuesChange);
+ valuesChangeRef.current = handleFormValuesChange;
+
+ React.useEffect(() => {
+ const subscription = form.watch((values, { name, type }) => {
+ if (type !== "change" || name === undefined) {
+ return;
+ }
+ valuesChangeRef.current(
+ singleBranchChange(name, values as MountedFormValues),
+ projectMountedValues(registry, form.getValues),
+ );
+ });
+ return () => subscription.unsubscribe();
+ }, [form, registry]);
+
// rendering
return (
= ({
}}
>
-
+ {/* Cost Configuration Section */}
+
+ allowedTools.includes(tool.name))}
+ disabled={false}
+ />
+
+
+
+
+ Cancel
+
+
+ {isLoading && }
+ {isLoading ? "Creating..." : "Add MCP Server"}
+
+
+
+
+
);
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/DcrBridgeToggle.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/DcrBridgeToggle.tsx
index 49c182aa6be..35b23f9873c 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/DcrBridgeToggle.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/DcrBridgeToggle.tsx
@@ -1,16 +1,19 @@
import React from "react";
-import { Form, Switch, Tooltip } from "antd";
+import { Switch, Tooltip } from "antd";
import { InfoCircleOutlined } from "@ant-design/icons";
+
+import { MountedFormField } from "@/components/common_components/MountedFormField";
import { isClientForwardedTokenMode } from "@/components/mcp_tools/types";
+import { switchControl } from "./mcpFieldRules";
/**
* DCR-bridge toggle for the client-forwarded token modes (true_passthrough /
* oauth_delegate); self-gates to those two auth types and renders nothing
* otherwise. When on, OAuth-only clients like Claude Desktop can register and
* sign in through the gateway; when off, the gateway relays the upstream
- * server's own OAuth metadata instead. `initialChecked` seeds the antd
- * Form.Item `initialValue` (not the Switch's DOM defaultChecked): the create
- * form defaults it on, the edit form seeds it from the stored value.
+ * server's own OAuth metadata instead. `initialChecked` seeds the field's
+ * default value (not the Switch's DOM defaultChecked): the create form defaults
+ * it on, the edit form seeds it from the stored value.
*/
export default function DcrBridgeToggle({
authType,
@@ -21,7 +24,7 @@ export default function DcrBridgeToggle({
}) {
if (!isClientForwardedTokenMode(authType)) return null;
return (
-
Gateway-hosted sign-in (DCR bridge)
@@ -31,10 +34,9 @@ export default function DcrBridgeToggle({
}
name="dcr_bridge"
- valuePropName="checked"
- initialValue={initialChecked}
+ defaultValue={initialChecked}
>
-
-
+ {(control) => }
+
);
}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.test.tsx
new file mode 100644
index 00000000000..a57bde7eb9d
--- /dev/null
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.test.tsx
@@ -0,0 +1,86 @@
+import React from "react";
+import { describe, it, expect, vi } from "vitest";
+import { render, screen } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
+import { FormProvider, useForm } from "react-hook-form";
+
+import {
+ MountedFormProvider,
+ projectMountedValues,
+ useMountRegistry,
+ type MountedFormValues,
+} from "@/components/common_components/MountedFormField";
+import EnvVarsSection from "./EnvVarsSection";
+
+const renderSection = (defaultValues: MountedFormValues) => {
+ const onFinish = vi.fn();
+ const Harness: React.FC = () => {
+ const form = useForm({ mode: "onChange", defaultValues });
+ const registry = useMountRegistry();
+ return (
+
+
+ {
+ event.preventDefault();
+ onFinish(projectMountedValues(registry, form.getValues));
+ }}
+ >
+
+ Submit
+
+
+
+ );
+ };
+ render( );
+ return onFinish;
+};
+
+describe("EnvVarsSection", () => {
+ it("submits a per-user row whole, keeping the value key whose input the scope hides", async () => {
+ const onFinish = renderSection({
+ env_vars: [{ name: "DB_USER", value: "admin", scope: "user", description: "Your DB username" }],
+ });
+
+ expect(screen.queryByPlaceholderText("e.g. postgresql")).not.toBeInTheDocument();
+ await userEvent.click(screen.getByText("Submit"));
+
+ expect(onFinish).toHaveBeenCalledWith(
+ expect.objectContaining({
+ env_vars: [{ name: "DB_USER", value: "admin", scope: "user", description: "Your DB username" }],
+ }),
+ );
+ });
+
+ it("submits an empty env_vars key when the list has no rows, rather than dropping the key", async () => {
+ const onFinish = renderSection({ env_vars: [] });
+
+ await userEvent.click(screen.getByText("Submit"));
+
+ expect(onFinish.mock.calls[0][0]).toHaveProperty("env_vars", []);
+ });
+
+ it("carries a row added after mount into the submitted list, scoped global without the user picking one", async () => {
+ const onFinish = renderSection({ env_vars: [] });
+
+ await userEvent.click(screen.getByText("Add Variable"));
+ await userEvent.type(screen.getByPlaceholderText("e.g. DB_PROTOCOL"), "DB_PROTOCOL");
+ await userEvent.type(screen.getByPlaceholderText("e.g. postgresql"), "postgresql");
+ await userEvent.click(screen.getByText("Submit"));
+
+ expect(onFinish).toHaveBeenCalledWith(
+ expect.objectContaining({
+ env_vars: [expect.objectContaining({ name: "DB_PROTOCOL", value: "postgresql", scope: "global" })],
+ }),
+ );
+ });
+
+ it("rejects a variable name that starts with a digit", async () => {
+ renderSection({ env_vars: [{ name: "", value: "", scope: "global", description: "" }] });
+
+ await userEvent.type(screen.getByPlaceholderText("e.g. DB_PROTOCOL"), "9LIVES");
+
+ expect(await screen.findByText("Use letters, digits, underscores; cannot start with a digit.")).toBeInTheDocument();
+ });
+});
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.tsx
index fbaacc40263..539a06910c1 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.tsx
@@ -1,6 +1,16 @@
import React from "react";
-import { Form, Input, Select, Button, Tooltip, Typography } from "antd";
+import { Input, Select, Button, Tooltip, Typography } from "antd";
import { InfoCircleOutlined, MinusCircleOutlined, PlusOutlined } from "@ant-design/icons";
+import { useFieldArray, useFormContext, useWatch } from "react-hook-form";
+
+import {
+ MountedFormField,
+ useMountedName,
+ type MountedFormValues,
+} from "@/components/common_components/MountedFormField";
+import { antdRequired } from "@/components/common_components/antdFormRules";
+import { matchesPattern, selectControl, textControl } from "./mcpFieldRules";
+import { listControl } from "./mcpFormStore";
const { Text } = Typography;
@@ -9,6 +19,8 @@ const SCOPE_OPTIONS = [
{ value: "user", label: "Per-user" },
];
+const VARIABLE_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/;
+
/**
* Form section for admin-configured MCP environment variables.
*
@@ -20,6 +32,10 @@ const SCOPE_OPTIONS = [
* The parent form reads the ``env_vars`` field from the form values.
*/
const EnvVarsSection: React.FC = () => {
+ const { control } = useFormContext();
+ const { fields, append, remove } = useFieldArray({ control: listControl(control), name: "env_vars" });
+ useMountedName("env_vars");
+
return (
@@ -48,60 +64,52 @@ const EnvVarsSection: React.FC = () => {
-
- {(fields, { add, remove }) => (
-
- {fields.length > 0 && (
-
-
Variable Name
-
Value / Description
-
Scope
-
-
- )}
- {fields.map(({ key, name, ...restField }) => (
-
-
-
-
-
-
-
-
-
-
-
- remove(name)}
- className="text-gray-500 hover:text-red-500 cursor-pointer"
- />
-
-
- ))}
-
add({ scope: "global" })} icon={ } block>
- Add Variable
-
+
+ {fields.length > 0 && (
+
+
Variable Name
+
Value / Description
+
Scope
+
)}
-
+ {fields.map((item, index) => (
+
+
+ {(control) => (
+
+ )}
+
+
+
+
+
+ {(control) => (control)} options={SCOPE_OPTIONS} />}
+
+
+ remove(index)}
+ className="text-gray-500 hover:text-red-500 cursor-pointer"
+ />
+
+
+ ))}
+
append({ scope: "global" })} icon={ } block>
+ Add Variable
+
+
);
};
@@ -109,33 +117,33 @@ const EnvVarsSection: React.FC = () => {
// For instance-scoped vars this column holds the admin value. For per-user
// vars the value comes from each user later, so the column instead captures an
// optional description that the per-user fill-in modal shows as a hint.
-const ScopedValueOrDescription: React.FC<{
- name: number;
- restField: object;
-}> = ({ name, restField }) => {
- const isPerUser = Form.useWatch(["env_vars", name, "scope"]) === "user";
+const ScopedValueOrDescription: React.FC<{ index: number }> = ({ index }) => {
+ const isPerUser = useWatch({ name: `env_vars.${index}.scope` }) === "user";
if (isPerUser) {
return (
-
-
-
-
- Hint
-
-
- }
- placeholder="e.g. Your DB username"
- styles={{ input: { color: "#9ca3af" } }}
- />
-
+
+ {(control) => (
+
+
+
+ Hint
+
+
+ }
+ placeholder="e.g. Your DB username"
+ styles={{ input: { color: "#9ca3af" } }}
+ />
+ )}
+
);
}
return (
-
-
-
+
+ {(control) => }
+
);
};
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/IdJagFormFields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/IdJagFormFields.tsx
index e8730a5b974..9a6f4ab571f 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/IdJagFormFields.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/IdJagFormFields.tsx
@@ -1,7 +1,11 @@
import React from "react";
-import { Form, Input, Select, Tooltip } from "antd";
+import { Input, Select, Tooltip } from "antd";
import { InfoCircleOutlined } from "@ant-design/icons";
+import { MountedFormField } from "@/components/common_components/MountedFormField";
+import { antdRequired } from "@/components/common_components/antdFormRules";
+import { requiredUnlessSiblingSet, selectControl, textControl } from "./mcpFieldRules";
+
interface IdJagFormFieldsProps {
isEditing?: boolean;
}
@@ -17,12 +21,16 @@ const FieldLabel: React.FC<{ label: string; tooltip: string }> = ({ label, toolt
);
+const PRIVATE_KEY_PATH = ["credentials", "client_private_key"] as const;
+
const IdJagFormFields: React.FC = ({ isEditing = false }) => {
const placeholderSuffix = isEditing ? " (leave blank to keep existing)" : "";
+ const requiredWhenCreating = (message: string) =>
+ isEditing ? undefined : { validate: { required: antdRequired(message) } };
return (
<>
- = ({ isEditing = false })
/>
}
name="token_exchange_endpoint"
- rules={[{ required: !isEditing, message: "The org token endpoint is required for ID-JAG" }]}
+ required={!isEditing}
+ rules={requiredWhenCreating("The org token endpoint is required for ID-JAG")}
>
-
-
- (
+
+ )}
+
+ = ({ isEditing = false })
/>
}
name={["credentials", "id_jag_resource_token_endpoint"]}
- rules={[{ required: !isEditing, message: "The resource token endpoint is required for ID-JAG" }]}
+ required={!isEditing}
+ rules={requiredWhenCreating("The resource token endpoint is required for ID-JAG")}
>
-
-
- (
+
+ )}
+
+ }
name={["credentials", "client_id"]}
- rules={[{ required: !isEditing, message: "Client ID is required for ID-JAG" }]}
+ required={!isEditing}
+ rules={requiredWhenCreating("Client ID is required for ID-JAG")}
>
-
-
- (
+
+ )}
+
+ = ({ isEditing = false })
/>
}
name={["credentials", "client_secret"]}
- dependencies={[["credentials", "client_private_key"]]}
- rules={[
- ({ getFieldValue }) => ({
- validator: (_, value) => {
- if (isEditing || value || getFieldValue(["credentials", "client_private_key"])) {
- return Promise.resolve();
+ rules={
+ isEditing
+ ? undefined
+ : {
+ deps: ["credentials.client_private_key"],
+ validate: {
+ secretOrPrivateKey: requiredUnlessSiblingSet(
+ PRIVATE_KEY_PATH,
+ "Provide either a client secret or a client private key",
+ ),
+ },
}
- return Promise.reject(new Error("Provide either a client secret or a client private key"));
- },
- }),
- ]}
+ }
>
-
-
- (
+
+ )}
+
+
}
- name={["credentials", "client_private_key"]}
+ name={PRIVATE_KEY_PATH}
>
-
-
- (
+
+ )}
+
+ = ({ isEditing = false })
}
name={["credentials", "client_private_key_id"]}
>
-
-
- }
+
+ = ({ isEditing = false })
}
name={["credentials", "client_assertion_signing_alg"]}
>
-
-
- }
+
+ = ({ isEditing = false })
}
name="audience"
>
-
-
- (
+
+ )}
+
+ = ({ isEditing = false })
}
name={["credentials", "id_jag_resource"]}
>
-
-
- (
+
+ )}
+
+ = ({ isEditing = false })
}
name="subject_token_type"
>
-
-
- (
+
+ )}
+
+ }
name={["credentials", "scopes"]}
>
-
-
+ {(control) => (
+
+ )}
+
>
);
};
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.test.tsx
index ee6aee86a4d..7d36872cee8 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.test.tsx
@@ -1,10 +1,10 @@
import React from "react";
-import { render, screen } from "@testing-library/react";
+import { screen } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { describe, it, expect } from "vitest";
-import { Form } from "antd";
import MCPPermissionManagement from "./MCPPermissionManagement";
+import { renderInMcpForm } from "./McpFormTestHarness";
const defaultProps = {
availableAccessGroups: [],
@@ -12,6 +12,7 @@ const defaultProps = {
searchValue: "",
setSearchValue: () => {},
getAccessGroupOptions: () => [],
+ mountedAuthType: undefined,
};
describe("MCPPermissionManagement", () => {
@@ -24,22 +25,8 @@ describe("MCPPermissionManagement", () => {
return user;
};
- const renderWithForm = (props = {}) => {
- const Wrapper: React.FC = ({ children }) => {
- const [form] = Form.useForm();
- return (
-
- {children}
-
- );
- };
-
- return render(
-
-
- ,
- );
- };
+ const renderWithForm = (props = {}) =>
+ renderInMcpForm( , { allow_all_keys: false });
it("should default allow_all_keys switch to unchecked for new servers", async () => {
renderWithForm();
@@ -51,27 +38,15 @@ describe("MCPPermissionManagement", () => {
expect(toggle).not.toBeChecked();
});
- const renderWithInitialValues = (initialValues: Record, props = {}) => {
- const Wrapper: React.FC = ({ children }) => {
- const [form] = Form.useForm();
- return (
-
- {/* In the real app auth_type is registered by the parent form; the
- component only watches it. Register a hidden field here so
- Form.useWatch("auth_type") resolves the initial value. */}
-
-
-
- {children}
-
- );
- };
- return render(
-
-
- ,
+ const renderWithInitialValues = (initialValues: Record, props = {}) =>
+ renderInMcpForm(
+ ,
+ initialValues,
);
- };
it("shows only the oauth2 PKCE-delegation toggle for oauth2 servers", async () => {
renderWithInitialValues({ allow_all_keys: false, auth_type: "oauth2" });
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.tsx
index aae13d4b467..3711140562f 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.tsx
@@ -1,7 +1,17 @@
import React, { useEffect } from "react";
-import { Alert, Form, Select, Tooltip, Collapse, Input, Space, Button, Switch } from "antd";
+import { Alert, Select, Tooltip, Collapse, Input, Space, Button, Switch } from "antd";
import { InfoCircleOutlined, MinusCircleOutlined, PlusOutlined } from "@ant-design/icons";
+import { useFieldArray, useFormContext, useWatch } from "react-hook-form";
import { MCPServer, AUTH_TYPE } from "@/components/mcp_tools/types";
+import {
+ MountedFormField,
+ useMountedName,
+ type MountedFormValues,
+} from "@/components/common_components/MountedFormField";
+import { antdRequired } from "@/components/common_components/antdFormRules";
+import { Field, FieldLabel } from "@/components/shared/form/field";
+import { invertedSwitchControl, selectControl, switchControl, textControl } from "./mcpFieldRules";
+import { listControl } from "./mcpFormStore";
const { Panel } = Collapse;
interface MCPPermissionManagementProps {
@@ -13,20 +23,79 @@ interface MCPPermissionManagementProps {
value: string;
label: React.ReactNode;
}>;
+ /**
+ * The auth type as seen through the gate that mounts the auth_type field.
+ * Callers pass undefined whenever that field is unmounted, because both
+ * toggles below are mounted from this value and the payload only carries
+ * what is mounted.
+ */
+ mountedAuthType: string | null | undefined;
}
+const StaticHeadersFieldArray: React.FC = () => {
+ const { control } = useFormContext();
+ const { fields, append, remove } = useFieldArray({ control: listControl(control), name: "static_headers" });
+ useMountedName("static_headers");
+
+ return (
+
+ );
+};
+
const MCPPermissionManagement: React.FC = ({
availableAccessGroups,
mcpServer,
searchValue,
setSearchValue,
getAccessGroupOptions,
+ mountedAuthType,
}) => {
- const form = Form.useFormInstance();
- const watchedAuthType = Form.useWatch("auth_type", form);
- const isOAuth2 = watchedAuthType === AUTH_TYPE.OAUTH2;
- const isNoneAuth = watchedAuthType === AUTH_TYPE.NONE || watchedAuthType == null;
- const watchedExtraHeaders = Form.useWatch("extra_headers", form);
+ const { setValue } = useFormContext();
+ const isOAuth2 = mountedAuthType === AUTH_TYPE.OAUTH2;
+ const isNoneAuth = mountedAuthType === AUTH_TYPE.NONE || mountedAuthType == null;
+ const watchedExtraHeaders = useWatch({ name: "extra_headers" });
const hasAuthorizationHeader =
Array.isArray(watchedExtraHeaders) &&
watchedExtraHeaders.some((h) => typeof h === "string" && h.toLowerCase() === "authorization");
@@ -39,8 +108,8 @@ const MCPPermissionManagement: React.FC = ({
// Kept as separate flags so neither silently implies the other and existing
// oauth2 servers can't regress into pass-through behavior.
const canEnableOAuthPassthrough = isNoneAuth && hasAuthorizationHeader;
- const watchedDelegateAuth = Form.useWatch("delegate_auth_to_upstream", form);
- const watchedPublicInternet = Form.useWatch("available_on_public_internet", form);
+ const watchedDelegateAuth = useWatch({ name: "delegate_auth_to_upstream" });
+ const watchedPublicInternet = useWatch({ name: "available_on_public_internet" });
const showInternalDelegatePkceWarning = isOAuth2 && watchedDelegateAuth === true && watchedPublicInternet === false;
// Set initial values when mcpServer changes
@@ -51,10 +120,10 @@ const MCPPermissionManagement: React.FC = ({
header,
value: value != null ? String(value) : "",
}));
- form.setFieldValue("static_headers", staticHeaders);
+ setValue("static_headers", staticHeaders);
}
if (Array.isArray(mcpServer.env_vars) && mcpServer.env_vars.length > 0) {
- form.setFieldValue(
+ setValue(
"env_vars",
mcpServer.env_vars.map((entry) => ({
name: entry.name,
@@ -65,41 +134,41 @@ const MCPPermissionManagement: React.FC = ({
);
}
if (typeof mcpServer.allow_all_keys === "boolean") {
- form.setFieldValue("allow_all_keys", mcpServer.allow_all_keys);
+ setValue("allow_all_keys", mcpServer.allow_all_keys);
}
if (typeof mcpServer.available_on_public_internet === "boolean") {
- form.setFieldValue("available_on_public_internet", mcpServer.available_on_public_internet);
+ setValue("available_on_public_internet", mcpServer.available_on_public_internet);
}
if (typeof mcpServer.delegate_auth_to_upstream === "boolean") {
- form.setFieldValue("delegate_auth_to_upstream", mcpServer.delegate_auth_to_upstream);
+ setValue("delegate_auth_to_upstream", mcpServer.delegate_auth_to_upstream);
}
if (typeof mcpServer.oauth_passthrough === "boolean") {
- form.setFieldValue("oauth_passthrough", mcpServer.oauth_passthrough);
+ setValue("oauth_passthrough", mcpServer.oauth_passthrough);
}
} else {
- form.setFieldValue("allow_all_keys", false);
- form.setFieldValue("available_on_public_internet", true);
- form.setFieldValue("delegate_auth_to_upstream", false);
- form.setFieldValue("oauth_passthrough", false);
+ setValue("allow_all_keys", false);
+ setValue("available_on_public_internet", true);
+ setValue("delegate_auth_to_upstream", false);
+ setValue("oauth_passthrough", false);
}
- }, [mcpServer, form]);
+ }, [mcpServer, setValue]);
// delegate_auth_to_upstream is only honored server-side for oauth2 servers.
// Force it back to false whenever the user switches away from oauth2 so a
// stale toggle value doesn't get persisted unexpectedly.
useEffect(() => {
if (!isOAuth2) {
- form.setFieldValue("delegate_auth_to_upstream", false);
+ setValue("delegate_auth_to_upstream", false);
}
- }, [isOAuth2, form]);
+ }, [isOAuth2, setValue]);
// oauth_passthrough is only honored for auth_type=none servers that forward
// Authorization upstream. Force it back to false otherwise.
useEffect(() => {
if (!canEnableOAuthPassthrough) {
- form.setFieldValue("oauth_passthrough", false);
+ setValue("oauth_passthrough", false);
}
- }, [canEnableOAuthPassthrough, form]);
+ }, [canEnableOAuthPassthrough, setValue]);
return (
@@ -130,14 +199,9 @@ const MCPPermissionManagement: React.FC = ({
Enable if this server should be "public" to all keys.
-
-
-
+
+ {(control) => }
+
@@ -152,16 +216,9 @@ const MCPPermissionManagement: React.FC = ({
Turn on to restrict access to callers within your internal network only.
- ({ checked: !value })}
- getValueFromEvent={(checked: boolean) => !checked}
- initialValue={true}
- className="mb-0"
- >
-
-
+
+ {(control) => }
+
{isOAuth2 && (
@@ -177,14 +234,13 @@ const MCPPermissionManagement: React.FC = ({
Bypass LiteLLM auth so clients authenticate directly with the upstream OAuth MCP server.
-
-
-
+ {(control) => }
+
)}
@@ -202,14 +258,13 @@ const MCPPermissionManagement: React.FC = ({
upstream MCP server.
-
-
-
+ {(control) => }
+
)}
@@ -223,7 +278,7 @@ const MCPPermissionManagement: React.FC = ({
/>
)}
-
MCP Access Groups
@@ -235,21 +290,24 @@ const MCPPermissionManagement: React.FC = ({
name="mcp_access_groups"
className="mb-4"
>
- (option?.value ?? "").toLowerCase().includes(input.toLowerCase())}
- onSearch={(value) => setSearchValue(value)}
- tokenSeparators={[","]}
- options={getAccessGroupOptions()}
- maxTagCount="responsive"
- allowClear
- />
-
+ {(control) => (
+ (option?.value ?? "").toLowerCase().includes(input.toLowerCase())}
+ onSearch={(value) => setSearchValue(value)}
+ tokenSeparators={[","]}
+ options={getAccessGroupOptions()}
+ maxTagCount="responsive"
+ allowClear
+ />
+ )}
+
-
Extra Headers
@@ -265,70 +323,34 @@ const MCPPermissionManagement: React.FC = ({
}
name="extra_headers"
>
- 0
- ? `Currently: ${mcpServer.extra_headers.join(", ")}`
- : "Enter header names (e.g., Authorization, X-Custom-Header)"
- }
- className="rounded-lg"
- size="large"
- tokenSeparators={[","]}
- allowClear
- />
-
+ {(control) => (
+ 0
+ ? `Currently: ${mcpServer.extra_headers.join(", ")}`
+ : "Enter header names (e.g., Authorization, X-Custom-Header)"
+ }
+ className="rounded-lg"
+ size="large"
+ tokenSeparators={[","]}
+ allowClear
+ />
+ )}
+
-
+
Static Headers
- }
- required={false}
- >
-
- {(fields, { add, remove }) => (
-
- )}
-
-
+
+
+
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/McpFormTestHarness.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/McpFormTestHarness.tsx
new file mode 100644
index 00000000000..0529c9c7c61
--- /dev/null
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/McpFormTestHarness.tsx
@@ -0,0 +1,37 @@
+import * as React from "react";
+import { render, type RenderResult } from "@testing-library/react";
+import { FormProvider, useForm } from "react-hook-form";
+
+import {
+ MountedFormProvider,
+ projectMountedValues,
+ useMountRegistry,
+ type MountedFormValues,
+} from "@/components/common_components/MountedFormField";
+
+export const McpFormHarness: React.FC<{
+ defaultValues?: MountedFormValues;
+ onFinish?: (values: MountedFormValues) => void;
+ children: React.ReactNode;
+}> = ({ defaultValues, onFinish, children }) => {
+ const form = useForm({ mode: "onChange", defaultValues });
+ const registry = useMountRegistry();
+ return (
+
+
+ {
+ event.preventDefault();
+ onFinish?.(projectMountedValues(registry, form.getValues));
+ }}
+ >
+ {children}
+ Submit
+
+
+
+ );
+};
+
+export const renderInMcpForm = (ui: React.ReactNode, defaultValues: MountedFormValues = {}): RenderResult =>
+ render({ui} );
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.test.tsx
index 48964490339..21bb2801e8c 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.test.tsx
@@ -1,24 +1,8 @@
import React from "react";
import { describe, it, expect, vi, beforeEach } from "vitest";
import { render, screen, waitFor, act, fireEvent } from "@testing-library/react";
-import { Form } from "antd";
import OAuthFormFields from "./OAuthFormFields";
-
-// ── helpers ──────────────────────────────────────────────────────────────────
-
-/** Minimal Ant Form wrapper so Form.Item registers correctly. */
-const WithForm: React.FC<{ children: React.ReactNode; onFinish?: (values: any) => void }> = ({
- children,
- onFinish,
-}) => {
- const [form] = Form.useForm();
- return (
-
- {children}
- Submit
-
- );
-};
+import { McpFormHarness as WithForm } from "./McpFormTestHarness";
// ── tests ─────────────────────────────────────────────────────────────────────
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.tsx
index cbe6ac18d22..545efb910c4 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.tsx
@@ -1,10 +1,13 @@
import React from "react";
-import { Form, Input as AntdInput, InputNumber, Select, Tooltip } from "antd";
+import { Input as AntdInput, InputNumber, Select, Tooltip } from "antd";
import { InfoCircleOutlined } from "@ant-design/icons";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { OAUTH_FLOW } from "@/components/mcp_tools/types";
+import { MountedFormField } from "@/components/common_components/MountedFormField";
+import { antdRequired } from "@/components/common_components/antdFormRules";
import TokenEndpointAuthMethodField from "./TokenEndpointAuthMethodField";
+import { numberControl, parsesAsJson, selectControl, textControl } from "./mcpFieldRules";
interface OAuthFlowStatus {
startOAuthFlow: () => void;
@@ -41,12 +44,14 @@ const FieldLabel: React.FC<{ label: string; tooltip: string }> = ({ label, toolt
);
const UpstreamResourceField: React.FC = () => (
- }
name={["credentials", "upstream_resource"]}
>
-
-
+ {(control) => (
+
+ )}
+
);
const OAuthFormFields: React.FC = ({
@@ -57,11 +62,12 @@ const OAuthFormFields: React.FC = ({
docsUrl,
}) => {
const placeholderSuffix = isEditing ? " (leave blank to keep existing)" : "";
- const requiredWhenCreating = (message: string) => (isEditing ? [] : [{ required: true, message }]);
+ const requiredWhenCreating = (message: string) =>
+ isEditing ? undefined : { validate: { required: antdRequired(message) } };
return (
<>
- = ({
/>
}
name="oauth_flow_type"
- {...(initialFlowType ? { initialValue: initialFlowType } : {})}
+ {...(initialFlowType ? { defaultValue: initialFlowType } : {})}
>
-
-
-
- Machine-to-Machine (M2M)
- server-to-server, no user interaction
-
-
-
-
- Interactive (PKCE)
- browser-based user authorization
-
-
-
-
+ {(control) => (
+
+
+
+ Machine-to-Machine (M2M)
+ server-to-server, no user interaction
+
+
+
+
+ Interactive (PKCE)
+ browser-based user authorization
+
+
+
+ )}
+
{isM2M ? (
<>
- }
name={["credentials", "client_id"]}
+ required={!isEditing}
rules={requiredWhenCreating("Client ID is required for M2M OAuth")}
>
-
-
- (
+
+ )}
+
+
}
name={["credentials", "client_secret"]}
+ required={!isEditing}
rules={requiredWhenCreating("Client Secret is required for M2M OAuth")}
>
-
-
- (
+
+ )}
+
+ }
name="token_url"
+ required={!isEditing}
rules={requiredWhenCreating("Token URL is required for M2M OAuth")}
>
-
-
+ {(control) => (
+
+ )}
+
- = ({
}
name={["credentials", "scopes"]}
>
-
-
+ {(control) => (
+
+ )}
+
>
) : (
<>
-
= ({
}
name={["credentials", "client_id"]}
>
-
-
- (
+
+ )}
+
+ = ({
}
name={["credentials", "client_secret"]}
>
-
-
- (
+
+ )}
+
+ = ({
}
name={["credentials", "scopes"]}
>
-
-
+ {(control) => (
+
+ )}
+
- = ({
}
name="issuer"
>
-
-
- (
+
+ )}
+
+ = ({
}
name="authorization_url"
>
-
-
- (
+
+ )}
+
+ }
name="token_url"
>
-
-
+ {(control) => (
+
+ )}
+
- = ({
}
name="registration_url"
>
-
-
- (
+
+ )}
+
+ = ({
/>
}
name="token_validation_json"
- rules={[
- {
- validator: (_: any, value: string) => {
- if (!value || value.trim() === "") return Promise.resolve();
- try {
- JSON.parse(value);
- return Promise.resolve();
- } catch {
- return Promise.reject(new Error("Must be valid JSON"));
- }
- },
- },
- ]}
+ rules={{ validate: { json: parsesAsJson("Must be valid JSON") } }}
>
-
-
- (
+
+ )}
+
+ = ({
}
name="token_storage_ttl_seconds"
>
-
-
+ {(control) => (
+
+ )}
+
{oauthFlow && (
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenAPIFormSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenAPIFormSection.tsx
index 073780b359f..78c8bbe73a9 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenAPIFormSection.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenAPIFormSection.tsx
@@ -1,12 +1,15 @@
import React, { useState } from "react";
-import { Form, Input, Tooltip } from "antd";
+import { Input, Tooltip } from "antd";
import { InfoCircleOutlined } from "@ant-design/icons";
-import { FormInstance } from "antd/es/form";
import { AUTH_TYPE, OAUTH_FLOW } from "@/components/mcp_tools/types";
+import { MountedFormField } from "@/components/common_components/MountedFormField";
+import { antdRequired } from "@/components/common_components/antdFormRules";
import OpenAPIQuickPicker, { OpenAPIRegistryEntry, OpenAPIKeyTool } from "./OpenAPIQuickPicker";
+import { McpForm, resetFields, setFieldsValue } from "./mcpFormStore";
+import { textControl } from "./mcpFieldRules";
interface OpenAPIFormSectionProps {
- form: FormInstance;
+ form: McpForm;
accessToken: string | null;
/** Called when a preset is selected so the parent can sync its formValues state. */
onValuesChange: (updates: Record) => void;
@@ -47,13 +50,11 @@ const OpenAPIFormSection: React.FC = ({
updates.oauth_flow_type = OAUTH_FLOW.INTERACTIVE;
updates.authorization_url = entry.oauth.authorization_url;
updates.token_url = entry.oauth.token_url;
- form.setFieldsValue(updates);
+ setFieldsValue(form, updates);
onOAuthDocsUrlChange?.(entry.oauth.docs_url ?? null);
} else {
- // resetFields is required to visually clear Ant Design form fields —
- // setFieldsValue with undefined silently skips undefined keys.
- form.resetFields(["auth_type", "authorization_url", "token_url"]);
- form.setFieldsValue(updates);
+ resetFields(form, ["auth_type", "authorization_url", "token_url"]);
+ setFieldsValue(form, updates);
onOAuthDocsUrlChange?.(null);
}
onValuesChange(updates);
@@ -63,7 +64,7 @@ const OpenAPIFormSection: React.FC = ({
<>
-
OpenAPI Spec URL
@@ -73,20 +74,25 @@ const OpenAPIFormSection: React.FC = ({
}
name="spec_path"
- rules={[{ required: true, message: "Please enter an OpenAPI spec URL" }]}
+ required
+ rules={{ validate: { required: antdRequired("Please enter an OpenAPI spec URL") } }}
>
- {
- // Clear the preset selection when the user manually edits the spec URL
- // so stale suggested tools from a previous preset don't persist.
- setSelectedPreset(null);
- onKeyToolsChange?.([]);
- onOAuthDocsUrlChange?.(null);
- }}
- />
-
+ {(control) => (
+ {
+ control.onChange(event);
+ // Clear the preset selection when the user manually edits the spec URL
+ // so stale suggested tools from a previous preset don't persist.
+ setSelectedPreset(null);
+ onKeyToolsChange?.([]);
+ onOAuthDocsUrlChange?.(null);
+ }}
+ />
+ )}
+
>
);
};
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenApiByokFields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenApiByokFields.tsx
index 2ac4279e20a..83c841439f0 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenApiByokFields.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenApiByokFields.tsx
@@ -1,91 +1,101 @@
import React from "react";
-import { Form, Input, Select, Switch, Tooltip } from "antd";
+import { Input, Select, Switch, Tooltip } from "antd";
import { InfoCircleOutlined } from "@ant-design/icons";
+import { useWatch } from "react-hook-form";
-const OpenApiByokFields: React.FC = () => (
- <>
-
- BYOK (Bring Your Own Key)
-
-
-
-
- }
- name="is_byok"
- valuePropName="checked"
- >
-
-
+import { MountedFormField } from "@/components/common_components/MountedFormField";
+import { selectControl, switchControl, textControl } from "./mcpFieldRules";
- prev.is_byok !== cur.is_byok || prev.auth_type !== cur.auth_type}>
- {({ getFieldValue }) =>
- getFieldValue("is_byok") ? (
- <>
- {/* Auth format hint */}
- {getFieldValue("auth_type") && getFieldValue("auth_type") !== "none" && (
-
-
-
- User keys will be sent as:{" "}
-
- {getFieldValue("auth_type") === "bearer_token" && "Authorization: Bearer {key}"}
- {getFieldValue("auth_type") === "token" && "Authorization: token {key}"}
- {getFieldValue("auth_type") === "api_key" && "x-api-key: {key}"}
- {getFieldValue("auth_type") === "basic" && "Authorization: Basic {key}"}
- {getFieldValue("auth_type") === "authorization" && "Authorization: {key}"}
-
- {!getFieldValue("auth_type") && "Set Authentication Type below to specify the format."}
-
-
- )}
- {!getFieldValue("auth_type") && (
-
-
-
- Set the Authentication Type below to specify how user keys are sent (e.g., Bearer
- Token, API Key header).
-
-
- )}
-
- Access Description
-
-
-
-
- }
- name="byok_description"
- >
+const AUTH_HEADER_FORMATS: Readonly> = {
+ bearer_token: "Authorization: Bearer {key}",
+ token: "Authorization: token {key}",
+ api_key: "x-api-key: {key}",
+ basic: "Authorization: Basic {key}",
+ authorization: "Authorization: {key}",
+};
+
+const OpenApiByokFields: React.FC = () => {
+ const isByok = Boolean(useWatch({ name: "is_byok" }));
+ const authType = useWatch({ name: "auth_type" }) as string | undefined;
+ const hasAuthType = Boolean(authType) && authType !== "none";
+
+ return (
+ <>
+
+ BYOK (Bring Your Own Key)
+
+
+
+
+ }
+ name="is_byok"
+ >
+ {(control) => }
+
+
+ {isByok && (
+ <>
+ {hasAuthType && (
+
+
+
+ User keys will be sent as:{" "}
+
+ {authType === undefined ? "" : AUTH_HEADER_FORMATS[authType]}
+
+
+
+ )}
+ {!authType && (
+
+
+
+ Set the Authentication Type below to specify how user keys are sent (e.g., Bearer
+ Token, API Key header).
+
+
+ )}
+
+ Access Description
+
+
+
+
+ }
+ name="byok_description"
+ >
+ {(control) => (
-
+ )}
+
-
- API Key Help URL
-
-
-
-
- }
- name="byok_api_key_help_url"
- >
-
-
- >
- ) : null
- }
-
- >
-);
+
+ API Key Help URL
+
+
+
+
+ }
+ name="byok_api_key_help_url"
+ >
+ {(control) => }
+
+ >
+ )}
+ >
+ );
+};
export default OpenApiByokFields;
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/PassthroughAuthorizeSection.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/PassthroughAuthorizeSection.test.tsx
index 0a09ef3f856..a7577ccb781 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/PassthroughAuthorizeSection.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/PassthroughAuthorizeSection.test.tsx
@@ -1,13 +1,10 @@
import React from "react";
import { describe, it, expect } from "vitest";
import { render, screen } from "@testing-library/react";
-import { Form } from "antd";
import PassthroughAuthorizeSection from "./PassthroughAuthorizeSection";
+import { McpFormHarness } from "./McpFormTestHarness";
-const WithForm: React.FC<{ children: React.ReactNode }> = ({ children }) => {
- const [form] = Form.useForm();
- return {children} ;
-};
+const WithForm = McpFormHarness;
const noopFlow = { startOAuthFlow: () => {}, status: "idle", error: null, tokenResponse: null };
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/PassthroughAuthorizeSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/PassthroughAuthorizeSection.tsx
index dc10f0f1392..375e1dfe515 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/PassthroughAuthorizeSection.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/PassthroughAuthorizeSection.tsx
@@ -1,6 +1,8 @@
import React from "react";
-import { Button, Checkbox, Form, Input } from "antd";
+import { Button, Checkbox, Input } from "antd";
import DcrBridgeToggle from "./DcrBridgeToggle";
+import { MountedFormField } from "@/components/common_components/MountedFormField";
+import { textControl } from "./mcpFieldRules";
import { credentialAuthClass, isClientForwardedTokenMode } from "@/components/mcp_tools/types";
interface PassthroughOAuthFlow {
@@ -81,27 +83,33 @@ export default function PassthroughAuthorizeSection({
and may not be valid. Update the client ID, or clear it to use dynamic client registration.
)}
-
OAuth Client ID (optional)}
name={["credentials", "client_id"]}
- extra={clientIdExtra}
+ help={clientIdExtra}
>
-
-
-
(
+
+ )}
+
+ OAuth Client Secret (optional)}
name={["credentials", "client_secret"]}
>
-
-
+ {(control) => (
+
+ )}
+
{isEditing && onRemoveStoredAppChange && (
onRemoveStoredAppChange(e.target.checked)}>
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/StdioConfiguration.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/StdioConfiguration.tsx
index 476a5b61683..bb410aa19dc 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/StdioConfiguration.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/StdioConfiguration.tsx
@@ -1,7 +1,11 @@
import React from "react";
-import { Form, Input, Tooltip } from "antd";
+import { Input, Tooltip } from "antd";
import { InfoCircleOutlined } from "@ant-design/icons";
+import { MountedFormField } from "@/components/common_components/MountedFormField";
+import { antdRequired } from "@/components/common_components/antdFormRules";
+import { parsesAsJson, textControl } from "./mcpFieldRules";
+
interface StdioConfigurationProps {
isVisible: boolean;
/**
@@ -11,37 +15,7 @@ interface StdioConfigurationProps {
required?: boolean;
}
-const StdioConfiguration: React.FC = ({ isVisible, required = true }) => {
- if (!isVisible) return null;
-
- return (
-
- Stdio Configuration (JSON)
-
-
-
-
- }
- name="stdio_config"
- rules={[
- ...(required ? [{ required: true, message: "Please enter stdio configuration" }] : []),
- {
- validator: (_, value) => {
- if (!value) return Promise.resolve();
- try {
- JSON.parse(value);
- return Promise.resolve();
- } catch {
- return Promise.reject("Please enter valid JSON");
- }
- },
- },
- ]}
- >
- = ({ isVisible, requ
}
}
}
-}`}
- rows={12}
- className="rounded-lg border-gray-300 focus:border-blue-500 focus:ring-blue-500 font-mono text-sm"
- />
-
+}`;
+
+const StdioConfiguration: React.FC = ({ isVisible, required = true }) => {
+ if (!isVisible) return null;
+
+ return (
+
+ Stdio Configuration (JSON)
+
+
+
+
+ }
+ name="stdio_config"
+ required={required}
+ rules={{
+ validate: {
+ ...(required ? { required: antdRequired("Please enter stdio configuration") } : {}),
+ json: parsesAsJson("Please enter valid JSON"),
+ },
+ }}
+ >
+ {(control) => (
+
+ )}
+
);
};
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenEndpointAuthMethodField.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenEndpointAuthMethodField.tsx
index c97ce96bfb1..38fa3573079 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenEndpointAuthMethodField.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenEndpointAuthMethodField.tsx
@@ -1,7 +1,10 @@
import React from "react";
-import { Form, Select, Tooltip } from "antd";
+import { Select, Tooltip } from "antd";
import { InfoCircleOutlined } from "@ant-design/icons";
+import { MountedFormField } from "@/components/common_components/MountedFormField";
+import { selectControl } from "./mcpFieldRules";
+
const TOKEN_ENDPOINT_AUTH_METHOD_OPTIONS = [
{ value: "client_secret_basic", label: "Client Secret Basic" },
{ value: "client_secret_post", label: "Client Secret Post" },
@@ -12,7 +15,7 @@ interface TokenEndpointAuthMethodFieldProps {
}
const TokenEndpointAuthMethodField: React.FC = ({ isEditing = false }) => (
-
Token Endpoint Auth Method (optional)
@@ -23,16 +26,19 @@ const TokenEndpointAuthMethodField: React.FC
}
name={["credentials", "token_endpoint_auth_method"]}
>
-
-
+ {(control) => (
+
+ )}
+
);
export default TokenEndpointAuthMethodField;
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenExchangeFormFields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenExchangeFormFields.tsx
index 9e1e1a85743..ba213b20655 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenExchangeFormFields.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenExchangeFormFields.tsx
@@ -1,6 +1,11 @@
import React from "react";
-import { Form, Input, Select, Tooltip } from "antd";
+import { Input, Select, Tooltip } from "antd";
import { InfoCircleOutlined } from "@ant-design/icons";
+import { useWatch } from "react-hook-form";
+
+import { MountedFormField } from "@/components/common_components/MountedFormField";
+import { antdRequired } from "@/components/common_components/antdFormRules";
+import { selectControl, textControl } from "./mcpFieldRules";
interface TokenExchangeFormFieldsProps {
isEditing?: boolean;
@@ -19,10 +24,13 @@ const FieldLabel: React.FC<{ label: string; tooltip: string }> = ({ label, toolt
const TokenExchangeFormFields: React.FC = ({ isEditing = false }) => {
const placeholderSuffix = isEditing ? " (leave blank to keep existing)" : "";
+ const isEntraObo = useWatch({ name: "token_exchange_profile" }) === "entra_obo";
+ const requiredWhenCreating = (message: string) =>
+ isEditing ? undefined : { validate: { required: antdRequired(message) } };
return (
<>
- = ({ isEdi
/>
}
name="token_exchange_profile"
- {...(isEditing ? {} : { initialValue: "rfc8693" })}
+ {...(isEditing ? {} : { defaultValue: "rfc8693" })}
>
-
-
- RFC 8693 (standard)
-
-
- Microsoft Entra OBO
-
-
-
- (
+
+
+ RFC 8693 (standard)
+
+
+ Microsoft Entra OBO
+
+
+ )}
+
+ = ({ isEdi
}
name="token_exchange_endpoint"
>
-
-
- (
+
+ )}
+
+ = ({ isEdi
/>
}
name={["credentials", "client_id"]}
- rules={[{ required: !isEditing, message: "Client ID is required for token exchange" }]}
+ required={!isEditing}
+ rules={requiredWhenCreating("Client ID is required for token exchange")}
>
-
-
- (
+
+ )}
+
+ = ({ isEdi
/>
}
name={["credentials", "client_secret"]}
- rules={[{ required: !isEditing, message: "Client Secret is required for token exchange" }]}
+ required={!isEditing}
+ rules={requiredWhenCreating("Client Secret is required for token exchange")}
>
-
-
- prev.token_exchange_profile !== cur.token_exchange_profile}>
- {({ getFieldValue }) => {
- const isEntraObo = getFieldValue("token_exchange_profile") === "entra_obo";
- return (
- <>
- {!isEntraObo && (
- <>
-
- }
- name="audience"
- >
-
-
-
- }
- name="subject_token_type"
- >
-
-
- >
- )}
- /.default)."
- : "Optional scopes to request during the token exchange."
- }
- />
- }
- name={["credentials", "scopes"]}
- rules={
- isEntraObo
- ? [
- {
- required: true,
- message: "Microsoft Entra OBO requires a scope, e.g. api:///.default",
- },
- ]
- : []
- }
- >
- /.default" : "Add scopes"}
- className="rounded-lg"
- size="large"
- />
-
- >
- );
- }}
-
+ {(control) => (
+
+ )}
+
+ {!isEntraObo && (
+ <>
+
+ }
+ name="audience"
+ >
+ {(control) => (
+
+ )}
+
+
+ }
+ name="subject_token_type"
+ >
+ {(control) => (
+
+ )}
+
+ >
+ )}
+ /.default)."
+ : "Optional scopes to request during the token exchange."
+ }
+ />
+ }
+ name={["credentials", "scopes"]}
+ required={isEntraObo}
+ rules={
+ isEntraObo
+ ? {
+ validate: {
+ required: antdRequired("Microsoft Entra OBO requires a scope, e.g. api:///.default"),
+ },
+ }
+ : undefined
+ }
+ >
+ {(control) => (
+ /.default" : "Add scopes"}
+ className="rounded-lg"
+ size="large"
+ />
+ )}
+
>
);
};
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editServerPayload.differential.cases.ts b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editServerPayload.differential.cases.ts
new file mode 100644
index 00000000000..c350ac085b6
--- /dev/null
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editServerPayload.differential.cases.ts
@@ -0,0 +1,391 @@
+import { MCPServer } from "@/components/mcp_tools/types";
+import type { EditServerUiState } from "./editServerPayload";
+
+const SERVER: MCPServer = {
+ server_id: "srv_1",
+ server_name: "srv",
+ alias: "srv_alias",
+ description: "a server",
+ transport: "http",
+ url: "https://example.com/mcp",
+ auth_type: "none",
+ created_at: "2024-01-01T00:00:00Z",
+ created_by: "user-1",
+ updated_at: "2024-01-01T00:00:00Z",
+ updated_by: "user-1",
+ mcp_access_groups: [],
+};
+
+export const baseUi: EditServerUiState = {
+ mcpServer: SERVER,
+ logoUrl: undefined,
+ costConfig: {},
+ allowedTools: [],
+ hasExistingToolAllowlist: false,
+ hasToolAllowlistInteraction: false,
+ toolNameToDisplayName: {},
+ toolNameToDescription: {},
+ removeStoredApp: false,
+};
+
+export interface DifferentialCase {
+ readonly label: string;
+ readonly values: Record;
+ readonly ui: Partial;
+}
+
+// Always-mounted root fields. A "collapsed" section is modelled as its keys being
+// ABSENT, which is what antd's mounted-only onFinish produces, and an expanded but
+// untouched one as the key present holding `undefined`.
+const ROOT = {
+ server_name: "srv",
+ alias: "srv_alias",
+ description: "a server",
+ transport: "http",
+ url: "https://example.com/mcp",
+ auth_type: "none",
+ max_concurrent_requests: undefined,
+ mcp_access_groups: [],
+ extra_headers: [],
+ static_headers: [],
+ env_vars: [],
+ allow_all_keys: false,
+ available_on_public_internet: true,
+};
+
+const CREDS = {
+ auth_value: "secret-value",
+ client_id: "cid",
+ client_secret: "csec",
+ scopes: ["read", "write"],
+ aws_region_name: "us-east-1",
+ aws_service_name: "bedrock",
+ aws_access_key_id: "AKIA",
+ aws_secret_access_key: "shh",
+ aws_session_token: "tok",
+ aws_role_name: "role",
+ aws_session_name: "sess",
+ id_jag_resource: "api://res",
+ id_jag_resource_token_endpoint: "https://idp/jag",
+ client_private_key: "-----KEY-----",
+ client_private_key_id: "kid",
+ client_assertion_signing_alg: "RS256",
+ token_endpoint_auth_method: "client_secret_basic",
+ upstream_resource: "api://up",
+};
+
+const AUTH_TYPES = [
+ "none",
+ "api_key",
+ "bearer_token",
+ "token",
+ "basic",
+ "oauth2",
+ "oauth2_token_exchange",
+ "oauth2_id_jag",
+ "aws_sigv4",
+ "true_passthrough",
+ "oauth_delegate",
+] as const;
+
+export const CASES: readonly DifferentialCase[] = [
+ // --- mount-state axis: everything collapsed / expanded-undefined / expanded-valued ---
+ { label: "everything collapsed, only the always-mounted root", values: { ...ROOT }, ui: {} },
+ {
+ label: "everything expanded, all optional keys present as undefined",
+ values: {
+ ...ROOT,
+ credentials: {},
+ stdio_config: undefined,
+ env_json: undefined,
+ command: undefined,
+ args: undefined,
+ delegate_auth_to_upstream: undefined,
+ oauth_passthrough: undefined,
+ dcr_bridge: undefined,
+ token_validation_json: undefined,
+ spec_path: undefined,
+ oauth_flow_type: undefined,
+ issuer: undefined,
+ authorization_url: undefined,
+ token_url: undefined,
+ registration_url: undefined,
+ token_storage_ttl_seconds: undefined,
+ token_exchange_endpoint: undefined,
+ token_exchange_profile: undefined,
+ audience: undefined,
+ subject_token_type: undefined,
+ disallowed_tools: undefined,
+ },
+ ui: {},
+ },
+ {
+ label: "expand then collapse: the key is absent again even though a value was typed",
+ values: { ...ROOT },
+ ui: {},
+ },
+
+ // --- transport axis ---
+ {
+ label: "stdio via the JSON config path",
+ values: {
+ ...ROOT,
+ transport: "stdio",
+ url: undefined,
+ stdio_config: JSON.stringify({ command: "npx", args: ["-y", "pkg"], env: { A: "1" } }),
+ },
+ ui: {},
+ },
+ {
+ label: "stdio via a wrapped mcpServers config",
+ values: {
+ ...ROOT,
+ transport: "stdio",
+ url: undefined,
+ stdio_config: JSON.stringify({ mcpServers: { first: { command: "uvx", args: [1, " ", "b"] } } }),
+ },
+ ui: {},
+ },
+ {
+ label: "stdio via the dedicated command/args/env fields",
+ values: {
+ ...ROOT,
+ transport: "stdio",
+ url: undefined,
+ command: " npx ",
+ args: ["-y", " ", "pkg"],
+ env_json: JSON.stringify({ A: 1, "": "skipped", B: null }),
+ },
+ ui: {},
+ },
+ {
+ label: "openapi swaps url for spec_path",
+ values: { ...ROOT, transport: "openapi", url: undefined, spec_path: "https://x/openapi.json" },
+ ui: {},
+ },
+
+ // --- auth-type axis, one case per type, credentials mounted ---
+ ...AUTH_TYPES.map((auth) => ({
+ label: `auth_type ${auth} with the full credential bag mounted`,
+ values: { ...ROOT, auth_type: auth, credentials: { ...CREDS } },
+ ui: {},
+ })),
+
+ // --- oauth2 flow sub-branches ---
+ {
+ label: "oauth2 m2m flow",
+ values: {
+ ...ROOT,
+ auth_type: "oauth2",
+ oauth_flow_type: "m2m",
+ token_url: "https://idp/token",
+ credentials: { ...CREDS },
+ },
+ ui: {},
+ },
+ {
+ label: "oauth2 interactive flow",
+ values: {
+ ...ROOT,
+ auth_type: "oauth2",
+ oauth_flow_type: "interactive",
+ issuer: "https://idp",
+ credentials: { ...CREDS },
+ },
+ ui: {},
+ },
+ {
+ label: "oauth2 with delegate_auth_to_upstream on",
+ values: { ...ROOT, auth_type: "oauth2", delegate_auth_to_upstream: true, credentials: { ...CREDS } },
+ ui: {},
+ },
+
+ // --- the five coalesced keys: unbound raw against each server-side fallback ---
+ {
+ label: "coalesced booleans unbound, server has them all true",
+ values: { ...ROOT, auth_type: "oauth2", allow_all_keys: undefined, available_on_public_internet: undefined },
+ ui: {
+ mcpServer: {
+ ...SERVER,
+ auth_type: "oauth2",
+ allow_all_keys: true,
+ available_on_public_internet: true,
+ delegate_auth_to_upstream: true,
+ oauth_passthrough: true,
+ dcr_bridge: true,
+ },
+ },
+ },
+ {
+ label: "coalesced booleans unbound, server has them all false",
+ values: { ...ROOT, allow_all_keys: undefined, available_on_public_internet: undefined },
+ ui: { mcpServer: { ...SERVER, allow_all_keys: false, available_on_public_internet: false } },
+ },
+ {
+ label: "dcr_bridge bound true on a client-forwarded mode",
+ values: { ...ROOT, auth_type: "true_passthrough", dcr_bridge: true },
+ ui: { mcpServer: { ...SERVER, auth_type: "true_passthrough" } },
+ },
+ {
+ label: "dcr_bridge bound true but auth switched away, forced false",
+ values: { ...ROOT, auth_type: "api_key", dcr_bridge: true },
+ ui: {},
+ },
+ {
+ label: "oauth_passthrough with an Authorization extra header",
+ values: { ...ROOT, auth_type: "none", extra_headers: ["Authorization"], oauth_passthrough: true },
+ ui: {},
+ },
+ {
+ label: "oauth_passthrough without the Authorization header, forced false",
+ values: { ...ROOT, auth_type: "none", extra_headers: ["X-Other"], oauth_passthrough: true },
+ ui: {},
+ },
+
+ // --- auth-type transitions that null out the previous subtree ---
+ {
+ label: "was oauth2, now api_key: nulls the four oauth endpoints",
+ values: { ...ROOT, auth_type: "api_key", credentials: { auth_value: "v" } },
+ ui: { mcpServer: { ...SERVER, auth_type: "oauth2" } },
+ },
+ {
+ label: "was token_exchange, now none: nulls the four exchange fields",
+ values: { ...ROOT, auth_type: "none" },
+ ui: { mcpServer: { ...SERVER, auth_type: "oauth2_token_exchange" } },
+ },
+
+ // --- credentials filtering ---
+ // ADMIN_CONFIG_CREDENTIAL_KEYS is exactly ["upstream_resource"], so only that key
+ // takes the blank-to-explicit-null branch. A blank client_id is dropped instead.
+ {
+ label: "blank upstream_resource becomes an explicit null",
+ values: { ...ROOT, auth_type: "oauth2", credentials: { upstream_resource: "", client_secret: "keep" } },
+ ui: {},
+ },
+ {
+ label: "blank non-admin credential is dropped, not nulled",
+ values: { ...ROOT, auth_type: "oauth2", credentials: { client_id: "", client_secret: "keep", scopes: [] } },
+ ui: {},
+ },
+ {
+ label: "scopes array filters empties and drops when nothing survives",
+ values: { ...ROOT, auth_type: "oauth2", credentials: { scopes: ["", null, "read"] } },
+ ui: {},
+ },
+ {
+ label: "scopes entirely empty drops the key",
+ values: { ...ROOT, auth_type: "oauth2", credentials: { scopes: ["", null] } },
+ ui: {},
+ },
+ { label: "credentials absent entirely", values: { ...ROOT, auth_type: "oauth2" }, ui: {} },
+ {
+ label: "removeStoredApp forces an explicit-null app write",
+ values: { ...ROOT, auth_type: "true_passthrough", credentials: { ...CREDS } },
+ ui: { removeStoredApp: true, mcpServer: { ...SERVER, auth_type: "true_passthrough" } },
+ },
+ {
+ label: "removeStoredApp ignored outside a client-forwarded mode",
+ values: { ...ROOT, auth_type: "api_key", credentials: { ...CREDS } },
+ ui: { removeStoredApp: true },
+ },
+
+ // --- headers, env vars, access groups ---
+ {
+ label: "static headers trim and drop blank names",
+ values: {
+ ...ROOT,
+ static_headers: [
+ { header: " X-A ", value: " v " },
+ { header: " ", value: "x" },
+ ],
+ },
+ ui: {},
+ },
+ {
+ label: "env vars normalize",
+ values: { ...ROOT, env_vars: [{ name: "A", value: "1", scope: "server", description: "d" }] },
+ ui: {},
+ },
+ { label: "access groups given as objects", values: { ...ROOT, mcp_access_groups: [{ name: "g1" }, "g2"] }, ui: {} },
+ { label: "extra_headers absent falls back to an empty array", values: { ...ROOT, extra_headers: undefined }, ui: {} },
+
+ // --- tool allowlist and overrides ---
+ {
+ label: "existing allowlist enforces and emits allowed_tools",
+ values: { ...ROOT },
+ ui: { hasExistingToolAllowlist: true, allowedTools: ["alpha"] },
+ },
+ {
+ label: "allowlist interaction with an empty list still enforces",
+ values: { ...ROOT },
+ ui: { hasToolAllowlistInteraction: true },
+ },
+ {
+ label: "tool display name and description maps present",
+ values: { ...ROOT },
+ ui: { toolNameToDisplayName: { a: "Alpha" }, toolNameToDescription: { a: "desc" } },
+ },
+
+ // --- cost config, logo, token validation ---
+ // mcp_info.server_name walks a six-step fallback chain. Every step needs a case whose
+ // earlier terms are falsy, or the chain is unreachable and a mutation deleting it survives.
+ { label: "mcp_info server name falls back to the form url", values: { ...ROOT, server_name: "" }, ui: {} },
+ {
+ label: "mcp_info server name falls back to the stored server_name",
+ values: { ...ROOT, server_name: "", url: "" },
+ ui: {},
+ },
+ {
+ label: "mcp_info server name falls back to the stored url",
+ values: { ...ROOT, server_name: "", url: "" },
+ ui: { mcpServer: { ...SERVER, server_name: "" } },
+ },
+ {
+ label: "mcp_info server name falls back to the form alias",
+ values: { ...ROOT, server_name: "", url: "" },
+ ui: { mcpServer: { ...SERVER, server_name: "", url: "" } },
+ },
+ {
+ label: "mcp_info server name falls back to unknown",
+ values: { ...ROOT, server_name: "", url: "", alias: "" },
+ ui: { mcpServer: { ...SERVER, server_name: "", url: "", alias: "" } },
+ },
+
+ { label: "cost config present", values: { ...ROOT }, ui: { costConfig: { default_cost_per_query: 0.01 } as never } },
+ { label: "logo url present", values: { ...ROOT }, ui: { logoUrl: "https://cdn/logo.png" } },
+ { label: "token validation JSON parses", values: { ...ROOT, token_validation_json: '{"aud":"x"}' }, ui: {} },
+ {
+ label: "token validation blank clears an existing value",
+ values: { ...ROOT, token_validation_json: " " },
+ ui: { mcpServer: { ...SERVER, token_validation: { aud: "old" } } },
+ },
+ {
+ label: "token validation blank with no existing value omits the key",
+ values: { ...ROOT, token_validation_json: "" },
+ ui: {},
+ },
+
+ // --- the six failure branches ---
+ { label: "ERR invalid tool display name", values: { ...ROOT }, ui: { toolNameToDisplayName: { a: "has spaces" } } },
+ {
+ label: "ERR stdio config missing a command",
+ values: { ...ROOT, transport: "stdio", stdio_config: JSON.stringify({ args: [] }) },
+ ui: {},
+ },
+ {
+ label: "ERR stdio config invalid JSON",
+ values: { ...ROOT, transport: "stdio", stdio_config: "{not json" },
+ ui: {},
+ },
+ {
+ label: "ERR stdio env invalid JSON",
+ values: { ...ROOT, transport: "stdio", command: "npx", env_json: "{not json" },
+ ui: {},
+ },
+ {
+ label: "ERR stdio dedicated path with a blank command",
+ values: { ...ROOT, transport: "stdio", command: " " },
+ ui: {},
+ },
+ { label: "ERR token validation invalid JSON", values: { ...ROOT, token_validation_json: "{not json" }, ui: {} },
+];
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editServerPayload.differential.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editServerPayload.differential.test.ts
new file mode 100644
index 00000000000..f9cab181e71
--- /dev/null
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editServerPayload.differential.test.ts
@@ -0,0 +1,331 @@
+import { describe, expect, it } from "vitest";
+import {
+ ADMIN_CONFIG_CREDENTIAL_KEYS,
+ AUTH_TYPE,
+ MCPServer,
+ MCP_OAUTH2_FLOW_INTERACTIVE,
+ MCP_OAUTH2_FLOW_M2M,
+ OAUTH_FLOW,
+ TRANSPORT,
+ isClientForwardedTokenMode,
+ preservedAdminCredentials,
+} from "@/components/mcp_tools/types";
+import { AUTH_TYPES_REQUIRING_CREDENTIALS } from "./createServerPayload";
+import { TOOL_DISPLAY_NAME_PATTERN, normalizeEnvVars } from "./utils";
+import { buildEditServerPayload, type EditServerUiState } from "./editServerPayload";
+import { CASES, baseUi } from "./editServerPayload.differential.cases";
+
+// GENERATED by scratchpad/emit_test.py. The body below is machine-extracted from
+// mcp_server_edit.tsx lines 675-950 and converted by exact string replacement; do
+// not hand-edit it. Regenerate instead, so the baseline can never drift toward
+// the implementation it is supposed to check.
+const legacyBuild = (values: Record, ui: EditServerUiState) => {
+ const {
+ mcpServer,
+ logoUrl,
+ costConfig,
+ allowedTools,
+ hasExistingToolAllowlist,
+ hasToolAllowlistInteraction,
+ toolNameToDisplayName,
+ toolNameToDescription,
+ removeStoredApp,
+ } = ui;
+ const invalidDisplayName = Object.entries(toolNameToDisplayName).find(
+ ([, displayName]) => displayName && !TOOL_DISPLAY_NAME_PATTERN.test(displayName),
+ );
+ if (invalidDisplayName) {
+ return { kind: "invalid_tool_display_name" as const, displayName: String(invalidDisplayName[1]) };
+ }
+ // Ensure access groups is always a string array
+ const {
+ static_headers: staticHeadersList,
+ env_vars: envVarsList,
+ credentials: credentialValues,
+ stdio_config: rawStdioConfig,
+ env_json: rawEnvJson,
+ command: rawCommand,
+ args: rawArgs,
+ allow_all_keys: allowAllKeysRaw,
+ available_on_public_internet: availableOnPublicInternetRaw,
+ delegate_auth_to_upstream: delegateAuthToUpstreamRaw,
+ oauth_passthrough: oauthPassthroughRaw,
+ dcr_bridge: dcrBridgeRaw,
+ token_validation_json: rawTokenValidationJson,
+ ...restValues
+ } = values;
+
+ const accessGroups = (restValues.mcp_access_groups || []).map((g: any) =>
+ typeof g === "string" ? g : g.name || String(g),
+ );
+
+ const staticHeaders = Array.isArray(staticHeadersList)
+ ? staticHeadersList.reduce((acc: Record, entry: Record) => {
+ const header = entry?.header?.trim();
+ if (!header) {
+ return acc;
+ }
+ acc[header] = (entry?.value ?? "").trim();
+ return acc;
+ }, {})
+ : ({} as Record);
+
+ const envVars = normalizeEnvVars(envVarsList);
+
+ const credentialsPayload =
+ credentialValues && typeof credentialValues === "object"
+ ? Object.entries(credentialValues).reduce((acc: Record, [key, value]) => {
+ if (value === undefined || value === null || value === "") {
+ if (value === "" && (ADMIN_CONFIG_CREDENTIAL_KEYS as readonly string[]).includes(key)) {
+ acc[key] = null;
+ }
+ return acc;
+ }
+ if (key === "scopes") {
+ if (Array.isArray(value)) {
+ const filteredScopes = value.filter((scope) => scope != null && scope !== "");
+ if (filteredScopes.length > 0) {
+ acc[key] = filteredScopes;
+ }
+ }
+ } else {
+ acc[key] = value;
+ }
+ return acc;
+ }, {})
+ : undefined;
+
+ let stdioFields: Record = {};
+
+ if (restValues.transport === "stdio") {
+ // Prefer JSON config if provided (matches Create screen behavior)
+ if (rawStdioConfig) {
+ try {
+ const stdioConfig = JSON.parse(rawStdioConfig);
+
+ let actualConfig = stdioConfig;
+ if (stdioConfig?.mcpServers && typeof stdioConfig.mcpServers === "object") {
+ const serverNames = Object.keys(stdioConfig.mcpServers);
+ if (serverNames.length > 0) {
+ actualConfig = stdioConfig.mcpServers[serverNames[0]];
+ }
+ }
+
+ const parsedArgs = Array.isArray(actualConfig?.args)
+ ? actualConfig.args.map((v: any) => String(v)).filter((v: string) => v.trim() !== "")
+ : [];
+
+ const parsedEnv =
+ actualConfig?.env && typeof actualConfig.env === "object" && !Array.isArray(actualConfig.env)
+ ? Object.entries(actualConfig.env).reduce((acc: Record, [k, v]) => {
+ if (k == null || String(k).trim() === "") return acc;
+ acc[String(k)] = v == null ? "" : String(v);
+ return acc;
+ }, {})
+ : {};
+
+ stdioFields = {
+ command: actualConfig?.command ? String(actualConfig.command) : undefined,
+ args: parsedArgs,
+ env: parsedEnv,
+ };
+
+ if (!stdioFields.command) {
+ return { kind: "stdio_config_missing_command" as const };
+ }
+ } catch {
+ return { kind: "invalid_stdio_json" as const };
+ }
+ } else {
+ // Dedicated fields path (command/args + env JSON)
+ let parsedEnv: Record = {};
+ if (rawEnvJson) {
+ try {
+ const env = JSON.parse(rawEnvJson);
+ if (env && typeof env === "object" && !Array.isArray(env)) {
+ parsedEnv = Object.entries(env).reduce((acc: Record, [k, v]) => {
+ if (k == null || String(k).trim() === "") return acc;
+ acc[String(k)] = v == null ? "" : String(v);
+ return acc;
+ }, {});
+ }
+ } catch {
+ return { kind: "invalid_stdio_env_json" as const };
+ }
+ }
+ const parsedArgs = Array.isArray(rawArgs)
+ ? rawArgs.map((v: any) => String(v)).filter((v: string) => v.trim() !== "")
+ : [];
+
+ const parsedCommand = rawCommand ? String(rawCommand).trim() : "";
+ if (!parsedCommand) {
+ return { kind: "stdio_command_required" as const };
+ }
+
+ stdioFields = {
+ command: parsedCommand,
+ args: parsedArgs,
+ env: parsedEnv,
+ };
+ }
+ }
+
+ // Map "openapi" transport to "http" for the backend
+ if (restValues.transport === TRANSPORT.OPENAPI) {
+ restValues.transport = "http";
+ }
+
+ // Parse token_validation JSON if provided
+ let tokenValidation: Record | null = null;
+ if (rawTokenValidationJson && rawTokenValidationJson.trim() !== "") {
+ try {
+ tokenValidation = JSON.parse(rawTokenValidationJson);
+ } catch {
+ return { kind: "invalid_token_validation_json" as const };
+ }
+ }
+
+ // Prepare the payload with cost configuration and permission fields
+ const mcpInfoServerName =
+ restValues.server_name ||
+ restValues.url ||
+ mcpServer.server_name ||
+ mcpServer.url ||
+ restValues.alias ||
+ mcpServer.alias ||
+ "unknown";
+
+ const toolAllowlistEnforced = hasExistingToolAllowlist || hasToolAllowlistInteraction || allowedTools.length > 0;
+
+ const payload: Record = {
+ ...restValues,
+ ...stdioFields,
+ // Remove UI-only fields
+ stdio_config: undefined,
+ env_json: undefined,
+ ...(mcpServer.auth_type === AUTH_TYPE.OAUTH2 && restValues.auth_type !== AUTH_TYPE.OAUTH2
+ ? { issuer: null, authorization_url: null, token_url: null, registration_url: null }
+ : {}),
+ ...(mcpServer.auth_type === AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE &&
+ restValues.auth_type !== AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE
+ ? { token_exchange_endpoint: null, audience: null, subject_token_type: null, token_exchange_profile: null }
+ : {}),
+ server_id: mcpServer.server_id,
+ mcp_info: {
+ ...(mcpServer.mcp_info ?? {}),
+ server_name: mcpInfoServerName,
+ description: restValues.description,
+ logo_url: logoUrl || undefined,
+ mcp_server_cost_info: Object.keys(costConfig).length > 0 ? costConfig : null,
+ tool_allowlist_enforced: toolAllowlistEnforced,
+ },
+ mcp_access_groups: accessGroups,
+ alias: restValues.alias,
+ // Include permission management fields
+ extra_headers: restValues.extra_headers || [],
+ ...(toolAllowlistEnforced
+ ? {
+ allowed_tools: allowedTools,
+ }
+ : {}),
+ tool_name_to_display_name: Object.keys(toolNameToDisplayName).length > 0 ? toolNameToDisplayName : null,
+ tool_name_to_description: Object.keys(toolNameToDescription).length > 0 ? toolNameToDescription : null,
+ disallowed_tools: restValues.disallowed_tools || [],
+ static_headers: staticHeaders,
+ env_vars: envVars,
+ allow_all_keys: Boolean(allowAllKeysRaw ?? mcpServer.allow_all_keys),
+ available_on_public_internet: Boolean(availableOnPublicInternetRaw ?? mcpServer.available_on_public_internet),
+ // ``delegate_auth_to_upstream`` is only honored server-side for
+ // ``auth_type=oauth2`` (PKCE passthrough). The field is
+ // conditionally rendered so the value drops out of the form on
+ // auth_type change; force false for any other configuration to avoid
+ // persisting a stale ``true`` that would silently re-activate if the
+ // configuration is later switched back.
+ delegate_auth_to_upstream: (() => {
+ const isOauth2 = restValues.auth_type === AUTH_TYPE.OAUTH2;
+ return isOauth2 ? Boolean(delegateAuthToUpstreamRaw ?? mcpServer.delegate_auth_to_upstream) : false;
+ })(),
+ // ``oauth_passthrough`` is the dedicated, non-oauth2 opt-in. It is only
+ // honored for ``auth_type=none`` servers that forward ``Authorization``
+ // upstream. Kept separate from ``delegate_auth_to_upstream`` so enabling
+ // pass-through never regresses oauth2 servers. Force false otherwise.
+ oauth_passthrough: (() => {
+ const isNoneAuth = restValues.auth_type === AUTH_TYPE.NONE || restValues.auth_type == null;
+ const extraHeaders = Array.isArray(restValues.extra_headers) ? restValues.extra_headers : [];
+ const hasAuthorizationHeader = extraHeaders.some(
+ (h: unknown) => typeof h === "string" && h.toLowerCase() === "authorization",
+ );
+ return isNoneAuth && hasAuthorizationHeader ? Boolean(oauthPassthroughRaw ?? mcpServer.oauth_passthrough) : false;
+ })(),
+ // ``dcr_bridge`` is only meaningful for the client-forwarded token
+ // modes (true_passthrough / oauth_delegate). The field is
+ // conditionally rendered so the value drops out of the form on
+ // auth_type change; force false for any other configuration to avoid
+ // persisting a stale ``true`` that would silently re-activate if the
+ // mode is later switched back.
+ dcr_bridge: isClientForwardedTokenMode(restValues.auth_type)
+ ? Boolean(dcrBridgeRaw ?? mcpServer.dcr_bridge)
+ : false,
+ ...(restValues.auth_type === AUTH_TYPE.OAUTH2 && restValues.oauth_flow_type
+ ? {
+ oauth2_flow:
+ restValues.oauth_flow_type === OAUTH_FLOW.M2M ? MCP_OAUTH2_FLOW_M2M : MCP_OAUTH2_FLOW_INTERACTIVE,
+ }
+ : {}),
+ // Include token_validation when it is set (non-null) or when clearing an existing value
+ ...(tokenValidation !== null || mcpServer.token_validation ? { token_validation: tokenValidation } : {}),
+ };
+
+ const includeCredentials = restValues.auth_type && AUTH_TYPES_REQUIRING_CREDENTIALS.includes(restValues.auth_type);
+
+ // Client-forwarded rows persist ONLY the declared app; strip any token material lingering in the
+ // form (e.g. from a prior oauth2 authorize this session) so it can never reach the row.
+ const submitCredentials = isClientForwardedTokenMode(restValues.auth_type)
+ ? preservedAdminCredentials(credentialsPayload)
+ : credentialsPayload;
+
+ if (includeCredentials && submitCredentials && Object.keys(submitCredentials).length > 0) {
+ payload.credentials = submitCredentials;
+ }
+
+ // Explicit removal of a saved app for the client-forwarded modes, applied AFTER the filter so it
+ // always wins. Blank fields are the keep-existing convention (the backend merges partial
+ // credential updates), so removal must be an explicit-null write: encrypt skips nulls and the
+ // merge overrides the stored keys, returning the server to dynamic client registration.
+ if (removeStoredApp && isClientForwardedTokenMode(restValues.auth_type)) {
+ payload.credentials = { client_id: null, client_secret: null };
+ }
+ return { kind: "ok" as const, payload };
+};
+
+const clone = (v: T): T => structuredClone(v);
+
+describe("buildEditServerPayload matches the pre-extraction handleSave body", () => {
+ it.each(CASES.map((c) => [c.label, c] as const))("%s", (_label, testCase) => {
+ const ui: EditServerUiState = { ...baseUi, ...testCase.ui } as EditServerUiState;
+ const legacy = legacyBuild(clone(testCase.values), clone(ui) as EditServerUiState);
+ const next = buildEditServerPayload(clone(testCase.values), clone(ui) as EditServerUiState);
+
+ expect(next.kind).toBe(legacy.kind);
+ if (legacy.kind !== "ok" || next.kind !== "ok") {
+ expect(next).toStrictEqual(legacy);
+ return;
+ }
+ expect(next.payload).toStrictEqual(legacy.payload);
+ expect(Object.keys(next.payload)).toStrictEqual(Object.keys(legacy.payload));
+ expect(JSON.stringify(next.payload)).toBe(JSON.stringify(legacy.payload));
+ });
+});
+
+void ADMIN_CONFIG_CREDENTIAL_KEYS;
+void AUTH_TYPE;
+void AUTH_TYPES_REQUIRING_CREDENTIALS;
+void MCP_OAUTH2_FLOW_INTERACTIVE;
+void MCP_OAUTH2_FLOW_M2M;
+void OAUTH_FLOW;
+void TOOL_DISPLAY_NAME_PATTERN;
+void TRANSPORT;
+void isClientForwardedTokenMode;
+void normalizeEnvVars;
+void preservedAdminCredentials;
+export type { MCPServer };
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editServerPayload.ts b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editServerPayload.ts
new file mode 100644
index 00000000000..3930f1edb9b
--- /dev/null
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editServerPayload.ts
@@ -0,0 +1,405 @@
+import {
+ ADMIN_CONFIG_CREDENTIAL_KEYS,
+ AUTH_TYPE,
+ MCPEnvVar,
+ MCPInfo,
+ MCPServer,
+ MCPServerCostInfo,
+ MCP_OAUTH2_FLOW_INTERACTIVE,
+ MCP_OAUTH2_FLOW_M2M,
+ OAUTH_FLOW,
+ TRANSPORT,
+ isClientForwardedTokenMode,
+ preservedAdminCredentials,
+} from "@/components/mcp_tools/types";
+import { AUTH_TYPES_REQUIRING_CREDENTIALS, reduceStaticHeaders } from "./createServerPayload";
+import { TOOL_DISPLAY_NAME_PATTERN, normalizeEnvVars } from "./utils";
+
+export type MCPAccessGroupValue = string | { readonly name?: string };
+
+export interface EditServerFormValues {
+ readonly static_headers?: unknown;
+ readonly env_vars?: unknown;
+ readonly credentials?: unknown;
+ readonly stdio_config?: string;
+ readonly env_json?: string;
+ readonly command?: string;
+ readonly args?: readonly string[];
+ readonly allow_all_keys?: boolean;
+ readonly available_on_public_internet?: boolean;
+ readonly delegate_auth_to_upstream?: boolean;
+ readonly oauth_passthrough?: boolean;
+ readonly dcr_bridge?: boolean;
+ readonly token_validation_json?: string;
+ readonly mcp_access_groups?: readonly MCPAccessGroupValue[];
+ readonly transport?: string;
+ readonly server_name?: string;
+ readonly url?: string;
+ readonly alias?: string;
+ readonly description?: string;
+ readonly auth_type?: string;
+ readonly extra_headers?: readonly string[];
+ readonly disallowed_tools?: readonly string[];
+ readonly oauth_flow_type?: string;
+ readonly [key: string]: unknown;
+}
+
+export interface EditServerPayload {
+ readonly server_id: string;
+ readonly mcp_info: MCPInfo;
+ readonly mcp_access_groups: readonly string[];
+ readonly alias: string | undefined;
+ readonly extra_headers: readonly string[];
+ readonly disallowed_tools: readonly string[];
+ readonly static_headers: Readonly>;
+ readonly env_vars: readonly MCPEnvVar[];
+ readonly allow_all_keys: boolean;
+ readonly available_on_public_internet: boolean;
+ readonly delegate_auth_to_upstream: boolean;
+ readonly oauth_passthrough: boolean;
+ readonly dcr_bridge: boolean;
+ readonly stdio_config: undefined;
+ readonly env_json: undefined;
+ readonly command?: string;
+ readonly args?: readonly string[];
+ readonly env?: Readonly>;
+ readonly allowed_tools?: readonly string[];
+ readonly tool_name_to_display_name: Readonly> | null;
+ readonly tool_name_to_description: Readonly> | null;
+ readonly oauth2_flow?: string;
+ readonly token_validation?: unknown;
+ readonly credentials?: Readonly>;
+ readonly [key: string]: unknown;
+}
+
+export interface EditServerUiState {
+ readonly mcpServer: MCPServer;
+ readonly logoUrl: string | undefined;
+ readonly costConfig: MCPServerCostInfo;
+ readonly allowedTools: readonly string[];
+ readonly hasExistingToolAllowlist: boolean;
+ readonly hasToolAllowlistInteraction: boolean;
+ readonly toolNameToDisplayName: Readonly>;
+ readonly toolNameToDescription: Readonly>;
+ readonly removeStoredApp: boolean;
+}
+
+export type BuildEditPayloadResult =
+ | { readonly kind: "ok"; readonly payload: EditServerPayload }
+ | { readonly kind: "invalid_tool_display_name"; readonly displayName: string }
+ | { readonly kind: "stdio_config_missing_command" }
+ | { readonly kind: "invalid_stdio_json" }
+ | { readonly kind: "invalid_stdio_env_json" }
+ | { readonly kind: "stdio_command_required" }
+ | { readonly kind: "invalid_token_validation_json" };
+
+interface StdioFields {
+ readonly command: string;
+ readonly args: readonly string[];
+ readonly env: Readonly>;
+}
+
+type StdioFieldsResult =
+ | { readonly kind: "ok"; readonly fields: StdioFields | Record }
+ | { readonly kind: "stdio_config_missing_command" }
+ | { readonly kind: "invalid_stdio_json" }
+ | { readonly kind: "invalid_stdio_env_json" }
+ | { readonly kind: "stdio_command_required" };
+
+type TokenValidationResult = { readonly kind: "ok"; readonly value: unknown } | { readonly kind: "invalid" };
+
+const assertNever = (value: never): never => {
+ throw new Error(`unhandled edit payload result: ${JSON.stringify(value)}`);
+};
+
+export const editPayloadErrorMessage = (result: Exclude): string => {
+ switch (result.kind) {
+ case "invalid_tool_display_name":
+ return `Tool display name "${result.displayName}" is invalid. Only letters, digits, underscores, and hyphens are allowed (no spaces).`;
+ case "stdio_config_missing_command":
+ return "Stdio configuration must include a command";
+ case "invalid_stdio_json":
+ return "Invalid JSON in stdio configuration";
+ case "invalid_stdio_env_json":
+ return "Invalid JSON in stdio env configuration";
+ case "stdio_command_required":
+ return "Stdio transport requires a command";
+ case "invalid_token_validation_json":
+ return "Invalid JSON in Token Validation Rules";
+ default:
+ return assertNever(result);
+ }
+};
+
+const toStringArgs = (raw: unknown): readonly string[] =>
+ Array.isArray(raw) ? raw.map((v: unknown) => String(v)).filter((v: string) => v.trim() !== "") : [];
+
+const toEnvRecord = (raw: unknown): Readonly> =>
+ raw && typeof raw === "object" && !Array.isArray(raw)
+ ? Object.fromEntries(
+ Object.entries(raw)
+ .filter(([k]) => k != null && String(k).trim() !== "")
+ .map(([k, v]) => [String(k), v == null ? "" : String(v)]),
+ )
+ : {};
+
+const buildStdioFields = (
+ rawStdioConfig: string | undefined,
+ rawEnvJson: string | undefined,
+ rawCommand: string | undefined,
+ rawArgs: readonly string[] | undefined,
+): StdioFieldsResult => {
+ if (rawStdioConfig) {
+ try {
+ const stdioConfig: unknown = JSON.parse(rawStdioConfig);
+ const configRecord = stdioConfig && typeof stdioConfig === "object" ? (stdioConfig as StdioConfigShape) : null;
+ const namedServers =
+ configRecord?.mcpServers && typeof configRecord.mcpServers === "object" ? configRecord.mcpServers : null;
+ const named = namedServers ? Object.keys(namedServers) : [];
+ const actualConfig = named.length > 0 && namedServers ? namedServers[named[0]] : configRecord;
+ const command = actualConfig?.command ? String(actualConfig.command) : undefined;
+ if (!command) {
+ return { kind: "stdio_config_missing_command" };
+ }
+ return {
+ kind: "ok",
+ fields: { command, args: toStringArgs(actualConfig?.args), env: toEnvRecord(actualConfig?.env) },
+ };
+ } catch {
+ return { kind: "invalid_stdio_json" };
+ }
+ }
+
+ const envResult = ((): Readonly> | "invalid" => {
+ if (!rawEnvJson) return {};
+ try {
+ return toEnvRecord(JSON.parse(rawEnvJson));
+ } catch {
+ return "invalid";
+ }
+ })();
+ if (envResult === "invalid") {
+ return { kind: "invalid_stdio_env_json" };
+ }
+
+ const parsedCommand = rawCommand ? String(rawCommand).trim() : "";
+ if (!parsedCommand) {
+ return { kind: "stdio_command_required" };
+ }
+ return { kind: "ok", fields: { command: parsedCommand, args: toStringArgs(rawArgs), env: envResult } };
+};
+
+interface StdioConfigShape {
+ readonly mcpServers?: Readonly>;
+ readonly command?: unknown;
+ readonly args?: unknown;
+ readonly env?: unknown;
+}
+
+const buildCredentials = (credentialValues: unknown): Readonly> | undefined => {
+ if (!credentialValues || typeof credentialValues !== "object") return undefined;
+ const kept = Object.entries(credentialValues).flatMap(([key, value]): readonly (readonly [string, unknown])[] => {
+ if (value === undefined || value === null || value === "") {
+ return value === "" && (ADMIN_CONFIG_CREDENTIAL_KEYS as readonly string[]).includes(key)
+ ? [[key, null] as const]
+ : [];
+ }
+ if (key !== "scopes") return [[key, value] as const];
+ if (!Array.isArray(value)) return [];
+ const filteredScopes = value.filter((scope: unknown) => scope != null && scope !== "");
+ return filteredScopes.length > 0 ? [[key, filteredScopes] as const] : [];
+ });
+ return Object.fromEntries(kept);
+};
+
+interface CredentialsEntryInput {
+ readonly authType: string | undefined;
+ readonly credentials: Readonly> | undefined;
+ readonly includeCredentials: boolean;
+ readonly removeStoredApp: boolean;
+}
+
+// Explicit removal of a saved app for the client-forwarded modes, applied AFTER the filter so it
+// always wins. Blank fields are the keep-existing convention (the backend merges partial credential
+// updates), so removal must be an explicit-null write: encrypt skips nulls and the merge overrides
+// the stored keys, returning the server to dynamic client registration.
+const resolveCredentialsEntry = ({
+ authType,
+ credentials,
+ includeCredentials,
+ removeStoredApp,
+}: CredentialsEntryInput): { readonly credentials?: Readonly> } => {
+ if (removeStoredApp && isClientForwardedTokenMode(authType)) {
+ return { credentials: { client_id: null, client_secret: null } };
+ }
+ if (includeCredentials && credentials && Object.keys(credentials).length > 0) {
+ return { credentials };
+ }
+ return {};
+};
+
+export const buildEditServerPayload = (values: EditServerFormValues, ui: EditServerUiState): BuildEditPayloadResult => {
+ const {
+ mcpServer,
+ logoUrl,
+ costConfig,
+ allowedTools,
+ hasExistingToolAllowlist,
+ hasToolAllowlistInteraction,
+ toolNameToDisplayName,
+ toolNameToDescription,
+ removeStoredApp,
+ } = ui;
+
+ const invalidDisplayName = Object.entries(toolNameToDisplayName).find(
+ ([, displayName]) => displayName && !TOOL_DISPLAY_NAME_PATTERN.test(displayName),
+ );
+ if (invalidDisplayName) {
+ return { kind: "invalid_tool_display_name", displayName: String(invalidDisplayName[1]) };
+ }
+
+ const {
+ static_headers: staticHeadersList,
+ env_vars: envVarsList,
+ credentials: credentialValues,
+ stdio_config: rawStdioConfig,
+ env_json: rawEnvJson,
+ command: rawCommand,
+ args: rawArgs,
+ allow_all_keys: allowAllKeysRaw,
+ available_on_public_internet: availableOnPublicInternetRaw,
+ delegate_auth_to_upstream: delegateAuthToUpstreamRaw,
+ oauth_passthrough: oauthPassthroughRaw,
+ dcr_bridge: dcrBridgeRaw,
+ token_validation_json: rawTokenValidationJson,
+ ...rawRestValues
+ } = values;
+
+ const accessGroups = (rawRestValues.mcp_access_groups || []).map((g) =>
+ typeof g === "string" ? g : g.name || String(g),
+ );
+ const staticHeaders = reduceStaticHeaders(staticHeadersList);
+ const envVars = normalizeEnvVars(envVarsList);
+ const credentialsPayload = buildCredentials(credentialValues);
+
+ const stdio =
+ rawRestValues.transport === "stdio"
+ ? buildStdioFields(rawStdioConfig, rawEnvJson, rawCommand, rawArgs)
+ : ({ kind: "ok", fields: {} } as StdioFieldsResult);
+ if (stdio.kind !== "ok") {
+ return stdio;
+ }
+
+ // Map "openapi" transport to "http" for the backend. Rewriting the existing key keeps its
+ // position, which the payload's serialised key order depends on.
+ const restValues =
+ rawRestValues.transport === TRANSPORT.OPENAPI ? { ...rawRestValues, transport: "http" } : rawRestValues;
+
+ const tokenValidation = ((): TokenValidationResult => {
+ if (!rawTokenValidationJson || rawTokenValidationJson.trim() === "") return { kind: "ok", value: null };
+ try {
+ return { kind: "ok", value: JSON.parse(rawTokenValidationJson) };
+ } catch {
+ return { kind: "invalid" };
+ }
+ })();
+ if (tokenValidation.kind === "invalid") {
+ return { kind: "invalid_token_validation_json" };
+ }
+
+ const mcpInfoServerName =
+ restValues.server_name ||
+ restValues.url ||
+ mcpServer.server_name ||
+ mcpServer.url ||
+ restValues.alias ||
+ mcpServer.alias ||
+ "unknown";
+
+ const toolAllowlistEnforced = hasExistingToolAllowlist || hasToolAllowlistInteraction || allowedTools.length > 0;
+
+ const extraHeaders = restValues.extra_headers || [];
+ const hasAuthorizationHeader = extraHeaders.some((h) => typeof h === "string" && h.toLowerCase() === "authorization");
+ const isNoneAuth = restValues.auth_type === AUTH_TYPE.NONE || restValues.auth_type == null;
+
+ // Client-forwarded rows persist ONLY the declared app; strip any token material lingering in the
+ // form (e.g. from a prior oauth2 authorize this session) so it can never reach the row.
+ const submitCredentials = isClientForwardedTokenMode(restValues.auth_type)
+ ? preservedAdminCredentials(credentialsPayload)
+ : credentialsPayload;
+ const includeCredentials = restValues.auth_type && AUTH_TYPES_REQUIRING_CREDENTIALS.includes(restValues.auth_type);
+
+ const credentialsEntryInput: CredentialsEntryInput = {
+ authType: restValues.auth_type,
+ credentials: submitCredentials,
+ includeCredentials: Boolean(includeCredentials),
+ removeStoredApp,
+ };
+ const credentialsEntry = resolveCredentialsEntry(credentialsEntryInput);
+
+ const payload: EditServerPayload = {
+ ...restValues,
+ ...stdio.fields,
+ // Remove UI-only fields
+ stdio_config: undefined,
+ env_json: undefined,
+ ...(mcpServer.auth_type === AUTH_TYPE.OAUTH2 && restValues.auth_type !== AUTH_TYPE.OAUTH2
+ ? { issuer: null, authorization_url: null, token_url: null, registration_url: null }
+ : {}),
+ ...(mcpServer.auth_type === AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE &&
+ restValues.auth_type !== AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE
+ ? { token_exchange_endpoint: null, audience: null, subject_token_type: null, token_exchange_profile: null }
+ : {}),
+ server_id: mcpServer.server_id,
+ mcp_info: {
+ ...(mcpServer.mcp_info ?? {}),
+ server_name: mcpInfoServerName,
+ description: restValues.description,
+ logo_url: logoUrl || undefined,
+ mcp_server_cost_info: Object.keys(costConfig).length > 0 ? costConfig : null,
+ tool_allowlist_enforced: toolAllowlistEnforced,
+ },
+ mcp_access_groups: accessGroups,
+ alias: restValues.alias,
+ // Include permission management fields
+ extra_headers: extraHeaders,
+ ...(toolAllowlistEnforced ? { allowed_tools: allowedTools } : {}),
+ tool_name_to_display_name: Object.keys(toolNameToDisplayName).length > 0 ? toolNameToDisplayName : null,
+ tool_name_to_description: Object.keys(toolNameToDescription).length > 0 ? toolNameToDescription : null,
+ disallowed_tools: restValues.disallowed_tools || [],
+ static_headers: staticHeaders,
+ env_vars: envVars,
+ allow_all_keys: Boolean(allowAllKeysRaw ?? mcpServer.allow_all_keys),
+ available_on_public_internet: Boolean(availableOnPublicInternetRaw ?? mcpServer.available_on_public_internet),
+ // ``delegate_auth_to_upstream`` is only honored server-side for ``auth_type=oauth2`` (PKCE
+ // passthrough). The field unmounts on auth_type change, so force false for any other
+ // configuration to avoid persisting a stale ``true`` that would silently re-activate.
+ delegate_auth_to_upstream:
+ restValues.auth_type === AUTH_TYPE.OAUTH2
+ ? Boolean(delegateAuthToUpstreamRaw ?? mcpServer.delegate_auth_to_upstream)
+ : false,
+ // ``oauth_passthrough`` is the dedicated, non-oauth2 opt-in, honored only for ``auth_type=none``
+ // servers that forward ``Authorization`` upstream. Kept separate from
+ // ``delegate_auth_to_upstream`` so enabling pass-through never regresses oauth2 servers.
+ oauth_passthrough:
+ isNoneAuth && hasAuthorizationHeader ? Boolean(oauthPassthroughRaw ?? mcpServer.oauth_passthrough) : false,
+ // ``dcr_bridge`` is only meaningful for the client-forwarded token modes. The field unmounts on
+ // auth_type change, so force false otherwise rather than persisting a stale ``true``.
+ dcr_bridge: isClientForwardedTokenMode(restValues.auth_type)
+ ? Boolean(dcrBridgeRaw ?? mcpServer.dcr_bridge)
+ : false,
+ ...(restValues.auth_type === AUTH_TYPE.OAUTH2 && restValues.oauth_flow_type
+ ? {
+ oauth2_flow:
+ restValues.oauth_flow_type === OAUTH_FLOW.M2M ? MCP_OAUTH2_FLOW_M2M : MCP_OAUTH2_FLOW_INTERACTIVE,
+ }
+ : {}),
+ // Include token_validation when it is set (non-null) or when clearing an existing value
+ ...(tokenValidation.value !== null || mcpServer.token_validation
+ ? { token_validation: tokenValidation.value }
+ : {}),
+ ...credentialsEntry,
+ };
+
+ return { kind: "ok", payload };
+};
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcpFieldRules.ts b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcpFieldRules.ts
new file mode 100644
index 00000000000..76ff74eac52
--- /dev/null
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcpFieldRules.ts
@@ -0,0 +1,98 @@
+import type { Validate } from "react-hook-form";
+
+import type { MountedFieldControlProps, MountedFormValues } from "@/components/common_components/MountedFormField";
+
+type McpValidate = Validate;
+
+const ariaOf = (control: MountedFieldControlProps) => ({
+ id: control.id,
+ onBlur: control.onBlur,
+ "aria-required": control["aria-required"],
+ "aria-invalid": control["aria-invalid"],
+ "aria-describedby": control["aria-describedby"],
+});
+
+export const textControl = (control: MountedFieldControlProps) => ({
+ ...ariaOf(control),
+ name: control.name,
+ value: control.value === null || control.value === undefined ? "" : String(control.value),
+ onChange: control.onChange,
+});
+
+export const selectControl = (control: MountedFieldControlProps) => ({
+ ...ariaOf(control),
+ value: control.value as TValue,
+ onChange: control.onChange,
+});
+
+export const numberControl = (control: MountedFieldControlProps) => ({
+ ...ariaOf(control),
+ value: control.value as number | null | undefined,
+ onChange: control.onChange,
+});
+
+export const switchControl = (control: MountedFieldControlProps) => ({
+ ...ariaOf(control),
+ checked: control.value === true,
+ onChange: control.onChange,
+});
+
+export const invertedSwitchControl = (control: MountedFieldControlProps) => ({
+ ...ariaOf(control),
+ checked: control.value !== true,
+ onChange: (checked: boolean) => control.onChange(!checked),
+});
+
+export const valueAt = (values: MountedFormValues, path: readonly string[]): unknown =>
+ path.reduce(
+ (node, segment) => (node === null || node === undefined ? undefined : (node as Record)[segment]),
+ values,
+ );
+
+export const parsesAsJson =
+ (message: string): McpValidate =>
+ (value) => {
+ if (typeof value !== "string" || value.trim() === "") {
+ return true;
+ }
+ try {
+ JSON.parse(value);
+ return true;
+ } catch {
+ return message;
+ }
+ };
+
+export const parsesAsJsonObject =
+ (message: string, notObjectMessage: string): McpValidate =>
+ (value) => {
+ if (typeof value !== "string" || value === "") {
+ return true;
+ }
+ try {
+ const parsed: unknown = JSON.parse(value);
+ return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) ? true : notObjectMessage;
+ } catch {
+ return message;
+ }
+ };
+
+export const matchesPattern =
+ (pattern: RegExp, message: string): McpValidate =>
+ (value) =>
+ typeof value === "string" && value !== "" && !pattern.test(value) ? message : true;
+
+export const notOnlyWhitespace =
+ (message: string): McpValidate =>
+ (value) =>
+ typeof value === "string" && value !== "" && value.trim() === "" ? message : true;
+
+export const requiredWhenSiblingSet =
+ (siblingPath: readonly string[], message: string): McpValidate =>
+ (value, values) =>
+ valueAt(values, siblingPath) && !value ? message : true;
+
+export const requiredUnlessSiblingSet =
+ (siblingPath: readonly string[], message: string): McpValidate =>
+ (value, values) =>
+ value || valueAt(values, siblingPath) ? true : message;
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcpFormStore.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcpFormStore.test.tsx
new file mode 100644
index 00000000000..25eba43335e
--- /dev/null
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcpFormStore.test.tsx
@@ -0,0 +1,132 @@
+import React from "react";
+import { describe, expect, it } from "vitest";
+import { render } from "@testing-library/react";
+import { useForm } from "react-hook-form";
+
+import type { MountedFormValues } from "@/components/common_components/MountedFormField";
+import { allFieldsValue, deepMergedFieldsValue, resetFields, setFieldsValue, singleBranchChange } from "./mcpFormStore";
+
+const withForm = (
+ defaultValues: MountedFormValues,
+ act: (form: ReturnType>) => void,
+) => {
+ let store: MountedFormValues = {};
+ const Probe: React.FC = () => {
+ const form = useForm({ defaultValues });
+ React.useEffect(() => {
+ act(form);
+ store = allFieldsValue(form);
+ }, [form]);
+ return null;
+ };
+ render( );
+ return store;
+};
+
+describe("deepMergedFieldsValue", () => {
+ it("keeps a sibling key when a nested object is written, which is what preserves a declared app", () => {
+ expect(
+ deepMergedFieldsValue(
+ { credentials: { client_id: "kept", access_token: "tok" } },
+ { credentials: { client_id: "typed" } },
+ ),
+ ).toStrictEqual({ credentials: { client_id: "typed", access_token: "tok" } });
+ });
+
+ it("replaces an array rather than merging it index by index", () => {
+ expect(deepMergedFieldsValue({ extra_headers: ["a", "b", "c"] }, { extra_headers: ["z"] })).toStrictEqual({
+ extra_headers: ["z"],
+ });
+ });
+
+ it("writes an explicit undefined instead of skipping the key, which is how a transport switch clears a field", () => {
+ const merged = deepMergedFieldsValue({ url: "https://old", auth_type: "api_key" }, { url: undefined });
+ expect(merged).toStrictEqual({ url: undefined, auth_type: "api_key" });
+ expect("url" in merged).toBe(true);
+ });
+
+ it("writes an explicit null rather than treating it as a merge target", () => {
+ expect(deepMergedFieldsValue({ credentials: { client_id: "x" } }, { credentials: null })).toStrictEqual({
+ credentials: null,
+ });
+ });
+
+ it("replaces a primitive with an object when the incoming value is an object", () => {
+ expect(deepMergedFieldsValue({ credentials: "not-an-object" }, { credentials: { client_id: "x" } })).toStrictEqual({
+ credentials: { client_id: "x" },
+ });
+ });
+
+ it("does not mutate the store it was handed", () => {
+ const store = { credentials: { client_id: "kept" } };
+ deepMergedFieldsValue(store, { credentials: { client_secret: "added" } });
+ expect(store).toStrictEqual({ credentials: { client_id: "kept" } });
+ });
+
+ it("treats a missing store as empty rather than throwing", () => {
+ expect(deepMergedFieldsValue(undefined, { alias: "a" })).toStrictEqual({ alias: "a" });
+ });
+});
+
+describe("singleBranchChange", () => {
+ it("carries only the changed leaf, so re-applying it cannot resurrect a sibling token key", () => {
+ expect(
+ singleBranchChange("credentials.client_id", { credentials: { client_id: "typed", access_token: "stale" } }),
+ ).toStrictEqual({ credentials: { client_id: "typed" } });
+ });
+
+ it("exposes the changed top-level key so an upstream-field check can test membership", () => {
+ const changed = singleBranchChange("url", { url: "https://new", alias: "a" });
+ expect("url" in changed).toBe(true);
+ expect("alias" in changed).toBe(false);
+ });
+
+ it("builds an array for a numeric segment so a list row does not become an object keyed by index", () => {
+ expect(
+ singleBranchChange("static_headers.1.value", { static_headers: [{ value: "a" }, { value: "b" }] }),
+ ).toStrictEqual({ static_headers: [undefined, { value: "b" }] });
+ });
+
+ it("yields an undefined leaf rather than throwing when the path is not in the store", () => {
+ expect(singleBranchChange("credentials.client_secret", {})).toStrictEqual({
+ credentials: { client_secret: undefined },
+ });
+ });
+});
+
+describe("resetFields", () => {
+ it("restores the seeded value rather than clearing the key, so an edit reset keeps the saved server's credentials", () => {
+ const store = withForm({ credentials: { client_id: "saved", access_token: "tok" } }, (form) => {
+ form.setValue("credentials", { client_id: "typed" });
+ resetFields(form, ["credentials"], { credentials: { client_id: "saved", access_token: "tok" } });
+ });
+
+ expect(store.credentials).toStrictEqual({ client_id: "saved", access_token: "tok" });
+ });
+
+ it("clears the key when no seed is supplied, which is what the create form's blank store means", () => {
+ const store = withForm({ credentials: { client_id: "typed" } }, (form) => {
+ resetFields(form, ["credentials"]);
+ });
+
+ expect(store).toHaveProperty("credentials", undefined);
+ });
+});
+
+describe("setFieldsValue", () => {
+ it("writes an undefined leaf into the live store, so a transport switch really clears the field", () => {
+ const store = withForm({ url: "https://example.com", command: "npx" }, (form) => {
+ setFieldsValue(form, { url: undefined });
+ });
+
+ expect(store).toStrictEqual({ url: undefined, command: "npx" });
+ });
+
+ it("merges a nested write into the live store instead of replacing the whole object", () => {
+ const store = withForm({ credentials: { client_id: "kept", scopes: ["a"] } }, (form) => {
+ setFieldsValue(form, { credentials: { client_secret: "new" } });
+ });
+
+ expect(store.credentials).toStrictEqual({ client_id: "kept", scopes: ["a"], client_secret: "new" });
+ });
+});
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcpFormStore.ts b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcpFormStore.ts
new file mode 100644
index 00000000000..20df3e32273
--- /dev/null
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcpFormStore.ts
@@ -0,0 +1,107 @@
+import { useWatch } from "react-hook-form";
+import type { Control, UseFormReturn } from "react-hook-form";
+
+import {
+ projectMountedValues,
+ type MountRegistry,
+ type MountedFormValues,
+} from "@/components/common_components/MountedFormField";
+
+export type McpForm = UseFormReturn;
+
+const isPlainObject = (value: unknown): value is Record =>
+ typeof value === "object" && value !== null && Object.getPrototypeOf(value) === Object.prototype;
+
+export const deepMergedFieldsValue = (store: unknown, values: Record): Record =>
+ Object.entries(values).reduce>(
+ (merged, [key, value]) => ({
+ ...merged,
+ [key]: isPlainObject(value) ? deepMergedFieldsValue(merged[key], value) : value,
+ }),
+ isPlainObject(store) ? { ...store } : {},
+ );
+
+export const setFieldsValue = (form: McpForm, values: Record): void => {
+ const merged = deepMergedFieldsValue(form.getValues(), values);
+ Object.keys(values).forEach((key) => form.setValue(key, merged[key]));
+};
+
+export const resetFields = (form: McpForm, names: readonly string[], defaults: MountedFormValues = {}): void => {
+ names.forEach((name) => {
+ form.setValue(name, defaults[name]);
+ form.clearErrors(name);
+ });
+};
+
+const branchAt = (segments: readonly string[], leaf: unknown): unknown => {
+ const [head, ...rest] = segments;
+ if (head === undefined) {
+ return leaf;
+ }
+ const child = branchAt(rest, leaf);
+ if (!/^\d+$/.test(head)) {
+ return { [head]: child };
+ }
+ const index = Number(head);
+ return Array.from({ length: index + 1 }, (_, position) => (position === index ? child : undefined));
+};
+
+export const singleBranchChange = (path: string, values: MountedFormValues): Record => {
+ const segments = path.split(".");
+ const leaf = segments.reduce(
+ (node, segment) => (node === null || node === undefined ? undefined : (node as Record)[segment]),
+ values,
+ );
+ return branchAt(segments, leaf) as Record;
+};
+
+export interface McpStaticHeaderRow {
+ header?: string;
+ value?: string;
+}
+
+export interface McpEnvVarRow {
+ name?: string;
+ value?: string;
+ scope?: string;
+ description?: string;
+}
+
+export interface McpListValues {
+ static_headers: McpStaticHeaderRow[];
+ env_vars: McpEnvVarRow[];
+}
+
+export interface McpFormSnapshot extends MountedFormValues {
+ readonly server_name?: string;
+ readonly alias?: string;
+ readonly description?: string;
+ readonly url?: string;
+ readonly spec_path?: string;
+ readonly transport?: string;
+ readonly auth_type?: string;
+ readonly oauth_flow_type?: string;
+ readonly credentials?: Record;
+ readonly issuer?: string;
+ readonly authorization_url?: string;
+ readonly token_url?: string;
+ readonly registration_url?: string;
+ readonly mcp_access_groups?: string[];
+ readonly static_headers?: readonly McpStaticHeaderRow[];
+ readonly command?: string;
+ readonly args?: string[];
+ readonly env?: Record;
+}
+
+export const allFieldsValue = (form: McpForm): McpFormSnapshot => form.getValues() as McpFormSnapshot;
+
+export const listControl = (control: Control): Control =>
+ control as unknown as Control;
+
+export const mountedPaths = (registry: MountRegistry): readonly string[] =>
+ registry.mountedNames().map((name) => (Array.isArray(name) ? name.join(".") : (name as string)));
+
+export const useMountedValues = (form: McpForm, registry: MountRegistry): MountedFormValues => {
+ useWatch({ control: form.control });
+ return projectMountedValues(registry, form.getValues);
+};
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.integration.test.tsx
new file mode 100644
index 00000000000..a9664191f3f
--- /dev/null
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.integration.test.tsx
@@ -0,0 +1,371 @@
+import React from "react";
+import { describe, it, expect, vi, beforeEach } from "vitest";
+import { render, screen, waitFor, act } from "@testing-library/react";
+
+import MCPServerEdit from "./mcp_server_edit";
+import * as networking from "@/components/networking";
+import { MCPServer } from "@/components/mcp_tools/types";
+
+vi.mock("@/components/networking", () => ({
+ updateMCPServer: vi.fn(),
+ listMCPTools: vi.fn().mockResolvedValue({ tools: [], error: null }),
+ storeMCPOAuthUserCredential: vi.fn().mockResolvedValue({}),
+ testMCPToolsListRequest: vi.fn().mockResolvedValue({ tools: [], error: null }),
+}));
+
+vi.mock("@/hooks/useMcpOAuthFlow", () => ({
+ useMcpOAuthFlow: () => ({
+ startOAuthFlow: vi.fn(),
+ status: "idle",
+ error: null,
+ tokenResponse: null,
+ reset: vi.fn(),
+ }),
+}));
+
+vi.mock("./mcp_server_cost_config", () => ({
+ default: () =>
,
+}));
+
+vi.mock("./mcp_tool_configuration", () => ({
+ default: () =>
,
+}));
+
+const BASE: MCPServer = {
+ server_id: "srv_1",
+ server_name: "srv",
+ alias: "srv_alias",
+ description: "a server",
+ transport: "http",
+ url: "https://example.com/mcp",
+ auth_type: "none",
+ created_at: "2024-01-01T00:00:00Z",
+ created_by: "user-1",
+ updated_at: "2024-01-01T00:00:00Z",
+ updated_by: "user-1",
+ mcp_access_groups: [],
+};
+
+const CREDS = {
+ auth_value: "secret-value",
+ client_id: "cid",
+ client_secret: "csec",
+ scopes: "read write",
+ aws_region_name: "us-east-1",
+ aws_service_name: "bedrock",
+ aws_access_key_id: "AKIA",
+ aws_secret_access_key: "shh",
+ aws_session_token: "tok",
+ aws_role_name: "role",
+ aws_session_name: "sess",
+ id_jag_resource: "api://res",
+ id_jag_resource_token_endpoint: "https://idp/jag",
+ client_private_key: "-----KEY-----",
+ client_private_key_id: "kid",
+ client_assertion_signing_alg: "RS256",
+ token_endpoint_auth_method: "client_secret_basic",
+ upstream_resource: "api://up",
+};
+
+const EXPECTED_BASE: Readonly> = {
+ alias: "srv_alias",
+ allow_all_keys: false,
+ auth_type: "none",
+ available_on_public_internet: true,
+ dcr_bridge: false,
+ delegate_auth_to_upstream: false,
+ description: "a server",
+ disallowed_tools: [],
+ env_json: undefined,
+ env_vars: [],
+ extra_headers: [],
+ max_concurrent_requests: undefined,
+ mcp_access_groups: [],
+ mcp_info: {
+ description: "a server",
+ logo_url: undefined,
+ mcp_server_cost_info: null,
+ server_name: "srv",
+ tool_allowlist_enforced: false,
+ },
+ oauth_passthrough: false,
+ server_id: "srv_1",
+ server_name: "srv",
+ static_headers: {},
+ stdio_config: undefined,
+ tool_name_to_description: null,
+ tool_name_to_display_name: null,
+ transport: "http",
+ url: "https://example.com/mcp",
+};
+
+const without = (keys: readonly string[]): Record =>
+ Object.fromEntries(Object.entries(EXPECTED_BASE).filter(([k]) => !keys.includes(k)));
+
+interface Case {
+ readonly label: string;
+ readonly server: MCPServer;
+ readonly expected: Record;
+}
+
+const CASES: readonly Case[] = [
+ { label: "http + none", server: BASE, expected: { ...EXPECTED_BASE } },
+ {
+ label: "stdio drops auth_type and url and adds the stdio trio",
+ server: { ...BASE, transport: "stdio", url: null, command: "npx", args: ["-y", "pkg"], env: { A: "1" } },
+ expected: { ...without(["auth_type", "url"]), args: ["-y", "pkg"], command: "npx", env: {}, transport: "stdio" },
+ },
+ {
+ label: "openapi swaps url for spec_path",
+ server: { ...BASE, transport: "openapi", url: null, spec_path: "https://example.com/openapi.json" },
+ expected: { ...without(["url"]), spec_path: "https://example.com/openapi.json" },
+ },
+ {
+ label: "api_key sends only auth_value under credentials",
+ server: { ...BASE, auth_type: "api_key", credentials: CREDS },
+ expected: { ...EXPECTED_BASE, auth_type: "api_key", credentials: { auth_value: "secret-value" } },
+ },
+ {
+ label: "bearer_token sends only auth_value under credentials",
+ server: { ...BASE, auth_type: "bearer_token", credentials: CREDS },
+ expected: { ...EXPECTED_BASE, auth_type: "bearer_token", credentials: { auth_value: "secret-value" } },
+ },
+ {
+ label: "basic sends only auth_value under credentials",
+ server: { ...BASE, auth_type: "basic", credentials: CREDS },
+ expected: { ...EXPECTED_BASE, auth_type: "basic", credentials: { auth_value: "secret-value" } },
+ },
+ {
+ label: "oauth2 m2m mounts token_url and omits the interactive endpoints entirely",
+ server: {
+ ...BASE,
+ auth_type: "oauth2",
+ oauth2_flow: "client_credentials",
+ token_url: "https://idp/token",
+ credentials: CREDS,
+ },
+ expected: {
+ ...EXPECTED_BASE,
+ auth_type: "oauth2",
+ credentials: {
+ client_id: "cid",
+ client_secret: "csec",
+ token_endpoint_auth_method: "client_secret_basic",
+ upstream_resource: "api://up",
+ },
+ oauth2_flow: "client_credentials",
+ oauth_flow_type: "m2m",
+ token_url: "https://idp/token",
+ },
+ },
+ {
+ label: "oauth2 interactive mounts the full endpoint set, empty ones present as undefined",
+ server: {
+ ...BASE,
+ auth_type: "oauth2",
+ oauth2_flow: "authorization_code",
+ issuer: "https://idp",
+ credentials: CREDS,
+ },
+ expected: {
+ ...EXPECTED_BASE,
+ auth_type: "oauth2",
+ authorization_url: undefined,
+ credentials: {
+ client_id: "cid",
+ client_secret: "csec",
+ token_endpoint_auth_method: "client_secret_basic",
+ upstream_resource: "api://up",
+ },
+ issuer: "https://idp",
+ oauth2_flow: "authorization_code",
+ oauth_flow_type: "interactive",
+ registration_url: undefined,
+ token_storage_ttl_seconds: undefined,
+ token_url: undefined,
+ },
+ },
+ {
+ label: "token_exchange mounts audience and subject_token_type as undefined",
+ server: {
+ ...BASE,
+ auth_type: "oauth2_token_exchange",
+ token_exchange_endpoint: "https://idp/x",
+ credentials: CREDS,
+ },
+ expected: {
+ ...EXPECTED_BASE,
+ audience: undefined,
+ auth_type: "oauth2_token_exchange",
+ credentials: { client_id: "cid", client_secret: "csec" },
+ subject_token_type: undefined,
+ token_exchange_endpoint: "https://idp/x",
+ token_exchange_profile: undefined,
+ },
+ },
+ {
+ label: "token_exchange entra_obo omits audience and subject_token_type as keys",
+ server: {
+ ...BASE,
+ auth_type: "oauth2_token_exchange",
+ token_exchange_profile: "entra_obo",
+ token_exchange_endpoint: "https://idp/x",
+ credentials: CREDS,
+ },
+ expected: {
+ ...EXPECTED_BASE,
+ auth_type: "oauth2_token_exchange",
+ credentials: { client_id: "cid", client_secret: "csec" },
+ token_exchange_endpoint: "https://idp/x",
+ token_exchange_profile: "entra_obo",
+ },
+ },
+ {
+ label: "id_jag sends its seven credential fields",
+ server: { ...BASE, auth_type: "oauth2_id_jag", credentials: CREDS },
+ expected: {
+ ...EXPECTED_BASE,
+ audience: undefined,
+ auth_type: "oauth2_id_jag",
+ credentials: {
+ client_assertion_signing_alg: "RS256",
+ client_id: "cid",
+ client_private_key: "-----KEY-----",
+ client_private_key_id: "kid",
+ client_secret: "csec",
+ id_jag_resource: "api://res",
+ id_jag_resource_token_endpoint: "https://idp/jag",
+ },
+ subject_token_type: undefined,
+ token_exchange_endpoint: undefined,
+ },
+ },
+ {
+ label: "aws_sigv4 sends its seven credential fields",
+ server: { ...BASE, auth_type: "aws_sigv4", credentials: CREDS },
+ expected: {
+ ...EXPECTED_BASE,
+ auth_type: "aws_sigv4",
+ credentials: {
+ aws_access_key_id: "AKIA",
+ aws_region_name: "us-east-1",
+ aws_role_name: "role",
+ aws_secret_access_key: "shh",
+ aws_service_name: "bedrock",
+ aws_session_name: "sess",
+ aws_session_token: "tok",
+ },
+ },
+ },
+ {
+ label: "true_passthrough sends only the passthrough app credentials",
+ server: { ...BASE, auth_type: "true_passthrough", credentials: CREDS },
+ expected: {
+ ...EXPECTED_BASE,
+ auth_type: "true_passthrough",
+ credentials: { client_id: "cid", client_secret: "csec" },
+ },
+ },
+ {
+ label: "oauth_delegate sends only the passthrough app credentials",
+ server: { ...BASE, auth_type: "oauth_delegate", credentials: CREDS },
+ expected: {
+ ...EXPECTED_BASE,
+ auth_type: "oauth_delegate",
+ credentials: { client_id: "cid", client_secret: "csec" },
+ },
+ },
+ {
+ label: "an Authorization entry survives in extra_headers without enabling passthrough",
+ server: { ...BASE, auth_type: "none", extra_headers: ["Authorization"] },
+ expected: { ...EXPECTED_BASE, extra_headers: ["Authorization"] },
+ },
+ {
+ label: "dcr_bridge stays true for a client-forwarded mode",
+ server: { ...BASE, auth_type: "true_passthrough", dcr_bridge: true },
+ expected: { ...EXPECTED_BASE, auth_type: "true_passthrough", dcr_bridge: true },
+ },
+ {
+ label: "an existing allowed_tools list enforces the allowlist and round-trips",
+ server: { ...BASE, allowed_tools: ["alpha"] },
+ expected: {
+ ...EXPECTED_BASE,
+ allowed_tools: ["alpha"],
+ mcp_info: { ...(EXPECTED_BASE.mcp_info as object), tool_allowlist_enforced: true },
+ },
+ },
+ {
+ label: "no allowed_tools key when the allowlist is not enforced",
+ server: { ...BASE, allowed_tools: [] },
+ expected: { ...EXPECTED_BASE },
+ },
+];
+
+const saveAndCapture = async (server: MCPServer): Promise> => {
+ vi.mocked(networking.updateMCPServer).mockResolvedValue(server as never);
+ render(
+ ,
+ );
+ await act(async () => {
+ screen.getAllByRole("button", { name: "Save Changes" })[0].click();
+ });
+ await waitFor(() => {
+ expect(networking.updateMCPServer).toHaveBeenCalled();
+ });
+ const [, payload] = vi.mocked(networking.updateMCPServer).mock.calls[0];
+ return payload as Record;
+};
+
+describe("mcp_server_edit save payload contract", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+
+ it.each(CASES.map((c) => [c.label, c] as const))("%s", async (_label, testCase) => {
+ const payload = await saveAndCapture(testCase.server);
+ expect(payload).toStrictEqual(testCase.expected);
+ });
+
+ it("never sends a server-owned field the form does not bind", async () => {
+ const payload = await saveAndCapture({
+ ...BASE,
+ status: "healthy",
+ last_health_check: "2024-01-02T00:00:00Z",
+ health_check_error: null,
+ teams: [{ team_id: "t1" }],
+ allowed_tools: ["a"],
+ has_user_credential: true,
+ approval_status: "approved",
+ submitted_by: "someone",
+ submitted_at: "2024-01-01T00:00:00Z",
+ reviewed_at: "2024-01-02T00:00:00Z",
+ review_notes: "looks fine",
+ } as MCPServer);
+
+ for (const leaked of [
+ "created_at",
+ "created_by",
+ "updated_at",
+ "updated_by",
+ "status",
+ "last_health_check",
+ "health_check_error",
+ "teams",
+ "has_user_credential",
+ "approval_status",
+ "submitted_by",
+ "submitted_at",
+ "reviewed_at",
+ "review_notes",
+ ]) {
+ expect(payload).not.toHaveProperty(leaked);
+ }
+ });
+});
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.test.tsx
index 4e85aa41cfc..14e26fa5393 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.test.tsx
@@ -781,7 +781,7 @@ describe("MCPServerEdit (interactive OAuth)", () => {
});
// Note: The M2M flow hiding logic is tested via OAuthFormFields.test.tsx (isM2M prop directly),
- // since Form.useWatch doesn't synchronously reflect initialValues in jsdom.
+ // since a mounted-values read doesn't synchronously reflect the seeded defaults in jsdom.
it("pre-populates token_validation_json from existing server token_validation", async () => {
const tokenValidation = { organization: "my-org", "team.id": "123" };
@@ -1003,7 +1003,7 @@ describe("MCPServerEdit (interactive OAuth)", () => {
fireEvent.click(saveButtons[0]);
});
- // The Form.Item inline validator intercepts invalid JSON before handleSave runs,
+ // The field's inline validator intercepts invalid JSON before handleSave runs,
// so the inline error message appears and updateMCPServer is never called.
await waitFor(() => {
expect(screen.getByText("Must be valid JSON")).toBeInTheDocument();
@@ -2190,7 +2190,7 @@ describe("MCPServerEdit (dcr_bridge toggle)", () => {
});
expect(getDcrToggle()).toHaveAttribute("aria-checked", "true");
- // The Form.Item stays mounted across the two client-forwarded modes, so the live toggle value is
+ // The field stays mounted across the two client-forwarded modes, so the live toggle value is
// preserved rather than forced false by the switch.
await selectAntOption("Authentication", "OAuth Delegate (client-supplied upstream token)");
await waitFor(() => {
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx
index 5c3fe6d8cda..3892b9d37b7 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx
@@ -1,5 +1,6 @@
import React, { useState, useEffect } from "react";
-import { Form, Select, Button as AntdButton, Tooltip, Input, InputNumber, Alert } from "antd";
+import { Select, Button as AntdButton, Tooltip, Input, InputNumber, Alert } from "antd";
+import { FormProvider, useForm } from "react-hook-form";
import { InfoCircleOutlined } from "@ant-design/icons";
import { Button } from "@/components/ui/button";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
@@ -11,7 +12,6 @@ import {
isHeldOAuthTokenStale,
preservedAdminCredentials,
preservedDeclaredAppCredentials,
- ADMIN_CONFIG_CREDENTIAL_KEYS,
withoutMintedTokenCredentials,
OAUTH_FLOW,
MCP_OAUTH2_FLOW_M2M,
@@ -41,15 +41,27 @@ import IdJagFormFields from "./IdJagFormFields";
import OAuthFormFields from "./OAuthFormFields";
import MCPLogoSelector from "./MCPLogoSelector";
import EnvVarsSection from "./EnvVarsSection";
-import {
- validateMCPServerUrl,
- validateMCPServerName,
- normalizeEnvVars,
- normalizeToolOverrideMap,
- TOOL_DISPLAY_NAME_PATTERN,
-} from "./utils";
+import { validateMCPServerUrl, validateMCPServerName, normalizeToolOverrideMap } from "./utils";
+import { EditServerFormValues, buildEditServerPayload, editPayloadErrorMessage } from "./editServerPayload";
import { toast } from "@/lib/toast";
import { useMcpOAuthFlow } from "@/hooks/useMcpOAuthFlow";
+import {
+ MountedFormField,
+ MountedFormProvider,
+ projectMountedValues,
+ useMountRegistry,
+ type MountedFormValues,
+} from "@/components/common_components/MountedFormField";
+import { antdRequired, antdRules } from "@/components/common_components/antdFormRules";
+import {
+ allFieldsValue,
+ mountedPaths,
+ resetFields,
+ setFieldsValue,
+ singleBranchChange,
+ useMountedValues,
+} from "./mcpFormStore";
+import { numberControl, notOnlyWhitespace, parsesAsJsonObject, selectControl, textControl } from "./mcpFieldRules";
import { getSecureItem, setSecureItem } from "@/utils/secureStorage";
interface MCPServerEditProps {
@@ -62,15 +74,6 @@ interface MCPServerEditProps {
}
const AUTH_TYPES_REQUIRING_AUTH_VALUE = [AUTH_TYPE.API_KEY, AUTH_TYPE.BEARER_TOKEN, AUTH_TYPE.TOKEN, AUTH_TYPE.BASIC];
-const AUTH_TYPES_REQUIRING_CREDENTIALS = [
- ...AUTH_TYPES_REQUIRING_AUTH_VALUE,
- AUTH_TYPE.OAUTH2,
- AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE,
- AUTH_TYPE.OAUTH2_ID_JAG,
- AUTH_TYPE.AWS_SIGV4,
- AUTH_TYPE.TRUE_PASSTHROUGH,
- AUTH_TYPE.OAUTH_DELEGATE,
-];
export const EDIT_OAUTH_UI_STATE_KEY = "litellm-mcp-oauth-edit-state";
const MCPServerEdit: React.FC = ({
@@ -81,174 +84,6 @@ const MCPServerEdit: React.FC = ({
onSuccess,
availableAccessGroups,
}) => {
- const [form] = Form.useForm();
- const [costConfig, setCostConfig] = useState({});
- const [tools, setTools] = useState([]);
- const [isLoadingTools, setIsLoadingTools] = useState(false);
- const [toolsError, setToolsError] = useState(null);
- const [searchValue, setSearchValue] = useState("");
- const [aliasManuallyEdited, setAliasManuallyEdited] = useState(false);
- const [removeStoredApp, setRemoveStoredApp] = useState(false);
- // Set when the upstream identity (url/endpoints) changed while a declared app is present, so the
- // section warns that the saved app may not match the new upstream (the app is kept, not wiped).
- const [appMayNotMatchUpstream, setAppMayNotMatchUpstream] = useState(false);
- const [allowedTools, setAllowedTools] = useState([]);
- const [hasToolAllowlistInteraction, setHasToolAllowlistInteraction] = useState(false);
- const [toolNameToDisplayName, setToolNameToDisplayName] = useState>({});
- const [toolNameToDescription, setToolNameToDescription] = useState>({});
- const [pendingRestoredValues, setPendingRestoredValues] = useState | null>(null);
- const [logoUrl, setLogoUrl] = useState(mcpServer.mcp_info?.logo_url || undefined);
- const authType = Form.useWatch("auth_type", form) as string | undefined;
- const transportType = Form.useWatch("transport", form) as string | undefined;
- const isStdioTransport = transportType === "stdio";
- const isOpenAPITransport = transportType === TRANSPORT.OPENAPI;
- const isMCPTransport = !isStdioTransport && !isOpenAPITransport;
- const shouldShowAuthValueField = authType ? AUTH_TYPES_REQUIRING_AUTH_VALUE.includes(authType) : false;
- const isOAuthAuthType = authType === AUTH_TYPE.OAUTH2;
- const isTokenExchangeAuthType = authType === AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE;
- const isIdJagAuthType = authType === AUTH_TYPE.OAUTH2_ID_JAG;
- const isAwsSigV4AuthType = authType === AUTH_TYPE.AWS_SIGV4;
- const oauthFlowTypeValue = Form.useWatch("oauth_flow_type", form) as string | undefined;
- const isM2MFlow = isOAuthAuthType && oauthFlowTypeValue === OAUTH_FLOW.M2M;
- // Watch reflects a live toggle when the delegate switch is mounted; fall back to
- // the stored value otherwise (useWatch returns undefined for an unmounted field,
- // the same trap the oauth_flow_type field originally hit).
- const delegateAuthWatched = Form.useWatch("delegate_auth_to_upstream", form) as boolean | undefined;
- const isDelegateAuth = delegateAuthWatched ?? Boolean(mcpServer.delegate_auth_to_upstream);
-
- // Watch form fields that affect tool fetching
- const currentUrl = Form.useWatch("url", form);
- const currentSpecPath = Form.useWatch("spec_path", form);
- const currentServerName = Form.useWatch("server_name", form);
- const currentAuthType = Form.useWatch("auth_type", form);
- const currentStaticHeaders = Form.useWatch("static_headers", form);
- const currentCredentials = Form.useWatch("credentials", form);
- const currentIssuer = Form.useWatch("issuer", form);
- const currentAuthorizationUrl = Form.useWatch("authorization_url", form);
- const currentTokenUrl = Form.useWatch("token_url", form);
- const currentRegistrationUrl = Form.useWatch("registration_url", form);
- const hasExistingToolAllowlist =
- Boolean(mcpServer.mcp_info?.tool_allowlist_enforced) || (mcpServer.allowed_tools?.length ?? 0) > 0;
- const existingAllowedTools = hasExistingToolAllowlist ? mcpServer.allowed_tools ?? [] : null;
-
- const persistEditUiState = () => {
- if (typeof window === "undefined") {
- return;
- }
- try {
- const values = form.getFieldsValue(true);
- setSecureItem(
- EDIT_OAUTH_UI_STATE_KEY,
- JSON.stringify({
- serverId: mcpServer.server_id,
- formValues: values,
- costConfig,
- allowedTools,
- hasToolAllowlistInteraction,
- searchValue,
- aliasManuallyEdited,
- }),
- );
- } catch (err) {
- console.warn("Failed to persist MCP edit state", err);
- }
- };
-
- // The auth mode every decision must key off: the admin's in-flight form selection wins over the
- // saved record, so authorizing, loading tools, and saving all agree with what the form shows. Paths
- // that read only mcpServer.auth_type go stale the moment the admin switches modes in the form.
- const getEffectiveAuthType = () => form.getFieldValue("auth_type") ?? mcpServer.auth_type;
-
- // The OAuth authorization identity (see getOAuthAuthorizationIdentity) captured when a token is fetched
- // in this edit session; undefined when none is held. If a mint-relevant field later diverges from it,
- // the held token (hook response + sessionStorage) is discarded so the admin must re-authorize.
- const authorizedIdentityRef = React.useRef(undefined);
-
- const {
- startOAuthFlow,
- status: oauthStatus,
- error: oauthError,
- tokenResponse: oauthTokenResponse,
- reset: resetOAuthFlow,
- } = useMcpOAuthFlow({
- accessToken,
- getCredentials: () => form.getFieldValue("credentials"),
- getTemporaryPayload: () => {
- const values = form.getFieldsValue(true);
- const url = values.url || mcpServer.url;
- const transport = values.transport || mcpServer.transport;
- if (!url || !transport) {
- return null;
- }
- const staticHeaders = Array.isArray(values.static_headers)
- ? values.static_headers.reduce((acc: Record, entry: Record) => {
- const header = entry?.header?.trim();
- if (!header) {
- return acc;
- }
- acc[header] = (entry?.value ?? "").trim();
- return acc;
- }, {})
- : ({} as Record);
-
- return {
- server_id: mcpServer.server_id,
- server_name: values.server_name || mcpServer.server_name || mcpServer.alias,
- alias: values.alias || mcpServer.alias,
- description: values.description || mcpServer.description,
- url,
- transport,
- auth_type: isClientForwardedTokenMode(values.auth_type) ? values.auth_type : AUTH_TYPE.OAUTH2,
- credentials: isClientForwardedTokenMode(values.auth_type)
- ? preservedAdminCredentials(values.credentials)
- : values.credentials,
- mcp_access_groups: values.mcp_access_groups || mcpServer.mcp_access_groups,
- static_headers: staticHeaders,
- command: values.command,
- args: values.args,
- env: values.env,
- };
- },
- onTokenReceived: (token) => {
- if (!token?.access_token) {
- return;
- }
-
- authorizedIdentityRef.current = getOAuthAuthorizationIdentity(form.getFieldsValue(true));
- if (isClientForwardedTokenMode(getEffectiveAuthType())) {
- const browserHeldToken = {
- access_token: token.access_token,
- expires_in: token.expires_in,
- token_type: token.token_type,
- };
- setToken(mcpServer.server_id, browserHeldToken, userID);
- toast.success(
- "Token held for this browser session. Tools can now be loaded and configured; the token is not saved to LiteLLM.",
- );
- return;
- }
-
- const current = (form.getFieldValue("credentials") as Record | undefined) ?? {};
- const nextCredentials = {
- ...(preservedAdminCredentials(current) ?? {}),
- ...(current.scopes !== undefined && { scopes: current.scopes }),
- access_token: token.access_token,
- ...(token.refresh_token && { refresh_token: token.refresh_token }),
- ...(token.expires_in && { expires_in: token.expires_in }),
- ...(token.scope && { scope: token.scope }),
- };
- // Path-replace (not deep-merge) so a re-authorize with fewer token fields does not leave stale
- // siblings behind; the admin-typed client keys and scopes are carried explicitly above.
- form.setFieldValue("credentials", nextCredentials);
- // Re-capture after writing credentials so the token is not invalidated by its own credential write.
- authorizedIdentityRef.current = getOAuthAuthorizationIdentity(form.getFieldsValue(true));
-
- toast.success("OAuth authorization successful! Please click 'Update MCP Server' to save the credentials.");
- },
- onBeforeRedirect: persistEditUiState,
- flowSource: "edit",
- });
-
const initialStaticHeaders = React.useMemo(() => {
if (!mcpServer.static_headers) {
return [];
@@ -307,6 +142,176 @@ const MCPServerEdit: React.FC = ({
[mcpServer, effectiveTransport, initialStaticHeaders, initialEnvVars, initialEnvJson],
);
+ const form = useForm({ mode: "onChange", defaultValues: initialValues });
+ const registry = useMountRegistry();
+ const mountedValues = useMountedValues(form, registry);
+ const [costConfig, setCostConfig] = useState({});
+ const [tools, setTools] = useState([]);
+ const [isLoadingTools, setIsLoadingTools] = useState(false);
+ const [toolsError, setToolsError] = useState(null);
+ const [searchValue, setSearchValue] = useState("");
+ const [aliasManuallyEdited, setAliasManuallyEdited] = useState(false);
+ const [removeStoredApp, setRemoveStoredApp] = useState(false);
+ // Set when the upstream identity (url/endpoints) changed while a declared app is present, so the
+ // section warns that the saved app may not match the new upstream (the app is kept, not wiped).
+ const [appMayNotMatchUpstream, setAppMayNotMatchUpstream] = useState(false);
+ const [allowedTools, setAllowedTools] = useState([]);
+ const [hasToolAllowlistInteraction, setHasToolAllowlistInteraction] = useState(false);
+ const [toolNameToDisplayName, setToolNameToDisplayName] = useState>({});
+ const [toolNameToDescription, setToolNameToDescription] = useState>({});
+ const [pendingRestoredValues, setPendingRestoredValues] = useState | null>(null);
+ const [logoUrl, setLogoUrl] = useState(mcpServer.mcp_info?.logo_url || undefined);
+ const authType = mountedValues.auth_type as string | undefined;
+ const transportType = mountedValues.transport as string | undefined;
+ const isStdioTransport = transportType === "stdio";
+ const isOpenAPITransport = transportType === TRANSPORT.OPENAPI;
+ const isMCPTransport = !isStdioTransport && !isOpenAPITransport;
+ const shouldShowAuthValueField = authType ? AUTH_TYPES_REQUIRING_AUTH_VALUE.includes(authType) : false;
+ const isOAuthAuthType = authType === AUTH_TYPE.OAUTH2;
+ const isTokenExchangeAuthType = authType === AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE;
+ const isIdJagAuthType = authType === AUTH_TYPE.OAUTH2_ID_JAG;
+ const isAwsSigV4AuthType = authType === AUTH_TYPE.AWS_SIGV4;
+ const oauthFlowTypeValue = mountedValues.oauth_flow_type as string | undefined;
+ const isM2MFlow = isOAuthAuthType && oauthFlowTypeValue === OAUTH_FLOW.M2M;
+ // Watch reflects a live toggle when the delegate switch is mounted; fall back to
+ // the stored value otherwise (useWatch returns undefined for an unmounted field,
+ // the same trap the oauth_flow_type field originally hit).
+ const delegateAuthWatched = mountedValues.delegate_auth_to_upstream as boolean | undefined;
+ const isDelegateAuth = delegateAuthWatched ?? Boolean(mcpServer.delegate_auth_to_upstream);
+
+ // Watch form fields that affect tool fetching
+ const currentUrl = mountedValues.url;
+ const currentSpecPath = mountedValues.spec_path;
+ const currentServerName = mountedValues.server_name;
+ const currentAuthType = mountedValues.auth_type;
+ const currentStaticHeaders = mountedValues.static_headers;
+ const currentCredentials = mountedValues.credentials;
+ const currentIssuer = mountedValues.issuer;
+ const currentAuthorizationUrl = mountedValues.authorization_url;
+ const currentTokenUrl = mountedValues.token_url;
+ const currentRegistrationUrl = mountedValues.registration_url;
+ const hasExistingToolAllowlist =
+ Boolean(mcpServer.mcp_info?.tool_allowlist_enforced) || (mcpServer.allowed_tools?.length ?? 0) > 0;
+ const existingAllowedTools = hasExistingToolAllowlist ? mcpServer.allowed_tools ?? [] : null;
+
+ const persistEditUiState = () => {
+ if (typeof window === "undefined") {
+ return;
+ }
+ try {
+ const values = allFieldsValue(form);
+ setSecureItem(
+ EDIT_OAUTH_UI_STATE_KEY,
+ JSON.stringify({
+ serverId: mcpServer.server_id,
+ formValues: values,
+ costConfig,
+ allowedTools,
+ hasToolAllowlistInteraction,
+ searchValue,
+ aliasManuallyEdited,
+ }),
+ );
+ } catch (err) {
+ console.warn("Failed to persist MCP edit state", err);
+ }
+ };
+
+ // The auth mode every decision must key off: the admin's in-flight form selection wins over the
+ // saved record, so authorizing, loading tools, and saving all agree with what the form shows. Paths
+ // that read only mcpServer.auth_type go stale the moment the admin switches modes in the form.
+ const getEffectiveAuthType = () => allFieldsValue(form).auth_type ?? mcpServer.auth_type;
+
+ // The OAuth authorization identity (see getOAuthAuthorizationIdentity) captured when a token is fetched
+ // in this edit session; undefined when none is held. If a mint-relevant field later diverges from it,
+ // the held token (hook response + sessionStorage) is discarded so the admin must re-authorize.
+ const authorizedIdentityRef = React.useRef(undefined);
+
+ const {
+ startOAuthFlow,
+ status: oauthStatus,
+ error: oauthError,
+ tokenResponse: oauthTokenResponse,
+ reset: resetOAuthFlow,
+ } = useMcpOAuthFlow({
+ accessToken,
+ getCredentials: () => allFieldsValue(form).credentials,
+ getTemporaryPayload: () => {
+ const values = allFieldsValue(form);
+ const url = values.url || mcpServer.url;
+ const transport = values.transport || mcpServer.transport;
+ if (!url || !transport) {
+ return null;
+ }
+ const staticHeaders = Array.isArray(values.static_headers)
+ ? values.static_headers.reduce((acc: Record, entry: Record) => {
+ const header = entry?.header?.trim();
+ if (!header) {
+ return acc;
+ }
+ acc[header] = (entry?.value ?? "").trim();
+ return acc;
+ }, {})
+ : ({} as Record);
+
+ return {
+ server_id: mcpServer.server_id,
+ server_name: values.server_name || mcpServer.server_name || mcpServer.alias,
+ alias: values.alias || mcpServer.alias,
+ description: values.description || mcpServer.description,
+ url,
+ transport,
+ auth_type: isClientForwardedTokenMode(values.auth_type) ? values.auth_type : AUTH_TYPE.OAUTH2,
+ credentials: isClientForwardedTokenMode(values.auth_type)
+ ? preservedAdminCredentials(values.credentials)
+ : values.credentials,
+ mcp_access_groups: values.mcp_access_groups || mcpServer.mcp_access_groups,
+ static_headers: staticHeaders,
+ command: values.command,
+ args: values.args,
+ env: values.env,
+ };
+ },
+ onTokenReceived: (token) => {
+ if (!token?.access_token) {
+ return;
+ }
+
+ authorizedIdentityRef.current = getOAuthAuthorizationIdentity(allFieldsValue(form));
+ if (isClientForwardedTokenMode(getEffectiveAuthType())) {
+ const browserHeldToken = {
+ access_token: token.access_token,
+ expires_in: token.expires_in,
+ token_type: token.token_type,
+ };
+ setToken(mcpServer.server_id, browserHeldToken, userID);
+ toast.success(
+ "Token held for this browser session. Tools can now be loaded and configured; the token is not saved to LiteLLM.",
+ );
+ return;
+ }
+
+ const current = (allFieldsValue(form).credentials as Record | undefined) ?? {};
+ const nextCredentials = {
+ ...(preservedAdminCredentials(current) ?? {}),
+ ...(current.scopes !== undefined && { scopes: current.scopes }),
+ access_token: token.access_token,
+ ...(token.refresh_token && { refresh_token: token.refresh_token }),
+ ...(token.expires_in && { expires_in: token.expires_in }),
+ ...(token.scope && { scope: token.scope }),
+ };
+ // Path-replace (not deep-merge) so a re-authorize with fewer token fields does not leave stale
+ // siblings behind; the admin-typed client keys and scopes are carried explicitly above.
+ form.setValue("credentials", nextCredentials);
+ // Re-capture after writing credentials so the token is not invalidated by its own credential write.
+ authorizedIdentityRef.current = getOAuthAuthorizationIdentity(allFieldsValue(form));
+
+ toast.success("OAuth authorization successful! Please click 'Update MCP Server' to save the credentials.");
+ },
+ onBeforeRedirect: persistEditUiState,
+ flowSource: "edit",
+ });
+
// antd applies `initialValues` only at first mount. When the server loads after
// mount (e.g. returning from the OAuth redirect lands on Overview and the form
// mounts before the server data is ready), the form would stay blank. Re-sync it
@@ -318,7 +323,7 @@ const MCPServerEdit: React.FC = ({
return;
}
syncedServerIdRef.current = mcpServer.server_id;
- form.setFieldsValue(initialValues);
+ setFieldsValue(form, initialValues);
// Reset per-server OAuth UI state so it never carries across a server switch without an unmount: a
// stale removeStoredApp would send an explicit-null credential write that deletes the new server's
// stored app, and a stale warning would show on a server whose upstream did not change.
@@ -409,11 +414,11 @@ const MCPServerEdit: React.FC = ({
// on the re-run triggered by the transportType watch (without it the effect's
// deps never change and the second pass never runs, leaving fields blank).
const transport = pendingRestoredValues.transport || mcpServer.transport;
- if (transport && transport !== form.getFieldValue("transport")) {
- form.setFieldsValue({ transport });
+ if (transport && transport !== allFieldsValue(form).transport) {
+ setFieldsValue(form, { transport });
return;
}
- form.setFieldsValue(pendingRestoredValues);
+ setFieldsValue(form, pendingRestoredValues);
setPendingRestoredValues(null);
}, [pendingRestoredValues, form, mcpServer.transport, transportType]);
@@ -422,7 +427,7 @@ const MCPServerEdit: React.FC = ({
if (mcpServer.mcp_access_groups) {
// If access groups are objects, extract the name property; if strings, use as is
const groupNames = mcpServer.mcp_access_groups.map((g: any) => (typeof g === "string" ? g : g.name || String(g)));
- form.setFieldValue("mcp_access_groups", groupNames);
+ form.setValue("mcp_access_groups", groupNames);
}
}, [mcpServer]);
@@ -454,16 +459,16 @@ const MCPServerEdit: React.FC = ({
resetOAuthFlow();
// The admin-typed app is upstream-scoped config, not minted material, so it survives every
// invalidation; only the held token is discarded. Token-shaped keys are excluded by the filter.
- const keptAdminCredentials = preservedAdminCredentials(form.getFieldValue("credentials"));
- form.resetFields([...CLEARED_ON_INVALIDATION]);
+ const keptAdminCredentials = preservedAdminCredentials(allFieldsValue(form).credentials);
+ resetFields(form, [...CLEARED_ON_INVALIDATION], initialValues as MountedFormValues);
if (keptAdminCredentials) {
- form.setFieldsValue({ credentials: keptAdminCredentials });
+ setFieldsValue(form, { credentials: keptAdminCredentials });
}
const preserved = Object.fromEntries(
CLEARED_ON_INVALIDATION.filter((key) => key in changedValues).map((key) => [key, changedValues[key]]),
);
if (Object.keys(preserved).length > 0) {
- form.setFieldsValue(preserved);
+ setFieldsValue(form, preserved);
}
};
@@ -478,12 +483,12 @@ const MCPServerEdit: React.FC = ({
const upstreamChanged = ["url", "spec_path", "issuer", "authorization_url", "token_url", "registration_url"].some(
(key) => key in changedValues,
);
- const hasDeclaredApp = preservedDeclaredAppCredentials(form.getFieldValue("credentials")) !== undefined;
+ const hasDeclaredApp = preservedDeclaredAppCredentials(allFieldsValue(form).credentials) !== undefined;
if (upstreamChanged && hasDeclaredApp) {
setAppMayNotMatchUpstream(true);
}
}
- if (isHeldOAuthTokenStale(form.getFieldsValue(true), authorizedIdentityRef.current)) {
+ if (isHeldOAuthTokenStale(allFieldsValue(form), authorizedIdentityRef.current)) {
clearHeldOAuthToken(changedValues);
}
};
@@ -507,7 +512,7 @@ const MCPServerEdit: React.FC = ({
setIsLoadingTools(true);
setToolsError(null);
try {
- const values = form.getFieldsValue(true);
+ const values = allFieldsValue(form);
const rawTransport = values.transport || mcpServer.transport;
// oauth2_flow must be explicit: the preview endpoint infers client_credentials from the
// inherited client_id/client_secret/token_url (common once DCR or discovery filled them) and
@@ -646,7 +651,7 @@ const MCPServerEdit: React.FC = ({
token_url: undefined,
registration_url: undefined,
};
- form.setFieldsValue(clearedForStdio);
+ setFieldsValue(form, clearedForStdio);
} else if (value === TRANSPORT.OPENAPI) {
const clearedForOpenapi = {
url: undefined,
@@ -655,9 +660,9 @@ const MCPServerEdit: React.FC = ({
env_json: undefined,
stdio_config: undefined,
};
- form.setFieldsValue(clearedForOpenapi);
+ setFieldsValue(form, clearedForOpenapi);
} else {
- form.setFieldsValue({
+ setFieldsValue(form, {
spec_path: undefined,
command: undefined,
args: undefined,
@@ -665,289 +670,51 @@ const MCPServerEdit: React.FC = ({
stdio_config: undefined,
});
}
- if (isHeldOAuthTokenStale(form.getFieldsValue(true), authorizedIdentityRef.current)) {
+ if (isHeldOAuthTokenStale(allFieldsValue(form), authorizedIdentityRef.current)) {
clearHeldOAuthToken();
}
};
- const handleSave = async (values: Record) => {
- if (!accessToken) return;
- const invalidDisplayName = Object.entries(toolNameToDisplayName).find(
- ([, displayName]) => displayName && !TOOL_DISPLAY_NAME_PATTERN.test(displayName),
- );
- if (invalidDisplayName) {
- toast.fromError(
- `Tool display name "${invalidDisplayName[1]}" is invalid. Only letters, digits, underscores, and hyphens are allowed (no spaces).`,
- );
+ const valuesChangeRef = React.useRef(handleFormValuesChange);
+ valuesChangeRef.current = handleFormValuesChange;
+
+ React.useEffect(() => {
+ const subscription = form.watch((values, { name, type }) => {
+ if (type !== "change" || name === undefined) {
+ return;
+ }
+ valuesChangeRef.current(singleBranchChange(name, values as MountedFormValues));
+ });
+ return () => subscription.unsubscribe();
+ }, [form]);
+
+ const submitForm = async () => {
+ const isValid = await form.trigger(mountedPaths(registry) as string[]);
+ if (!isValid) {
return;
}
+ await handleSave(projectMountedValues(registry, form.getValues) as unknown as EditServerFormValues);
+ };
+
+ const handleSave = async (values: EditServerFormValues) => {
+ if (!accessToken) return;
try {
- // Ensure access groups is always a string array
- const {
- static_headers: staticHeadersList,
- env_vars: envVarsList,
- credentials: credentialValues,
- stdio_config: rawStdioConfig,
- env_json: rawEnvJson,
- command: rawCommand,
- args: rawArgs,
- allow_all_keys: allowAllKeysRaw,
- available_on_public_internet: availableOnPublicInternetRaw,
- delegate_auth_to_upstream: delegateAuthToUpstreamRaw,
- oauth_passthrough: oauthPassthroughRaw,
- dcr_bridge: dcrBridgeRaw,
- token_validation_json: rawTokenValidationJson,
- ...restValues
- } = values;
-
- const accessGroups = (restValues.mcp_access_groups || []).map((g: any) =>
- typeof g === "string" ? g : g.name || String(g),
- );
-
- const staticHeaders = Array.isArray(staticHeadersList)
- ? staticHeadersList.reduce((acc: Record, entry: Record) => {
- const header = entry?.header?.trim();
- if (!header) {
- return acc;
- }
- acc[header] = (entry?.value ?? "").trim();
- return acc;
- }, {})
- : ({} as Record);
-
- const envVars = normalizeEnvVars(envVarsList);
-
- const credentialsPayload =
- credentialValues && typeof credentialValues === "object"
- ? Object.entries(credentialValues).reduce((acc: Record, [key, value]) => {
- if (value === undefined || value === null || value === "") {
- if (value === "" && (ADMIN_CONFIG_CREDENTIAL_KEYS as readonly string[]).includes(key)) {
- acc[key] = null;
- }
- return acc;
- }
- if (key === "scopes") {
- if (Array.isArray(value)) {
- const filteredScopes = value.filter((scope) => scope != null && scope !== "");
- if (filteredScopes.length > 0) {
- acc[key] = filteredScopes;
- }
- }
- } else {
- acc[key] = value;
- }
- return acc;
- }, {})
- : undefined;
-
- let stdioFields: Record = {};
-
- if (restValues.transport === "stdio") {
- // Prefer JSON config if provided (matches Create screen behavior)
- if (rawStdioConfig) {
- try {
- const stdioConfig = JSON.parse(rawStdioConfig);
-
- let actualConfig = stdioConfig;
- if (stdioConfig?.mcpServers && typeof stdioConfig.mcpServers === "object") {
- const serverNames = Object.keys(stdioConfig.mcpServers);
- if (serverNames.length > 0) {
- actualConfig = stdioConfig.mcpServers[serverNames[0]];
- }
- }
-
- const parsedArgs = Array.isArray(actualConfig?.args)
- ? actualConfig.args.map((v: any) => String(v)).filter((v: string) => v.trim() !== "")
- : [];
-
- const parsedEnv =
- actualConfig?.env && typeof actualConfig.env === "object" && !Array.isArray(actualConfig.env)
- ? Object.entries(actualConfig.env).reduce((acc: Record, [k, v]) => {
- if (k == null || String(k).trim() === "") return acc;
- acc[String(k)] = v == null ? "" : String(v);
- return acc;
- }, {})
- : {};
-
- stdioFields = {
- command: actualConfig?.command ? String(actualConfig.command) : undefined,
- args: parsedArgs,
- env: parsedEnv,
- };
-
- if (!stdioFields.command) {
- toast.fromError("Stdio configuration must include a command");
- return;
- }
- } catch {
- toast.fromError("Invalid JSON in stdio configuration");
- return;
- }
- } else {
- // Dedicated fields path (command/args + env JSON)
- let parsedEnv: Record = {};
- if (rawEnvJson) {
- try {
- const env = JSON.parse(rawEnvJson);
- if (env && typeof env === "object" && !Array.isArray(env)) {
- parsedEnv = Object.entries(env).reduce((acc: Record, [k, v]) => {
- if (k == null || String(k).trim() === "") return acc;
- acc[String(k)] = v == null ? "" : String(v);
- return acc;
- }, {});
- }
- } catch {
- toast.fromError("Invalid JSON in stdio env configuration");
- return;
- }
- }
- const parsedArgs = Array.isArray(rawArgs)
- ? rawArgs.map((v: any) => String(v)).filter((v: string) => v.trim() !== "")
- : [];
-
- const parsedCommand = rawCommand ? String(rawCommand).trim() : "";
- if (!parsedCommand) {
- toast.fromError("Stdio transport requires a command");
- return;
- }
-
- stdioFields = {
- command: parsedCommand,
- args: parsedArgs,
- env: parsedEnv,
- };
- }
- }
-
- // Map "openapi" transport to "http" for the backend
- if (restValues.transport === TRANSPORT.OPENAPI) {
- restValues.transport = "http";
- }
-
- // Parse token_validation JSON if provided
- let tokenValidation: Record | null = null;
- if (rawTokenValidationJson && rawTokenValidationJson.trim() !== "") {
- try {
- tokenValidation = JSON.parse(rawTokenValidationJson);
- } catch {
- toast.fromError("Invalid JSON in Token Validation Rules");
- return;
- }
- }
-
- // Prepare the payload with cost configuration and permission fields
- const mcpInfoServerName =
- restValues.server_name ||
- restValues.url ||
- mcpServer.server_name ||
- mcpServer.url ||
- restValues.alias ||
- mcpServer.alias ||
- "unknown";
-
- const toolAllowlistEnforced = hasExistingToolAllowlist || hasToolAllowlistInteraction || allowedTools.length > 0;
-
- const payload: Record = {
- ...restValues,
- ...stdioFields,
- // Remove UI-only fields
- stdio_config: undefined,
- env_json: undefined,
- ...(mcpServer.auth_type === AUTH_TYPE.OAUTH2 && restValues.auth_type !== AUTH_TYPE.OAUTH2
- ? { issuer: null, authorization_url: null, token_url: null, registration_url: null }
- : {}),
- ...(mcpServer.auth_type === AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE &&
- restValues.auth_type !== AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE
- ? { token_exchange_endpoint: null, audience: null, subject_token_type: null, token_exchange_profile: null }
- : {}),
- server_id: mcpServer.server_id,
- mcp_info: {
- ...(mcpServer.mcp_info ?? {}),
- server_name: mcpInfoServerName,
- description: restValues.description,
- logo_url: logoUrl || undefined,
- mcp_server_cost_info: Object.keys(costConfig).length > 0 ? costConfig : null,
- tool_allowlist_enforced: toolAllowlistEnforced,
- },
- mcp_access_groups: accessGroups,
- alias: restValues.alias,
- // Include permission management fields
- extra_headers: restValues.extra_headers || [],
- ...(toolAllowlistEnforced
- ? {
- allowed_tools: allowedTools,
- }
- : {}),
- tool_name_to_display_name: Object.keys(toolNameToDisplayName).length > 0 ? toolNameToDisplayName : null,
- tool_name_to_description: Object.keys(toolNameToDescription).length > 0 ? toolNameToDescription : null,
- disallowed_tools: restValues.disallowed_tools || [],
- static_headers: staticHeaders,
- env_vars: envVars,
- allow_all_keys: Boolean(allowAllKeysRaw ?? mcpServer.allow_all_keys),
- available_on_public_internet: Boolean(availableOnPublicInternetRaw ?? mcpServer.available_on_public_internet),
- // ``delegate_auth_to_upstream`` is only honored server-side for
- // ``auth_type=oauth2`` (PKCE passthrough). The Form.Item is
- // conditionally rendered so the value drops out of the form on
- // auth_type change; force false for any other configuration to avoid
- // persisting a stale ``true`` that would silently re-activate if the
- // configuration is later switched back.
- delegate_auth_to_upstream: (() => {
- const isOauth2 = restValues.auth_type === AUTH_TYPE.OAUTH2;
- return isOauth2 ? Boolean(delegateAuthToUpstreamRaw ?? mcpServer.delegate_auth_to_upstream) : false;
- })(),
- // ``oauth_passthrough`` is the dedicated, non-oauth2 opt-in. It is only
- // honored for ``auth_type=none`` servers that forward ``Authorization``
- // upstream. Kept separate from ``delegate_auth_to_upstream`` so enabling
- // pass-through never regresses oauth2 servers. Force false otherwise.
- oauth_passthrough: (() => {
- const isNoneAuth = restValues.auth_type === AUTH_TYPE.NONE || restValues.auth_type == null;
- const extraHeaders = Array.isArray(restValues.extra_headers) ? restValues.extra_headers : [];
- const hasAuthorizationHeader = extraHeaders.some(
- (h: unknown) => typeof h === "string" && h.toLowerCase() === "authorization",
- );
- return isNoneAuth && hasAuthorizationHeader
- ? Boolean(oauthPassthroughRaw ?? mcpServer.oauth_passthrough)
- : false;
- })(),
- // ``dcr_bridge`` is only meaningful for the client-forwarded token
- // modes (true_passthrough / oauth_delegate). The Form.Item is
- // conditionally rendered so the value drops out of the form on
- // auth_type change; force false for any other configuration to avoid
- // persisting a stale ``true`` that would silently re-activate if the
- // mode is later switched back.
- dcr_bridge: isClientForwardedTokenMode(restValues.auth_type)
- ? Boolean(dcrBridgeRaw ?? mcpServer.dcr_bridge)
- : false,
- ...(restValues.auth_type === AUTH_TYPE.OAUTH2 && restValues.oauth_flow_type
- ? {
- oauth2_flow:
- restValues.oauth_flow_type === OAUTH_FLOW.M2M ? MCP_OAUTH2_FLOW_M2M : MCP_OAUTH2_FLOW_INTERACTIVE,
- }
- : {}),
- // Include token_validation when it is set (non-null) or when clearing an existing value
- ...(tokenValidation !== null || mcpServer.token_validation ? { token_validation: tokenValidation } : {}),
- };
-
- const includeCredentials =
- restValues.auth_type && AUTH_TYPES_REQUIRING_CREDENTIALS.includes(restValues.auth_type);
-
- // Client-forwarded rows persist ONLY the declared app; strip any token material lingering in the
- // form (e.g. from a prior oauth2 authorize this session) so it can never reach the row.
- const submitCredentials = isClientForwardedTokenMode(restValues.auth_type)
- ? preservedAdminCredentials(credentialsPayload)
- : credentialsPayload;
-
- if (includeCredentials && submitCredentials && Object.keys(submitCredentials).length > 0) {
- payload.credentials = submitCredentials;
- }
-
- // Explicit removal of a saved app for the client-forwarded modes, applied AFTER the filter so it
- // always wins. Blank fields are the keep-existing convention (the backend merges partial
- // credential updates), so removal must be an explicit-null write: encrypt skips nulls and the
- // merge overrides the stored keys, returning the server to dynamic client registration.
- if (removeStoredApp && isClientForwardedTokenMode(restValues.auth_type)) {
- payload.credentials = { client_id: null, client_secret: null };
+ const built = buildEditServerPayload(values, {
+ mcpServer,
+ logoUrl,
+ costConfig,
+ allowedTools,
+ hasExistingToolAllowlist,
+ hasToolAllowlistInteraction,
+ toolNameToDisplayName,
+ toolNameToDescription,
+ removeStoredApp,
+ });
+ if (built.kind !== "ok") {
+ toast.fromError(editPayloadErrorMessage(built));
+ return;
}
+ const payload = built.payload;
const updated = await updateMCPServer(accessToken, payload);
@@ -957,9 +724,9 @@ const MCPServerEdit: React.FC = ({
// never in the server row. M2M/static auth resolve server-side and need neither.
if (oauthTokenResponse?.access_token) {
const oauthMode = getMcpOAuthMode({
- auth_type: restValues.auth_type,
+ auth_type: values.auth_type,
oauth2_flow: isM2MFlow ? MCP_OAUTH2_FLOW_M2M : null,
- delegate_auth_to_upstream: Boolean(delegateAuthToUpstreamRaw ?? mcpServer.delegate_auth_to_upstream),
+ delegate_auth_to_upstream: Boolean(values.delegate_auth_to_upstream ?? mcpServer.delegate_auth_to_upstream),
});
try {
if (oauthMode === "authorization_code") {
@@ -971,7 +738,7 @@ const MCPServerEdit: React.FC = ({
scopes: typeof scope === "string" && scope ? scope.split(" ") : undefined,
};
await storeMCPOAuthUserCredential(accessToken, mcpServer.server_id, oauthCredentialPayload);
- } else if (oauthMode === "passthrough" || isClientForwardedTokenMode(restValues.auth_type)) {
+ } else if (oauthMode === "passthrough" || isClientForwardedTokenMode(values.auth_type)) {
const browserHeldToken = {
access_token: oauthTokenResponse.access_token,
expires_in: oauthTokenResponse.expires_in,
@@ -1006,452 +773,510 @@ const MCPServerEdit: React.FC = ({
-
- validateMCPServerName(value),
- },
- ]}
- >
-
-
- validateMCPServerName(value),
- },
- ]}
- >
- setAliasManuallyEdited(true)}
- className="rounded-lg border-gray-300 focus:border-blue-500 focus:ring-blue-500"
- />
-
-
-
-
-
-
-
- Streamable HTTP (Recommended)
- Server-Sent Events (SSE)
- Standard Input/Output (stdio)
- OpenAPI Spec
-
-
-
- {/* URL field - only for HTTP/SSE */}
- {isMCPTransport && (
- validateMCPServerUrl(value) },
- ]}
- >
-
-
- )}
-
- {/* OpenAPI Spec URL - only for OpenAPI transport */}
- {isOpenAPITransport && (
-
- OpenAPI Spec URL
-
-
-
-
- }
- name="spec_path"
- rules={[{ required: true, message: "Please enter an OpenAPI spec URL" }]}
- >
-
-
- )}
-
-
- Max Concurrent Requests (optional)
-
-
-
-
- }
- name="max_concurrent_requests"
- >
-
-
-
- {/* Authentication - for HTTP, SSE, and OpenAPI */}
- {!isStdioTransport && (
- <>
-
-
- None
- API Key
- Bearer Token
- Token
- Basic Auth
- OAuth
- OAuth Token Exchange (OBO)
- ID-JAG (Okta Cross App Access)
- AWS SigV4 (Bedrock AgentCore MCPs)
- True Passthrough (no LiteLLM auth)
-
- OAuth Delegate (client-supplied upstream token)
-
-
-
-
-
- >
- )}
-
- {isStdioTransport && (
-
-
- Configure the stdio transport used to launch the MCP server process. You can either fill in the fields
- below or paste a JSON configuration.
-
-
-
-
-
-
-
-
-
-
-
{
- if (!value) return Promise.resolve();
- try {
- const parsed = JSON.parse(value);
- if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
- return Promise.resolve();
- }
- return Promise.reject(new Error("Env must be a JSON object"));
- } catch {
- return Promise.reject(new Error("Please enter valid JSON"));
- }
- },
- },
- ]}
- >
-
-
-
- {/* Optional JSON config (if provided, it overrides command/args/env on save) */}
-
-
- )}
-
- {!isStdioTransport && shouldShowAuthValueField && (
-
- Authentication Value
-
-
-
-
- }
- name={["credentials", "auth_value"]}
- rules={[
- {
- validator: (_, value) =>
- value && typeof value === "string" && value.trim() === ""
- ? Promise.reject(new Error("Authentication value cannot be empty"))
- : Promise.resolve(),
- },
- ]}
- >
-
-
- )}
-
- {!isStdioTransport && isOAuthAuthType && (
- <>
- {!oauthFlowTypeValue && !isDelegateAuth && (
-
- )}
-
- >
- )}
-
- {!isStdioTransport && isTokenExchangeAuthType && }
-
- {!isStdioTransport && isIdJagAuthType && }
-
- {!isStdioTransport && isAwsSigV4AuthType && (
- <>
-
- For MCP servers hosted on AWS Bedrock AgentCore.{" "}
-
- View docs →
-
-
-
- AWS Region
-
-
-
-
- }
- name={["credentials", "aws_region_name"]}
- rules={[]}
- >
-
-
-
- AWS Service Name
-
-
-
-
- }
- name={["credentials", "aws_service_name"]}
- >
-
-
-
- AWS Access Key ID
-
-
-
-
- }
- name={["credentials", "aws_access_key_id"]}
- rules={[]}
- >
-
-
-
- AWS Secret Access Key
-
-
-
-
- }
- name={["credentials", "aws_secret_access_key"]}
- rules={[]}
- >
-
-
-
- AWS Session Token
-
-
-
-
- }
- name={["credentials", "aws_session_token"]}
- >
-
-
-
- AWS Role ARN
-
-
-
-
- }
- name={["credentials", "aws_role_name"]}
- >
-
-
-
- AWS Session Name
-
-
-
-
- }
- name={["credentials", "aws_session_name"]}
- >
-
-
- >
- )}
-
- {/* Environment Variables Section */}
-
-
-
-
- {/* Permission Management / Access Control Section */}
-
-
-
-
- {/* Tool Configuration Section */}
-
-
+
+ {
+ event.preventDefault();
+ void submitForm();
}}
- allowedTools={allowedTools}
- existingAllowedTools={existingAllowedTools}
- hasToolAllowlistInteraction={hasToolAllowlistInteraction}
- isEditMode
- onAllowedToolsChange={setAllowedTools}
- onToolAllowlistInteraction={() => setHasToolAllowlistInteraction(true)}
- toolNameToDisplayName={toolNameToDisplayName}
- toolNameToDescription={toolNameToDescription}
- onToolNameToDisplayNameChange={setToolNameToDisplayName}
- onToolNameToDescriptionChange={setToolNameToDescription}
- externalTools={tools}
- externalIsLoading={isLoadingTools}
- externalError={toolsError}
- externalCanFetch={true}
- />
-
+ >
+ validateMCPServerName(value) }) }}
+ >
+ {(control) => (
+
+ )}
+
+ validateMCPServerName(value) }) }}
+ >
+ {(control) => (
+ {
+ control.onChange(event);
+ setAliasManuallyEdited(true);
+ }}
+ className="rounded-lg border-gray-300 focus:border-blue-500 focus:ring-blue-500"
+ />
+ )}
+
+
+ {(control) => (
+
+ )}
+
+
+
+ {(control) => (
+ (control)}
+ onChange={(value: string) => {
+ control.onChange(value);
+ handleTransportChange(value);
+ }}
+ >
+ Streamable HTTP (Recommended)
+ Server-Sent Events (SSE)
+ Standard Input/Output (stdio)
+ OpenAPI Spec
+
+ )}
+
-
-
Cancel
-
Save Changes
-
-
+ {/* URL field - only for HTTP/SSE */}
+ {isMCPTransport && (
+ validateMCPServerUrl(value) }),
+ },
+ }}
+ >
+ {(control) => (
+
+ )}
+
+ )}
+
+ {/* OpenAPI Spec URL - only for OpenAPI transport */}
+ {isOpenAPITransport && (
+
+ OpenAPI Spec URL
+
+
+
+
+ }
+ name="spec_path"
+ required
+ rules={{ validate: { required: antdRequired("Please enter an OpenAPI spec URL") } }}
+ >
+ {(control) => (
+
+ )}
+
+ )}
+
+
+ Max Concurrent Requests (optional)
+
+
+
+
+ }
+ name="max_concurrent_requests"
+ >
+ {(control) => (
+
+ )}
+
+
+ {/* Authentication - for HTTP, SSE, and OpenAPI */}
+ {!isStdioTransport && (
+ <>
+
+ {(control) => (
+ (control)} virtual={false}>
+ None
+ API Key
+ Bearer Token
+ Token
+ Basic Auth
+ OAuth
+ OAuth Token Exchange (OBO)
+ ID-JAG (Okta Cross App Access)
+ AWS SigV4 (Bedrock AgentCore MCPs)
+ True Passthrough (no LiteLLM auth)
+
+ OAuth Delegate (client-supplied upstream token)
+
+
+ )}
+
+
+
+ >
+ )}
+
+ {isStdioTransport && (
+
+
+ Configure the stdio transport used to launch the MCP server process. You can either fill in the
+ fields below or paste a JSON configuration.
+
+
+
+ {(control) => (
+
+ )}
+
+
+
+ {(control) => (
+ (control)}
+ mode="tags"
+ size="large"
+ tokenSeparators={[","]}
+ placeholder="Add args (press enter or comma)"
+ className="rounded-lg"
+ />
+ )}
+
+
+
+ {(control) => (
+
+ )}
+
+
+ {/* Optional JSON config (if provided, it overrides command/args/env on save) */}
+
+
+ )}
+
+ {!isStdioTransport && shouldShowAuthValueField && (
+
+ Authentication Value
+
+
+
+
+ }
+ name={["credentials", "auth_value"]}
+ rules={{ validate: { notWhitespace: notOnlyWhitespace("Authentication value cannot be empty") } }}
+ >
+ {(control) => (
+
+ )}
+
+ )}
+
+ {!isStdioTransport && isOAuthAuthType && (
+ <>
+ {!oauthFlowTypeValue && !isDelegateAuth && (
+
+ )}
+
+ >
+ )}
+
+ {!isStdioTransport && isTokenExchangeAuthType && }
+
+ {!isStdioTransport && isIdJagAuthType && }
+
+ {!isStdioTransport && isAwsSigV4AuthType && (
+ <>
+
+ For MCP servers hosted on AWS Bedrock AgentCore.{" "}
+
+ View docs →
+
+
+
+ AWS Region
+
+
+
+
+ }
+ name={["credentials", "aws_region_name"]}
+ >
+ {(control) => (
+
+ )}
+
+
+ AWS Service Name
+
+
+
+
+ }
+ name={["credentials", "aws_service_name"]}
+ >
+ {(control) => (
+
+ )}
+
+
+ AWS Access Key ID
+
+
+
+
+ }
+ name={["credentials", "aws_access_key_id"]}
+ >
+ {(control) => (
+
+ )}
+
+
+ AWS Secret Access Key
+
+
+
+
+ }
+ name={["credentials", "aws_secret_access_key"]}
+ >
+ {(control) => (
+
+ )}
+
+
+ AWS Session Token
+
+
+
+
+ }
+ name={["credentials", "aws_session_token"]}
+ >
+ {(control) => (
+
+ )}
+
+
+ AWS Role ARN
+
+
+
+
+ }
+ name={["credentials", "aws_role_name"]}
+ >
+ {(control) => (
+
+ )}
+
+
+ AWS Session Name
+
+
+
+
+ }
+ name={["credentials", "aws_session_name"]}
+ >
+ {(control) => (
+
+ )}
+
+ >
+ )}
+
+ {/* Environment Variables Section */}
+
+
+
+
+ {/* Permission Management / Access Control Section */}
+
+
+
+
+ {/* Tool Configuration Section */}
+
+ setHasToolAllowlistInteraction(true)}
+ toolNameToDisplayName={toolNameToDisplayName}
+ toolNameToDescription={toolNameToDescription}
+ onToolNameToDisplayNameChange={setToolNameToDisplayName}
+ onToolNameToDescriptionChange={setToolNameToDescription}
+ externalTools={tools}
+ externalIsLoading={isLoadingTools}
+ externalError={toolsError}
+ externalCanFetch={true}
+ />
+
+
+
+
Cancel
+
Save Changes
+
+
+
+
@@ -1460,7 +1285,7 @@ const MCPServerEdit: React.FC = ({
Cancel
-
form.submit()}>Save Changes
+
void submitForm()}>Save Changes
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mountedServerFields.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mountedServerFields.test.ts
new file mode 100644
index 00000000000..dd9c8db6d30
--- /dev/null
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mountedServerFields.test.ts
@@ -0,0 +1,497 @@
+import { describe, expect, it } from "vitest";
+import {
+ mountedCreateFieldNames,
+ mountedEditFieldNames,
+ projectMountedCreateValues,
+ projectMountedEditValues,
+} from "./mountedServerFields";
+
+const editRoot = (values: Record) => mountedEditFieldNames(values).root;
+const editCreds = (values: Record) => mountedEditFieldNames(values).credentials;
+const createRoot = (values: Record) => mountedCreateFieldNames(values).root;
+const createCreds = (values: Record) => mountedCreateFieldNames(values).credentials;
+
+const HTTP_NONE = { transport: "http", auth_type: "none" };
+
+describe("edit root: transport gates", () => {
+ it("mounts url for http but not spec_path or the stdio group", () => {
+ const root = editRoot(HTTP_NONE);
+ expect(root).toContain("url");
+ expect(root).not.toContain("spec_path");
+ expect(root).not.toContain("command");
+ expect(root).not.toContain("stdio_config");
+ });
+
+ it("mounts url for sse", () => {
+ expect(editRoot({ transport: "sse", auth_type: "none" })).toContain("url");
+ });
+
+ it("mounts spec_path and not url for openapi", () => {
+ const root = editRoot({ transport: "openapi", auth_type: "none" });
+ expect(root).toContain("spec_path");
+ expect(root).not.toContain("url");
+ });
+
+ it("swaps the whole auth subtree for the stdio group on stdio", () => {
+ const root = editRoot({ transport: "stdio", auth_type: "oauth2" });
+ expect(root).toStrictEqual([
+ "server_name",
+ "alias",
+ "description",
+ "transport",
+ "max_concurrent_requests",
+ "command",
+ "args",
+ "env_json",
+ "stdio_config",
+ "env_vars",
+ "allow_all_keys",
+ "available_on_public_internet",
+ "mcp_access_groups",
+ "extra_headers",
+ "static_headers",
+ ]);
+ });
+
+ it("drops every credential on stdio even when auth_type is stored as oauth2", () => {
+ expect(editCreds({ transport: "stdio", auth_type: "oauth2" })).toStrictEqual([]);
+ });
+});
+
+describe("edit root: auth_type gates", () => {
+ it("mounts credentials.auth_value only for the four value-bearing auth types", () => {
+ for (const authType of ["api_key", "bearer_token", "token", "basic"]) {
+ expect(editCreds({ transport: "http", auth_type: authType })).toStrictEqual(["auth_value"]);
+ }
+ expect(editCreds(HTTP_NONE)).toStrictEqual([]);
+ });
+
+ it("swaps the oauth2 endpoint set on the M2M flow", () => {
+ const m2m = editRoot({ transport: "http", auth_type: "oauth2", oauth_flow_type: "m2m" });
+ const interactive = editRoot({ transport: "http", auth_type: "oauth2", oauth_flow_type: "interactive" });
+ expect(m2m).toContain("token_url");
+ expect(m2m).not.toContain("issuer");
+ expect(m2m).not.toContain("registration_url");
+ expect(interactive).toContain("issuer");
+ expect(interactive).toContain("registration_url");
+ });
+
+ it("mounts token_validation_json ONLY on the interactive oauth2 branch", () => {
+ expect(editRoot({ transport: "http", auth_type: "oauth2", oauth_flow_type: "interactive" })).toContain(
+ "token_validation_json",
+ );
+ expect(editRoot({ transport: "http", auth_type: "oauth2", oauth_flow_type: "m2m" })).not.toContain(
+ "token_validation_json",
+ );
+ expect(editRoot({ transport: "http", auth_type: "oauth2_token_exchange" })).not.toContain("token_validation_json");
+ expect(editRoot(HTTP_NONE)).not.toContain("token_validation_json");
+ });
+
+ it("mounts token_storage_ttl_seconds only on the interactive oauth2 branch", () => {
+ expect(editRoot({ transport: "http", auth_type: "oauth2", oauth_flow_type: "interactive" })).toContain(
+ "token_storage_ttl_seconds",
+ );
+ expect(editRoot({ transport: "http", auth_type: "oauth2", oauth_flow_type: "m2m" })).not.toContain(
+ "token_storage_ttl_seconds",
+ );
+ });
+
+ it("gates audience and subject_token_type on the entra_obo token-exchange profile", () => {
+ const rfc = editRoot({ transport: "http", auth_type: "oauth2_token_exchange", token_exchange_profile: "rfc8693" });
+ const entra = editRoot({
+ transport: "http",
+ auth_type: "oauth2_token_exchange",
+ token_exchange_profile: "entra_obo",
+ });
+ expect(rfc).toContain("audience");
+ expect(rfc).toContain("subject_token_type");
+ expect(entra).not.toContain("audience");
+ expect(entra).not.toContain("subject_token_type");
+ expect(entra).toContain("token_exchange_profile");
+ });
+
+ it("mounts the seven aws credentials only for aws_sigv4", () => {
+ expect(sorted(editCreds({ transport: "http", auth_type: "aws_sigv4" }))).toStrictEqual(
+ sorted([
+ "aws_region_name",
+ "aws_service_name",
+ "aws_access_key_id",
+ "aws_secret_access_key",
+ "aws_session_token",
+ "aws_role_name",
+ "aws_session_name",
+ ]),
+ );
+ expect(editCreds(HTTP_NONE)).not.toContain("aws_region_name");
+ });
+
+ it("mounts the id-jag credential set only for oauth2_id_jag", () => {
+ const creds = editCreds({ transport: "http", auth_type: "oauth2_id_jag" });
+ expect(creds).toContain("id_jag_resource_token_endpoint");
+ expect(creds).toContain("client_private_key");
+ expect(creds).toContain("client_assertion_signing_alg");
+ expect(editCreds(HTTP_NONE)).not.toContain("id_jag_resource_token_endpoint");
+ });
+});
+
+describe("edit root: children that gate by early return null", () => {
+ it("mounts dcr_bridge and the declared-app credentials only for the client-forwarded modes", () => {
+ for (const authType of ["true_passthrough", "oauth_delegate"]) {
+ expect(editRoot({ transport: "http", auth_type: authType })).toContain("dcr_bridge");
+ expect(sorted(editCreds({ transport: "http", auth_type: authType }))).toStrictEqual(
+ sorted(["client_id", "client_secret"]),
+ );
+ }
+ expect(editRoot(HTTP_NONE)).not.toContain("dcr_bridge");
+ expect(editRoot({ transport: "http", auth_type: "oauth2" })).not.toContain("dcr_bridge");
+ });
+
+ it("unmounts dcr_bridge with its parent section on stdio", () => {
+ expect(editRoot({ transport: "stdio", auth_type: "true_passthrough" })).not.toContain("dcr_bridge");
+ });
+});
+
+describe("edit root: permission-section gates", () => {
+ it("mounts delegate_auth_to_upstream only for oauth2", () => {
+ expect(editRoot({ transport: "http", auth_type: "oauth2" })).toContain("delegate_auth_to_upstream");
+ expect(editRoot(HTTP_NONE)).not.toContain("delegate_auth_to_upstream");
+ expect(editRoot({ transport: "http", auth_type: "api_key" })).not.toContain("delegate_auth_to_upstream");
+ });
+
+ it("mounts oauth_passthrough only for none-auth WITH an Authorization extra header", () => {
+ expect(editRoot({ ...HTTP_NONE, extra_headers: ["Authorization"] })).toContain("oauth_passthrough");
+ expect(editRoot({ ...HTTP_NONE, extra_headers: ["authorization"] })).toContain("oauth_passthrough");
+ expect(editRoot({ ...HTTP_NONE, extra_headers: ["X-Other"] })).not.toContain("oauth_passthrough");
+ expect(editRoot(HTTP_NONE)).not.toContain("oauth_passthrough");
+ expect(editRoot({ transport: "http", auth_type: "oauth2", extra_headers: ["Authorization"] })).not.toContain(
+ "oauth_passthrough",
+ );
+ });
+
+ it("treats an absent auth_type as none-auth for the oauth_passthrough gate", () => {
+ expect(editRoot({ transport: "http", extra_headers: ["Authorization"] })).toContain("oauth_passthrough");
+ });
+});
+
+describe("create root: where it diverges from edit", () => {
+ it("mounts source_url, which the edit root has no binding for", () => {
+ expect(createRoot({ transport: "http", auth_type: "none" })).toContain("source_url");
+ expect(editRoot(HTTP_NONE)).not.toContain("source_url");
+ });
+
+ it("gates url on an allow-list, so a blank transport mounts NEITHER url nor auth_type", () => {
+ const blank = createRoot({ transport: "" });
+ expect(blank).not.toContain("url");
+ expect(blank).not.toContain("auth_type");
+ expect(editRoot({ transport: "" })).toContain("url");
+ expect(editRoot({ transport: "" })).toContain("auth_type");
+ });
+
+ it("mounts stdio_config on stdio but never the edit root's command/args/env_json", () => {
+ const root = createRoot({ transport: "stdio" });
+ expect(root).toContain("stdio_config");
+ expect(root).not.toContain("command");
+ expect(root).not.toContain("args");
+ expect(root).not.toContain("env_json");
+ });
+
+ it("mounts the byok fields only for openapi with is_byok on", () => {
+ expect(createRoot({ transport: "openapi", auth_type: "none" })).toContain("is_byok");
+ expect(createRoot({ transport: "openapi", auth_type: "none" })).not.toContain("byok_description");
+ const on = createRoot({ transport: "openapi", auth_type: "none", is_byok: true });
+ expect(on).toContain("byok_description");
+ expect(on).toContain("byok_api_key_help_url");
+ expect(createRoot({ transport: "http", auth_type: "none", is_byok: true })).not.toContain("byok_description");
+ });
+
+ it("drops every credential while the transport is unset", () => {
+ expect(createCreds({ transport: "", auth_type: "aws_sigv4" })).toStrictEqual([]);
+ expect(createCreds({ transport: "http", auth_type: "aws_sigv4" })).toContain("aws_region_name");
+ });
+});
+
+const ALWAYS = ["server_name", "alias", "description", "transport", "max_concurrent_requests"];
+const PERMS = [
+ "allow_all_keys",
+ "available_on_public_internet",
+ "mcp_access_groups",
+ "extra_headers",
+ "static_headers",
+];
+const sorted = (xs: readonly string[]) => [...xs].sort();
+
+const expectEditSets = (
+ values: Record,
+ expected: { root: readonly string[]; credentials: readonly string[] },
+) => {
+ expect(sorted(editRoot(values))).toStrictEqual(sorted(expected.root));
+ expect(sorted(editCreds(values))).toStrictEqual(sorted(expected.credentials));
+};
+
+const expectCreateSets = (
+ values: Record,
+ expected: { root: readonly string[]; credentials: readonly string[] },
+) => {
+ expect(sorted(createRoot(values))).toStrictEqual(sorted(expected.root));
+ expect(sorted(createCreds(values))).toStrictEqual(sorted(expected.credentials));
+};
+
+describe("edit root: exact mounted set per auth configuration", () => {
+ it("http + none", () => {
+ expectEditSets(HTTP_NONE, { root: [...ALWAYS, "url", "auth_type", "env_vars", ...PERMS], credentials: [] });
+ });
+
+ it("http + api_key", () => {
+ expectEditSets(
+ { transport: "http", auth_type: "api_key" },
+ { root: [...ALWAYS, "url", "auth_type", "env_vars", ...PERMS], credentials: ["auth_value"] },
+ );
+ });
+
+ it("http + oauth2 M2M", () => {
+ expectEditSets(
+ { transport: "http", auth_type: "oauth2", oauth_flow_type: "m2m" },
+ {
+ root: [
+ ...ALWAYS,
+ "url",
+ "auth_type",
+ "oauth_flow_type",
+ "token_url",
+ "env_vars",
+ ...PERMS,
+ "delegate_auth_to_upstream",
+ ],
+ credentials: ["client_id", "client_secret", "token_endpoint_auth_method", "scopes", "upstream_resource"],
+ },
+ );
+ });
+
+ it("http + oauth2 interactive", () => {
+ expectEditSets(
+ { transport: "http", auth_type: "oauth2", oauth_flow_type: "interactive" },
+ {
+ root: [
+ ...ALWAYS,
+ "url",
+ "auth_type",
+ "oauth_flow_type",
+ "issuer",
+ "authorization_url",
+ "token_url",
+ "registration_url",
+ "token_validation_json",
+ "token_storage_ttl_seconds",
+ "env_vars",
+ ...PERMS,
+ "delegate_auth_to_upstream",
+ ],
+ credentials: ["client_id", "client_secret", "scopes", "upstream_resource", "token_endpoint_auth_method"],
+ },
+ );
+ });
+
+ it("http + token exchange, rfc8693", () => {
+ expectEditSets(
+ { transport: "http", auth_type: "oauth2_token_exchange", token_exchange_profile: "rfc8693" },
+ {
+ root: [
+ ...ALWAYS,
+ "url",
+ "auth_type",
+ "token_exchange_profile",
+ "token_exchange_endpoint",
+ "audience",
+ "subject_token_type",
+ "env_vars",
+ ...PERMS,
+ ],
+ credentials: ["client_id", "client_secret", "scopes"],
+ },
+ );
+ });
+
+ it("http + token exchange, entra_obo keeps the endpoint while dropping audience and subject_token_type", () => {
+ expectEditSets(
+ { transport: "http", auth_type: "oauth2_token_exchange", token_exchange_profile: "entra_obo" },
+ {
+ root: [
+ ...ALWAYS,
+ "url",
+ "auth_type",
+ "token_exchange_profile",
+ "token_exchange_endpoint",
+ "env_vars",
+ ...PERMS,
+ ],
+ credentials: ["client_id", "client_secret", "scopes"],
+ },
+ );
+ });
+
+ it("http + id-jag", () => {
+ expectEditSets(
+ { transport: "http", auth_type: "oauth2_id_jag" },
+ {
+ root: [
+ ...ALWAYS,
+ "url",
+ "auth_type",
+ "token_exchange_endpoint",
+ "audience",
+ "subject_token_type",
+ "env_vars",
+ ...PERMS,
+ ],
+ credentials: [
+ "id_jag_resource_token_endpoint",
+ "client_id",
+ "client_secret",
+ "client_private_key",
+ "client_private_key_id",
+ "client_assertion_signing_alg",
+ "id_jag_resource",
+ "scopes",
+ ],
+ },
+ );
+ });
+
+ it("http + true_passthrough", () => {
+ expectEditSets(
+ { transport: "http", auth_type: "true_passthrough" },
+ {
+ root: [...ALWAYS, "url", "auth_type", "dcr_bridge", "env_vars", ...PERMS],
+ credentials: ["client_id", "client_secret"],
+ },
+ );
+ });
+
+ it("openapi + none", () => {
+ expectEditSets(
+ { transport: "openapi", auth_type: "none" },
+ { root: [...ALWAYS, "spec_path", "auth_type", "env_vars", ...PERMS], credentials: [] },
+ );
+ });
+});
+
+describe("create root: exact mounted set per configuration", () => {
+ it("http + none", () => {
+ expectCreateSets(
+ { transport: "http", auth_type: "none" },
+ { root: [...ALWAYS, "source_url", "url", "auth_type", "env_vars", ...PERMS], credentials: [] },
+ );
+ });
+
+ it("openapi with byok on", () => {
+ expectCreateSets(
+ { transport: "openapi", auth_type: "none", is_byok: true },
+ {
+ root: [
+ ...ALWAYS,
+ "source_url",
+ "spec_path",
+ "is_byok",
+ "byok_description",
+ "byok_api_key_help_url",
+ "auth_type",
+ "env_vars",
+ ...PERMS,
+ ],
+ credentials: [],
+ },
+ );
+ });
+
+ it("stdio", () => {
+ expectCreateSets(
+ { transport: "stdio" },
+ { root: [...ALWAYS, "source_url", "stdio_config", "env_vars", ...PERMS], credentials: [] },
+ );
+ });
+
+ it("transport still unset", () => {
+ expectCreateSets({ transport: "" }, { root: [...ALWAYS, "source_url", "env_vars", ...PERMS], credentials: [] });
+ });
+
+ it("http + oauth2 interactive", () => {
+ expectCreateSets(
+ { transport: "http", auth_type: "oauth2", oauth_flow_type: "interactive" },
+ {
+ root: [
+ ...ALWAYS,
+ "source_url",
+ "url",
+ "auth_type",
+ "oauth_flow_type",
+ "issuer",
+ "authorization_url",
+ "token_url",
+ "registration_url",
+ "token_validation_json",
+ "token_storage_ttl_seconds",
+ "env_vars",
+ ...PERMS,
+ "delegate_auth_to_upstream",
+ ],
+ credentials: ["client_id", "client_secret", "scopes", "upstream_resource", "token_endpoint_auth_method"],
+ },
+ );
+ });
+
+ it("http + oauth_delegate mounts dcr_bridge and the declared app", () => {
+ expectCreateSets(
+ { transport: "http", auth_type: "oauth_delegate" },
+ {
+ root: [...ALWAYS, "source_url", "url", "auth_type", "dcr_bridge", "env_vars", ...PERMS],
+ credentials: ["client_id", "client_secret"],
+ },
+ );
+ });
+});
+
+describe("projection shape", () => {
+ it("EMITS a mounted-but-unset field as a key holding undefined, matching antd onFinish", () => {
+ const projected = projectMountedEditValues({ transport: "http", auth_type: "none", server_name: "s" });
+ expect("description" in projected).toBe(true);
+ expect(projected.description).toBeUndefined();
+ expect(Object.keys(projected)).toContain("max_concurrent_requests");
+ });
+
+ it("emits mounted-but-unset CREDENTIAL keys as undefined rather than omitting them", () => {
+ const projected = projectMountedEditValues({ transport: "http", auth_type: "api_key" });
+ expect(Object.keys(projected.credentials as object)).toStrictEqual(["auth_value"]);
+ expect((projected.credentials as Record).auth_value).toBeUndefined();
+ });
+
+ it("omits the credentials key entirely when no credential field is mounted", () => {
+ expect("credentials" in projectMountedEditValues(HTTP_NONE)).toBe(false);
+ });
+
+ it("drops an unmounted field even when the store still holds a value for it", () => {
+ const storeWithStaleHttpValues = {
+ transport: "stdio",
+ auth_type: "oauth2",
+ url: "https://kept-in-store.example",
+ issuer: "https://kept-in-store.example",
+ command: "npx",
+ };
+ const projected = projectMountedEditValues(storeWithStaleHttpValues);
+ expect("url" in projected).toBe(false);
+ expect("issuer" in projected).toBe(false);
+ expect(projected.command).toBe("npx");
+ });
+
+ it("passes list rows through whole, since a list field is projected as one key and not per mounted sub-field", () => {
+ const row = { name: "N", value: "V", scope: "user", description: "D" };
+ const projected = projectMountedEditValues({ ...HTTP_NONE, env_vars: [row] });
+ expect(projected.env_vars).toStrictEqual([row]);
+ });
+
+ it("keeps static_headers rows whole", () => {
+ const rows = [{ header: "X-A", value: "1" }];
+ expect(
+ projectMountedCreateValues({ transport: "http", auth_type: "none", static_headers: rows }).static_headers,
+ ).toStrictEqual(rows);
+ });
+});
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mountedServerFields.ts b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mountedServerFields.ts
new file mode 100644
index 00000000000..22f0146afc9
--- /dev/null
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mountedServerFields.ts
@@ -0,0 +1,195 @@
+import { AUTH_TYPE, OAUTH_FLOW, TRANSPORT, isClientForwardedTokenMode } from "@/components/mcp_tools/types";
+import { AUTH_TYPES_REQUIRING_AUTH_VALUE } from "./createServerPayload";
+
+export interface MountedFieldNames {
+ readonly root: readonly string[];
+ readonly credentials: readonly string[];
+}
+
+const ENTRA_OBO_PROFILE = "entra_obo";
+
+const ALWAYS_MOUNTED_ROOT = ["server_name", "alias", "description", "transport", "max_concurrent_requests"] as const;
+
+const PERMISSION_SECTION_ROOT = [
+ "allow_all_keys",
+ "available_on_public_internet",
+ "mcp_access_groups",
+ "extra_headers",
+ "static_headers",
+] as const;
+
+const OAUTH_M2M_CREDENTIALS = [
+ "client_id",
+ "client_secret",
+ "token_endpoint_auth_method",
+ "scopes",
+ "upstream_resource",
+] as const;
+
+const OAUTH_INTERACTIVE_CREDENTIALS = [
+ "client_id",
+ "client_secret",
+ "scopes",
+ "upstream_resource",
+ "token_endpoint_auth_method",
+] as const;
+
+const OAUTH_INTERACTIVE_ROOT = [
+ "issuer",
+ "authorization_url",
+ "token_url",
+ "registration_url",
+ "token_validation_json",
+ "token_storage_ttl_seconds",
+] as const;
+
+const ID_JAG_CREDENTIALS = [
+ "id_jag_resource_token_endpoint",
+ "client_id",
+ "client_secret",
+ "client_private_key",
+ "client_private_key_id",
+ "client_assertion_signing_alg",
+ "id_jag_resource",
+ "scopes",
+] as const;
+
+const AWS_SIGV4_CREDENTIALS = [
+ "aws_region_name",
+ "aws_service_name",
+ "aws_access_key_id",
+ "aws_secret_access_key",
+ "aws_session_token",
+ "aws_role_name",
+ "aws_session_name",
+] as const;
+
+const hasAuthorizationExtraHeader = (extraHeaders: unknown): boolean =>
+ Array.isArray(extraHeaders) && extraHeaders.some((h) => typeof h === "string" && h.toLowerCase() === "authorization");
+
+interface AuthSubtreeGates {
+ readonly authType: string | undefined;
+ readonly oauthFlowType: string | undefined;
+ readonly tokenExchangeProfile: string | undefined;
+}
+
+const authSubtreeRoot = ({ authType, oauthFlowType, tokenExchangeProfile }: AuthSubtreeGates): readonly string[] => {
+ if (authType === AUTH_TYPE.OAUTH2) {
+ return oauthFlowType === OAUTH_FLOW.M2M
+ ? ["oauth_flow_type", "token_url"]
+ : ["oauth_flow_type", ...OAUTH_INTERACTIVE_ROOT];
+ }
+ if (authType === AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE) {
+ return tokenExchangeProfile === ENTRA_OBO_PROFILE
+ ? ["token_exchange_profile", "token_exchange_endpoint"]
+ : ["token_exchange_profile", "token_exchange_endpoint", "audience", "subject_token_type"];
+ }
+ if (authType === AUTH_TYPE.OAUTH2_ID_JAG) {
+ return ["token_exchange_endpoint", "audience", "subject_token_type"];
+ }
+ return [];
+};
+
+const authSubtreeCredentials = ({ authType, oauthFlowType }: AuthSubtreeGates): readonly string[] => {
+ const authValue = AUTH_TYPES_REQUIRING_AUTH_VALUE.includes(authType as string) ? ["auth_value"] : [];
+ const clientForwarded = isClientForwardedTokenMode(authType) ? ["client_id", "client_secret"] : [];
+ if (authType === AUTH_TYPE.OAUTH2) {
+ return [
+ ...authValue,
+ ...(oauthFlowType === OAUTH_FLOW.M2M ? OAUTH_M2M_CREDENTIALS : OAUTH_INTERACTIVE_CREDENTIALS),
+ ];
+ }
+ if (authType === AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE) {
+ return [...authValue, "client_id", "client_secret", "scopes"];
+ }
+ if (authType === AUTH_TYPE.OAUTH2_ID_JAG) {
+ return [...authValue, ...ID_JAG_CREDENTIALS];
+ }
+ if (authType === AUTH_TYPE.AWS_SIGV4) {
+ return [...authValue, ...AWS_SIGV4_CREDENTIALS];
+ }
+ return [...authValue, ...clientForwarded];
+};
+
+const permissionSectionRoot = (authType: string | undefined, extraHeaders: unknown): readonly string[] => {
+ const isNoneAuth = authType === AUTH_TYPE.NONE || authType == null;
+ return [
+ ...PERMISSION_SECTION_ROOT,
+ ...(authType === AUTH_TYPE.OAUTH2 ? ["delegate_auth_to_upstream"] : []),
+ ...(isNoneAuth && hasAuthorizationExtraHeader(extraHeaders) ? ["oauth_passthrough"] : []),
+ ];
+};
+
+const dedupe = (names: readonly string[]): readonly string[] => Array.from(new Set(names));
+
+export const mountedEditFieldNames = (values: Record): MountedFieldNames => {
+ const transport = values.transport as string | undefined;
+ const isStdio = transport === "stdio";
+ const isOpenApi = transport === TRANSPORT.OPENAPI;
+ const isMcp = !isStdio && !isOpenApi;
+ const gates: AuthSubtreeGates = {
+ authType: isStdio ? undefined : (values.auth_type as string | undefined),
+ oauthFlowType: values.oauth_flow_type as string | undefined,
+ tokenExchangeProfile: values.token_exchange_profile as string | undefined,
+ };
+
+ return {
+ root: dedupe([
+ ...ALWAYS_MOUNTED_ROOT,
+ ...(isMcp ? ["url"] : []),
+ ...(isOpenApi ? ["spec_path"] : []),
+ ...(isStdio ? ["command", "args", "env_json", "stdio_config"] : ["auth_type"]),
+ ...(isStdio ? [] : authSubtreeRoot(gates)),
+ ...(!isStdio && isClientForwardedTokenMode(gates.authType) ? ["dcr_bridge"] : []),
+ "env_vars",
+ ...permissionSectionRoot(gates.authType, values.extra_headers),
+ ]),
+ credentials: isStdio ? [] : dedupe(authSubtreeCredentials(gates)),
+ };
+};
+
+export const mountedCreateFieldNames = (values: Record): MountedFieldNames => {
+ const transport = values.transport as string | undefined;
+ const isStdio = transport === "stdio";
+ const isOpenApi = transport === TRANSPORT.OPENAPI;
+ const authSectionMounted = !isStdio && transport !== "" && transport !== undefined;
+ const gates: AuthSubtreeGates = {
+ authType: authSectionMounted ? (values.auth_type as string | undefined) : undefined,
+ oauthFlowType: values.oauth_flow_type as string | undefined,
+ tokenExchangeProfile: values.token_exchange_profile as string | undefined,
+ };
+
+ return {
+ root: dedupe([
+ ...ALWAYS_MOUNTED_ROOT,
+ "source_url",
+ ...(transport === "http" || transport === "sse" ? ["url"] : []),
+ ...(isOpenApi ? ["spec_path", "is_byok"] : []),
+ ...(isOpenApi && values.is_byok ? ["byok_description", "byok_api_key_help_url"] : []),
+ ...(authSectionMounted ? ["auth_type"] : []),
+ ...(authSectionMounted ? authSubtreeRoot(gates) : []),
+ ...(authSectionMounted && isClientForwardedTokenMode(gates.authType) ? ["dcr_bridge"] : []),
+ ...(isStdio ? ["stdio_config"] : []),
+ "env_vars",
+ ...permissionSectionRoot(gates.authType, values.extra_headers),
+ ]),
+ credentials: authSectionMounted ? dedupe(authSubtreeCredentials(gates)) : [],
+ };
+};
+
+const pickEmitting = (source: Record | undefined, names: readonly string[]): Record =>
+ Object.fromEntries(names.map((name) => [name, source?.[name]]));
+
+const projectWith =
+ (namesOf: (values: Record) => MountedFieldNames) =>
+ (values: Record): Record => {
+ const names = namesOf(values);
+ const credentials = values.credentials as Record | undefined;
+ return {
+ ...pickEmitting(values, names.root),
+ ...(names.credentials.length > 0 ? { credentials: pickEmitting(credentials, names.credentials) } : {}),
+ };
+ };
+
+export const projectMountedEditValues = projectWith(mountedEditFieldNames);
+export const projectMountedCreateValues = projectWith(mountedCreateFieldNames);
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/testUtils.ts b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/testUtils.ts
index 7911c3eb800..c1692f436d7 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/testUtils.ts
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/testUtils.ts
@@ -4,6 +4,7 @@ import { expect } from "vitest";
export async function selectAntOption(labelText: string, optionText: string) {
const label = screen.getByText(labelText);
const select =
+ label.closest("[data-slot='field']")?.querySelector(".ant-select") ??
label.closest(".ant-form-item")?.querySelector(".ant-select") ??
label.closest(".ant-collapse-item")?.querySelector(".ant-select") ??
label.closest("div")?.querySelector(".ant-select") ??
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.test.tsx
index 282cc2722db..8b34d61ebad 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.test.tsx
@@ -3,11 +3,6 @@ import { render } from "@testing-library/react";
import { describe, expect, it, vi } from "vitest";
import PriceDataManagementTab from "./PriceDataManagementTab";
-// Deliberately do NOT mock @tremor/react. These tab components render standalone
-// (inside antd Tabs / directly as a route page), no longer inside a Tremor
-// . A Tremor root renders nothing without that context, so
-// this asserts the component's content is visible on its own — reverting the root
-// back to makes the title disappear and fails this test.
vi.mock("@/components/price_data_reload", () => ({ default: () => reload
}));
vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: () => ({ accessToken: "sk-test" }) }));
vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({
@@ -15,7 +10,7 @@ vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({
}));
describe("PriceDataManagementTab", () => {
- it("renders its content standalone, without a Tremor TabGroup ancestor", () => {
+ it("renders its content standalone, without a tab-panel ancestor", () => {
const { getByText } = render( );
expect(getByText("Price Data Management")).toBeInTheDocument();
});
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx
new file mode 100644
index 00000000000..b762f006261
--- /dev/null
+++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx
@@ -0,0 +1,424 @@
+import { renderWithProviders, screen, waitFor } from "../../../../../tests/test-utils";
+import userEvent, { PointerEventsCheckLevel } from "@testing-library/user-event";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import AddModelPanel from "./AddModelPanel";
+
+const modelCreateCall = vi.fn();
+const mockPtuEnabled = vi.fn();
+const mockAuthorized = vi.fn();
+
+vi.mock("@/components/networking", async (importOriginal) => {
+ const actual = await importOriginal();
+ return {
+ ...actual,
+ modelCreateCall: (accessToken: string, model: unknown) => modelCreateCall(accessToken, model),
+ modelAvailableCall: vi.fn().mockResolvedValue({ data: [{ id: "group-a" }] }),
+ };
+});
+
+vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: () => mockAuthorized() }));
+
+vi.mock("@/app/(dashboard)/hooks/uiSettings/usePtuCostAttributionEnabled", () => ({
+ usePtuCostAttributionEnabled: () => mockPtuEnabled(),
+}));
+
+vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({ useModelCostMap: () => ({ data: {} }) }));
+
+vi.mock("@/app/(dashboard)/hooks/credentials/useCredentials", () => ({
+ useCredentials: () => ({ data: { credentials: [] } }),
+}));
+
+vi.mock("@/app/(dashboard)/hooks/teams/useTeams", () => ({
+ useTeams: () => ({ data: [] }),
+ useInfiniteTeams: () => ({
+ data: { pages: [{ teams: [], total: 0, page: 1, page_size: 20, total_pages: 1 }] },
+ fetchNextPage: vi.fn(),
+ hasNextPage: false,
+ isFetchingNextPage: false,
+ isLoading: false,
+ }),
+}));
+
+vi.mock("@/app/(dashboard)/hooks/guardrails/useGuardrails", () => ({
+ useGuardrails: () => ({ data: { guardrails: [{ guardrail_name: "g-1" }] }, isLoading: false, error: null }),
+}));
+
+vi.mock("@/app/(dashboard)/hooks/tags/useTags", () => ({
+ useTags: () => ({ data: {}, isLoading: false, error: null }),
+}));
+
+vi.mock("@/app/(dashboard)/hooks/providers/useProviderFields", () => ({
+ useProviderFields: () => ({
+ data: [
+ {
+ provider: "OpenAI",
+ provider_display_name: "OpenAI",
+ litellm_provider: "openai",
+ default_model_placeholder: "gpt-4o",
+ credential_fields: [
+ { key: "api_key", label: "API Key", field_type: "password", required: false },
+ { key: "api_base", label: "API Base", field_type: "text", required: false },
+ ],
+ },
+ ],
+ isLoading: false,
+ error: null,
+ }),
+}));
+
+vi.mock("@/components/vector_store_management/VectorStoreSelector", () => ({
+ default: () =>
,
+}));
+
+const lastCreatedModel = () => modelCreateCall.mock.calls.at(-1)?.[1];
+
+const PROXY_ADMIN = {
+ token: "t",
+ accessToken: "test-access-token",
+ userId: "user-1",
+ userEmail: "a@b.c",
+ userRole: "proxy_admin",
+ premiumUser: true,
+ disabledPersonalKeyCreation: false,
+ showSSOBanner: false,
+};
+
+const alwaysMounted = {
+ api_key: undefined,
+ api_base: undefined,
+ custom_llm_provider: "openai",
+ litellm_credential_name: null,
+ model: "gpt-4o",
+};
+
+const advancedOpenExtras = {
+ guardrails: undefined,
+ tags: undefined,
+ use_in_pass_through: undefined,
+ vector_store_ids: undefined,
+};
+
+const baseModelInfo = { access_groups: undefined, mode: undefined };
+
+const { api_base: _omitted, ...ALWAYS_MOUNTED_WITHOUT_API_BASE } = alwaysMounted;
+
+const setup = async () => {
+ const user = userEvent.setup({ pointerEventsCheck: PointerEventsCheckLevel.Never });
+ renderWithProviders( );
+ await screen.findByText("Provider");
+
+ const openAdvanced = async () => {
+ await user.click(screen.getByText("Advanced Settings"));
+ await screen.findByText("Tags");
+ };
+
+ const closeAdvanced = async () => {
+ await user.click(screen.getByText("Advanced Settings"));
+ await waitFor(() => expect(screen.queryByText("Tags")).not.toBeInTheDocument());
+ };
+
+ const fillRequired = async (modelName = "gpt-4o") => {
+ await user.click(screen.getByRole("combobox", { name: /provider/i }));
+ await user.click(await screen.findByText("OpenAI"));
+ await user.type(await screen.findByPlaceholderText("gpt-3.5-turbo"), modelName);
+ };
+
+ const submit = async () => {
+ await user.click(screen.getByTestId("add-model-btn"));
+ await waitFor(() => expect(modelCreateCall).toHaveBeenCalled());
+ };
+
+ const submitExpectingRejection = async (message: string) => {
+ await user.click(screen.getByTestId("add-model-btn"));
+ await screen.findByText(message);
+ };
+
+ return { user, openAdvanced, closeAdvanced, fillRequired, submit, submitExpectingRejection };
+};
+
+describe("AddModelPanel submit payload contract", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ mockPtuEnabled.mockReturnValue(false);
+ mockAuthorized.mockReturnValue(PROXY_ADMIN);
+ });
+
+ it("sends only the always-mounted fields while Advanced Settings stays closed", async () => {
+ const { fillRequired, submit } = await setup();
+ await fillRequired();
+ await submit();
+
+ expect(lastCreatedModel()).toStrictEqual({
+ model_name: "gpt-4o",
+ litellm_params: { ...alwaysMounted },
+ model_info: { ...baseModelInfo },
+ });
+ });
+
+ it("registers four more keys as undefined once Advanced Settings opens", async () => {
+ const { openAdvanced, fillRequired, submit } = await setup();
+ await fillRequired();
+ await openAdvanced();
+ await submit();
+
+ expect(lastCreatedModel()).toStrictEqual({
+ model_name: "gpt-4o",
+ litellm_params: { ...alwaysMounted, ...advancedOpenExtras },
+ model_info: { ...baseModelInfo },
+ });
+ });
+
+ it("merges typed LiteLLM Params into litellm_params", async () => {
+ const { user, openAdvanced, fillRequired, submit } = await setup();
+ await fillRequired();
+ await openAdvanced();
+ await user.type(screen.getByLabelText("LiteLLM Params"), '{{"rpm": 7}');
+ await submit();
+
+ expect(lastCreatedModel()).toStrictEqual({
+ model_name: "gpt-4o",
+ litellm_params: { ...alwaysMounted, ...advancedOpenExtras, rpm: 7 },
+ model_info: { ...baseModelInfo },
+ });
+ });
+
+ it("drops a collapsed section's keys and the value typed into it", async () => {
+ const { user, openAdvanced, closeAdvanced, fillRequired, submit } = await setup();
+ await fillRequired();
+ await openAdvanced();
+ await user.type(screen.getByLabelText("LiteLLM Params"), '{{"rpm": 7}');
+ await closeAdvanced();
+ await submit();
+
+ expect(lastCreatedModel()).toStrictEqual({
+ model_name: "gpt-4o",
+ litellm_params: { ...alwaysMounted },
+ model_info: { ...baseModelInfo },
+ });
+ });
+
+ it("restores the typed value when the section is expanded again", async () => {
+ const { user, openAdvanced, closeAdvanced, fillRequired, submit } = await setup();
+ await fillRequired();
+ await openAdvanced();
+ await user.type(screen.getByLabelText("LiteLLM Params"), '{{"rpm": 7}');
+ await closeAdvanced();
+ await openAdvanced();
+ expect(screen.getByLabelText("LiteLLM Params")).toHaveValue('{"rpm": 7}');
+
+ await submit();
+
+ expect(lastCreatedModel()).toStrictEqual({
+ model_name: "gpt-4o",
+ litellm_params: { ...alwaysMounted, ...advancedOpenExtras, rpm: 7 },
+ model_info: { ...baseModelInfo },
+ });
+ });
+
+ it("converts per-million pricing to per-token and falls back to input cost for cache reads", async () => {
+ const { user, openAdvanced, fillRequired, submit } = await setup();
+ await fillRequired();
+ await openAdvanced();
+ await user.click(screen.getByLabelText("Custom Pricing"));
+ await user.type(await screen.findByLabelText("Input Cost (per 1M tokens)"), "3");
+ await user.type(screen.getByLabelText("Output Cost (per 1M tokens)"), "9");
+ await submit();
+
+ expect(lastCreatedModel()).toStrictEqual({
+ model_name: "gpt-4o",
+ litellm_params: {
+ ...alwaysMounted,
+ ...advancedOpenExtras,
+ input_cost_per_token: 0.000003,
+ output_cost_per_token: 0.000009,
+ cache_read_input_token_cost: 0.000003,
+ },
+ model_info: { ...baseModelInfo },
+ });
+ });
+
+ it("sends the seeded injection point when cache control is switched on", async () => {
+ const { user, openAdvanced, fillRequired, submit } = await setup();
+ await fillRequired();
+ await openAdvanced();
+ await user.click(screen.getByLabelText("Cache Control Injection Points"));
+ await screen.findByText("Add Injection Point");
+ await submit();
+
+ expect(lastCreatedModel()).toStrictEqual({
+ model_name: "gpt-4o",
+ litellm_params: {
+ ...alwaysMounted,
+ ...advancedOpenExtras,
+ cache_control_injection_points: [{ location: "message" }],
+ },
+ model_info: { ...baseModelInfo },
+ });
+ });
+
+ it("carries a role picked inside the injection point editor, with the index kept a string", async () => {
+ const { user, openAdvanced, fillRequired, submit } = await setup();
+ await fillRequired();
+ await openAdvanced();
+ await user.click(screen.getByLabelText("Cache Control Injection Points"));
+ await screen.findByText("Add Injection Point");
+ await user.click(screen.getByText("Select a role"));
+ await user.click(await screen.findByText("System"));
+ await user.type(screen.getByPlaceholderText("Optional"), "3");
+ await submit();
+
+ expect(lastCreatedModel()).toStrictEqual({
+ model_name: "gpt-4o",
+ litellm_params: {
+ ...alwaysMounted,
+ ...advancedOpenExtras,
+ cache_control_injection_points: [{ location: "message", role: "system", index: "3" }],
+ },
+ model_info: { ...baseModelInfo },
+ });
+ });
+
+ it("mounts team_id only once the Team-BYOK switch is on", async () => {
+ const { user, fillRequired, submit } = await setup();
+ await fillRequired();
+ await user.click(screen.getByRole("switch", { name: "Team-BYOK Model" }));
+ await screen.findByText("Select Team");
+ await submit();
+
+ expect(lastCreatedModel()).toStrictEqual({
+ model_name: "gpt-4o",
+ litellm_params: { ...alwaysMounted },
+ model_info: { ...baseModelInfo, team_id: undefined },
+ });
+ });
+});
+
+describe("AddModelPanel empty-string skip", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ mockPtuEnabled.mockReturnValue(false);
+ mockAuthorized.mockReturnValue(PROXY_ADMIN);
+ });
+
+ it("sends a typed api_base, so the binding behind the next case is known to be live", async () => {
+ const { user, fillRequired, submit } = await setup();
+ await fillRequired();
+ await user.type(screen.getByLabelText("API Base"), "https://example.test");
+ await submit();
+
+ expect(lastCreatedModel().litellm_params).toStrictEqual({
+ ...alwaysMounted,
+ api_base: "https://example.test",
+ });
+ });
+
+ it("omits api_base entirely once it is cleared, rather than sending an empty string", async () => {
+ const { user, fillRequired, submit } = await setup();
+ await fillRequired();
+ const apiBase = screen.getByLabelText("API Base");
+ await user.type(apiBase, "https://example.test");
+ await user.clear(apiBase);
+ await submit();
+
+ const params = lastCreatedModel().litellm_params;
+ expect(params).not.toHaveProperty("api_base");
+ expect(params).toStrictEqual(ALWAYS_MOUNTED_WITHOUT_API_BASE);
+ });
+});
+
+describe("AddModelPanel validation gates", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ mockPtuEnabled.mockReturnValue(true);
+ mockAuthorized.mockReturnValue(PROXY_ADMIN);
+ });
+
+ it("blocks the submit when a PTU count carries no effective-from date", async () => {
+ const { user, openAdvanced, fillRequired, submitExpectingRejection } = await setup();
+ await fillRequired();
+ await openAdvanced();
+ await user.type(screen.getByLabelText("PTU Count"), "15");
+ await user.type(screen.getByLabelText("Calculated Cost per PTU / Hour (USD)"), "2");
+ await submitExpectingRejection("PTU Effective From is required when PTU Count is set");
+
+ expect(modelCreateCall).not.toHaveBeenCalled();
+ });
+
+ it("hides the PTU fields entirely when the capability is off", async () => {
+ mockPtuEnabled.mockReturnValue(false);
+ const { openAdvanced, fillRequired } = await setup();
+ await fillRequired();
+ await openAdvanced();
+
+ expect(screen.queryByLabelText("PTU Count")).not.toBeInTheDocument();
+ });
+
+ it("requires a model before anything is sent", async () => {
+ const { user, submitExpectingRejection } = await setup();
+ await user.click(screen.getByRole("combobox", { name: /provider/i }));
+ await user.click(await screen.findByText("OpenAI"));
+ await submitExpectingRejection("Please enter at least one model.");
+
+ expect(modelCreateCall).not.toHaveBeenCalled();
+ });
+
+ it("blocks the submit when LiteLLM Params is not valid JSON", async () => {
+ mockPtuEnabled.mockReturnValue(false);
+ const { user, openAdvanced, fillRequired, submitExpectingRejection } = await setup();
+ await fillRequired();
+ await openAdvanced();
+ await user.type(screen.getByLabelText("LiteLLM Params"), "rpm: 7");
+ await submitExpectingRejection("Please enter valid JSON");
+
+ expect(modelCreateCall).not.toHaveBeenCalled();
+ });
+});
+
+describe("AddModelPanel behaviours the removed Advanced Settings form instance never drove", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ mockPtuEnabled.mockReturnValue(false);
+ mockAuthorized.mockReturnValue(PROXY_ADMIN);
+ });
+
+ it("leaves LiteLLM Params untouched when pass through routes is switched on", async () => {
+ const { user, openAdvanced, fillRequired, submit } = await setup();
+ await fillRequired();
+ await openAdvanced();
+ await user.click(screen.getByLabelText("Use in pass through routes"));
+ expect(screen.getByLabelText("LiteLLM Params")).toHaveValue("");
+
+ await submit();
+
+ expect(lastCreatedModel()).toStrictEqual({
+ model_name: "gpt-4o",
+ litellm_params: { ...alwaysMounted, ...advancedOpenExtras, use_in_pass_through: true },
+ model_info: { ...baseModelInfo },
+ });
+ });
+
+ it("keeps a typed cost when custom pricing is switched off and back on", async () => {
+ const { user, openAdvanced, fillRequired, submit } = await setup();
+ await fillRequired();
+ await openAdvanced();
+ await user.click(screen.getByLabelText("Custom Pricing"));
+ await user.type(await screen.findByLabelText("Input Cost (per 1M tokens)"), "3");
+ await user.click(screen.getByLabelText("Custom Pricing"));
+ await waitFor(() => expect(screen.queryByLabelText("Input Cost (per 1M tokens)")).not.toBeInTheDocument());
+ await user.click(screen.getByLabelText("Custom Pricing"));
+ expect(await screen.findByLabelText("Input Cost (per 1M tokens)")).toHaveValue("3");
+
+ await submit();
+
+ expect(lastCreatedModel()).toStrictEqual({
+ model_name: "gpt-4o",
+ litellm_params: {
+ ...alwaysMounted,
+ ...advancedOpenExtras,
+ input_cost_per_token: 0.000003,
+ cache_read_input_token_cost: 0.000003,
+ },
+ model_info: { ...baseModelInfo },
+ });
+ });
+});
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx
index 35be19531d5..59d4f95c038 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx
@@ -1,21 +1,28 @@
"use client";
-import { Form } from "antd";
import { useState } from "react";
+import { useForm } from "react-hook-form";
import { useQueryClient } from "@tanstack/react-query";
import AddModelForm from "@/components/add_model/AddModelForm";
import { handleAddModelSubmit } from "@/components/add_model/handle_add_model_submit";
+import {
+ projectMountedValues,
+ useMountRegistry,
+ type MountedFormValues,
+} from "@/components/common_components/MountedFormField";
import { Providers, getPlaceholder, getProviderModels } from "@/components/provider_info_helpers";
-import { toast } from "@/lib/toast";
import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap";
import { useCredentials } from "@/app/(dashboard)/hooks/credentials/useCredentials";
import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams";
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
import { vertexCredentialsUploadProps } from "@/app/(dashboard)/models-and-endpoints/vertexCredentialsUpload";
+const INITIAL_VALUES: MountedFormValues = { litellm_credential_name: null };
+
export default function AddModelPanel() {
const { accessToken } = useAuthorized();
- const [form] = Form.useForm();
+ const form = useForm({ mode: "onChange", defaultValues: INITIAL_VALUES });
+ const registry = useMountRegistry();
const queryClient = useQueryClient();
const { data: modelCostMapData } = useModelCostMap();
const { data: credentialsResponse } = useCredentials();
@@ -26,28 +33,36 @@ export default function AddModelPanel() {
const refresh = () => queryClient.invalidateQueries({ queryKey: ["models", "list"] });
- const handleOk = async () => {
- try {
- const values = await form.validateFields();
- await handleAddModelSubmit(values, accessToken, form, refresh);
- } catch (error: any) {
- const errorMessages =
- error.errorFields?.map((field: any) => `${field.name.join(".")}: ${field.errors.join(", ")}`).join(" | ") ||
- "Unknown validation error";
- toast.fromError(`Please fill in the following required fields: ${errorMessages}`);
+ const mountedValues = () => projectMountedValues(registry, form.getValues);
+
+ const handleOk = async (): Promise => {
+ const isValid = await form.trigger(registry.mountedNames() as string[]);
+ if (!isValid) {
+ return false;
}
+ await handleAddModelSubmit(
+ mountedValues(),
+ accessToken,
+ { resetFields: () => form.reset(INITIAL_VALUES) },
+ refresh,
+ );
+ return true;
};
return (
setProviderModels(getProviderModels(provider, modelCostMapData))}
getPlaceholder={getPlaceholder}
- uploadProps={vertexCredentialsUploadProps(form)}
+ uploadProps={vertexCredentialsUploadProps({
+ setFieldsValue: (values) => form.setValue("vertex_credentials", values.vertex_credentials),
+ })}
showAdvancedSettings={showAdvancedSettings}
setShowAdvancedSettings={setShowAdvancedSettings}
teams={teams ?? null}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel.tsx
index 6112e6bffbd..7251da7c3c3 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel.tsx
@@ -1,10 +1,17 @@
"use client";
-import { Form } from "antd";
+import { useForm } from "react-hook-form";
import CredentialsPanel from "@/components/model_add/CredentialsPanel";
+import type { MountedFormValues } from "@/components/common_components/MountedFormField";
import { vertexCredentialsUploadProps } from "@/app/(dashboard)/models-and-endpoints/vertexCredentialsUpload";
export default function LlmCredentialsPanel() {
- const [form] = Form.useForm();
- return ;
+ const form = useForm();
+ return (
+ form.setValue("vertex_credentials", values.vertex_credentials),
+ })}
+ />
+ );
}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/impact_popover.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/impact_popover.test.tsx
index 66f03577e20..69c471a7c3d 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/impact_popover.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/impact_popover.test.tsx
@@ -30,16 +30,6 @@ vi.mock("@heroicons/react/outline", () => ({
},
}));
-vi.mock("@tremor/react", async (importOriginal) => {
- const actual = await importOriginal();
- return {
- ...actual,
- Icon: React.forwardRef(({ icon: _icon, ...props }, ref) => (
-
- )),
- };
-});
-
vi.mock("antd", async (importOriginal) => {
const actual = await importOriginal();
return {
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/index.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/index.test.tsx
index f508f6866d6..cddc04f6d9c 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/index.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/index.test.tsx
@@ -56,33 +56,6 @@ vi.mock("./impact_popover", () => ({
default: () => ,
}));
-vi.mock("@tremor/react", async (importOriginal) => {
- const actual = await importOriginal();
- return {
- ...actual,
- Button: React.forwardRef(({ children, ...props }, ref) =>
- React.createElement("button", { ...props, ref }, children),
- ),
- Tooltip: ({ children }: { children?: React.ReactNode }) => React.createElement(React.Fragment, null, children),
- Switch: ({
- checked,
- onChange,
- className,
- }: {
- checked?: boolean;
- onChange?: (v: boolean) => void;
- className?: string;
- }) =>
- React.createElement("input", {
- type: "checkbox",
- role: "switch",
- checked,
- onChange: (e: React.ChangeEvent) => onChange?.(e.target.checked),
- className,
- }),
- };
-});
-
vi.mock("./policy_templates", () => ({
__esModule: true,
default: () =>
,
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/tag-management/_components/tag_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/tag-management/_components/tag_info.tsx
index 6db19c6d090..5c561f38197 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/tag-management/_components/tag_info.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/tag-management/_components/tag_info.tsx
@@ -1,7 +1,8 @@
"use client";
import React, { useState, useEffect } from "react";
-import { Tooltip, Button as AntdButton } from "antd";
+import { SimpleTooltip } from "@/components/ui/tooltip";
+import { Button as AntdButton } from "antd";
import { z } from "zod/v4";
import { fetchUserModels } from "@/components/organisms/create_key_button";
import { getModelDisplayName } from "@/components/key_team_helpers/fetch_available_models_team_key";
@@ -280,7 +281,9 @@ const TagInfoView: React.FC = ({ tagId, onClose, accessToken,
) : (
tagDetails.models.map((modelId) => (
- {tagDetails.model_info?.[modelId] || modelId}
+
+ {tagDetails.model_info?.[modelId] || modelId}
+
))
)}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx
index a862e59d793..5ea15df13f0 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx
@@ -17,7 +17,8 @@ import {
teamMemberDeleteCall,
Member,
} from "@/components/networking";
-import { Button as AntdButton, Modal, Tooltip } from "antd";
+import { SimpleTooltip } from "@/components/ui/tooltip";
+import { Button as AntdButton, Modal } from "antd";
import { Field, FieldGroup, FieldLabel } from "@/components/shared/form/field";
import {
Combobox,
@@ -747,10 +748,10 @@ export default function UserInfoView({
{MEMBER_ROLE_OPTIONS.map((option) => (
-
+
{option.value}
- {option.hint}
-
+
))}
diff --git a/ui/litellm-dashboard/src/app/globals.css b/ui/litellm-dashboard/src/app/globals.css
index dddeb576720..442adf98fa2 100644
--- a/ui/litellm-dashboard/src/app/globals.css
+++ b/ui/litellm-dashboard/src/app/globals.css
@@ -2,9 +2,6 @@
@import "tailwindcss";
@import "tw-animate-css";
-@import "./tremor-v3-compat.css" layer(utilities);
-
-@source '../../node_modules/@tremor/react';
@plugin '@headlessui/tailwindcss';
@plugin '@tailwindcss/forms';
@@ -95,6 +92,9 @@
--accent: oklch(0.967 0.003 264.542);
--accent-foreground: oklch(0.21 0.034 264.665);
--destructive: oklch(0.577 0.245 27.325);
+ --success: oklch(0.527 0.154 150.069);
+ --warning: oklch(0.555 0.163 48.998);
+ --info: oklch(0.546 0.245 262.881);
--border: oklch(0.928 0.006 264.531);
--input: oklch(0.928 0.006 264.531);
--ring: oklch(0.707 0.022 261.325);
@@ -130,6 +130,9 @@
--accent: oklch(0.278 0.033 256.848);
--accent-foreground: oklch(0.985 0.002 247.839);
--destructive: oklch(0.704 0.191 22.216);
+ --success: oklch(0.792 0.209 151.711);
+ --warning: oklch(0.828 0.189 84.429);
+ --info: oklch(0.707 0.165 254.624);
--border: oklch(1 0 0 / 10%);
--input: oklch(1 0 0 / 15%);
--ring: oklch(0.551 0.027 264.364);
@@ -168,6 +171,9 @@
--color-accent: var(--accent);
--color-accent-foreground: var(--accent-foreground);
--color-destructive: var(--destructive);
+ --color-success: var(--success);
+ --color-warning: var(--warning);
+ --color-info: var(--info);
--color-border: var(--border);
--color-input: var(--input);
--color-ring: var(--ring);
@@ -186,66 +192,6 @@
--color-sidebar-ring: var(--sidebar-ring);
}
-@theme {
- --color-tremor-brand-muted: #8688ef;
- --color-tremor-brand-subtle: #8e91eb;
- --color-tremor-brand: #6366f1;
- --color-tremor-brand-emphasis: #4338ca;
- --color-tremor-brand-inverted: #ffffff;
- --color-tremor-background-muted: #f9fafb;
- --color-tremor-background-subtle: #f3f4f6;
- --color-tremor-background: #ffffff;
- --color-tremor-background-emphasis: #374151;
- --color-tremor-border: #e5e7eb;
- --color-tremor-ring: #e5e7eb;
- --color-tremor-content-subtle: #9ca3af;
- --color-tremor-content: #6b7280;
- --color-tremor-content-emphasis: #374151;
- --color-tremor-content-strong: #111827;
- --color-tremor-content-inverted: #ffffff;
-
- --color-dark-tremor-brand-faint: #0b1229;
- --color-dark-tremor-brand-muted: #1e1b4b;
- --color-dark-tremor-brand-subtle: #3730a3;
- --color-dark-tremor-brand: #6366f1;
- --color-dark-tremor-brand-emphasis: #818cf8;
- --color-dark-tremor-brand-inverted: #1e1b4b;
- --color-dark-tremor-background-muted: #131a2b;
- --color-dark-tremor-background-subtle: #1f2937;
- --color-dark-tremor-background: #111827;
- --color-dark-tremor-background-emphasis: #d1d5db;
- --color-dark-tremor-border: #374151;
- --color-dark-tremor-ring: #1f2937;
- --color-dark-tremor-content-subtle: #4b5563;
- --color-dark-tremor-content: #6b7280;
- --color-dark-tremor-content-emphasis: #e5e7eb;
- --color-dark-tremor-content-strong: #f9fafb;
- --color-dark-tremor-content-inverted: #030712;
-
- --shadow-tremor-input: 0 1px 2px 0 rgb(0 0 0 / 0.05);
- --shadow-tremor-card: 0 1px 3px 0 rgb(0 0 0 / 0.1), 0 1px 2px -1px rgb(0 0 0 / 0.1);
- --shadow-tremor-dropdown: 0 4px 6px -1px rgb(0 0 0 / 0.1), 0 2px 4px -2px rgb(0 0 0 / 0.1);
- --shadow-dark-tremor-input: 0 1px 2px 0 rgb(0 0 0 / 0.05);
- --shadow-dark-tremor-card: 0 1px 3px 0 rgb(0 0 0 / 0.1), 0 1px 2px -1px rgb(0 0 0 / 0.1);
- --shadow-dark-tremor-dropdown: 0 4px 6px -1px rgb(0 0 0 / 0.1), 0 2px 4px -2px rgb(0 0 0 / 0.1);
-
- --radius-tremor-small: 0.375rem;
- --radius-tremor-default: 0.5rem;
- --radius-tremor-full: 9999px;
-
- --text-tremor-label: 0.75rem;
- --text-tremor-label--line-height: 0.3rem;
- --text-tremor-default: 0.775rem;
- --text-tremor-default--line-height: 1.15rem;
- --text-tremor-title: 1.025rem;
- --text-tremor-title--line-height: 1.65rem;
- --text-tremor-metric: 1.675rem;
- --text-tremor-metric--line-height: 2.15rem;
-}
-
-@source inline("{,hover:,ui-selected:}{bg,text,border}-{slate,gray,zinc,neutral,stone,red,orange,amber,yellow,lime,green,emerald,teal,cyan,sky,blue,indigo,violet,purple,fuchsia,pink,rose}-{50,{100..900..100},950}");
-@source inline("{ring,stroke,fill}-{slate,gray,zinc,neutral,stone,red,orange,amber,yellow,lime,green,emerald,teal,cyan,sky,blue,indigo,violet,purple,fuchsia,pink,rose}-{50,{100..900..100},950}");
-
@layer base {
*,
::after,
diff --git a/ui/litellm-dashboard/src/app/tremor-v3-compat.css b/ui/litellm-dashboard/src/app/tremor-v3-compat.css
deleted file mode 100644
index 0ade5e4dd88..00000000000
--- a/ui/litellm-dashboard/src/app/tremor-v3-compat.css
+++ /dev/null
@@ -1,615 +0,0 @@
-.bg-slate-500.bg-opacity-10 {
- background-color: color-mix(in oklab, var(--color-slate-500) 10%, transparent);
-}
-
-.bg-slate-500.bg-opacity-20 {
- background-color: color-mix(in oklab, var(--color-slate-500) 20%, transparent);
-}
-
-.bg-slate-500.bg-opacity-40 {
- background-color: color-mix(in oklab, var(--color-slate-500) 40%, transparent);
-}
-
-.hover\:bg-slate-500.hover\:bg-opacity-20:hover {
- background-color: color-mix(in oklab, var(--color-slate-500) 20%, transparent);
-}
-
-.group:hover .bg-slate-500.group-hover\:bg-opacity-30 {
- background-color: color-mix(in oklab, var(--color-slate-500) 30%, transparent);
-}
-
-.ring-slate-500.ring-opacity-20 {
- --tw-ring-color: color-mix(in oklab, var(--color-slate-500) 20%, transparent);
-}
-
-.ring-slate-300.ring-opacity-40 {
- --tw-ring-color: color-mix(in oklab, var(--color-slate-300) 40%, transparent);
-}
-
-.bg-gray-500.bg-opacity-10 {
- background-color: color-mix(in oklab, var(--color-gray-500) 10%, transparent);
-}
-
-.bg-gray-500.bg-opacity-20 {
- background-color: color-mix(in oklab, var(--color-gray-500) 20%, transparent);
-}
-
-.bg-gray-500.bg-opacity-40 {
- background-color: color-mix(in oklab, var(--color-gray-500) 40%, transparent);
-}
-
-.hover\:bg-gray-500.hover\:bg-opacity-20:hover {
- background-color: color-mix(in oklab, var(--color-gray-500) 20%, transparent);
-}
-
-.group:hover .bg-gray-500.group-hover\:bg-opacity-30 {
- background-color: color-mix(in oklab, var(--color-gray-500) 30%, transparent);
-}
-
-.ring-gray-500.ring-opacity-20 {
- --tw-ring-color: color-mix(in oklab, var(--color-gray-500) 20%, transparent);
-}
-
-.ring-gray-300.ring-opacity-40 {
- --tw-ring-color: color-mix(in oklab, var(--color-gray-300) 40%, transparent);
-}
-
-.bg-zinc-500.bg-opacity-10 {
- background-color: color-mix(in oklab, var(--color-zinc-500) 10%, transparent);
-}
-
-.bg-zinc-500.bg-opacity-20 {
- background-color: color-mix(in oklab, var(--color-zinc-500) 20%, transparent);
-}
-
-.bg-zinc-500.bg-opacity-40 {
- background-color: color-mix(in oklab, var(--color-zinc-500) 40%, transparent);
-}
-
-.hover\:bg-zinc-500.hover\:bg-opacity-20:hover {
- background-color: color-mix(in oklab, var(--color-zinc-500) 20%, transparent);
-}
-
-.group:hover .bg-zinc-500.group-hover\:bg-opacity-30 {
- background-color: color-mix(in oklab, var(--color-zinc-500) 30%, transparent);
-}
-
-.ring-zinc-500.ring-opacity-20 {
- --tw-ring-color: color-mix(in oklab, var(--color-zinc-500) 20%, transparent);
-}
-
-.ring-zinc-300.ring-opacity-40 {
- --tw-ring-color: color-mix(in oklab, var(--color-zinc-300) 40%, transparent);
-}
-
-.bg-neutral-500.bg-opacity-10 {
- background-color: color-mix(in oklab, var(--color-neutral-500) 10%, transparent);
-}
-
-.bg-neutral-500.bg-opacity-20 {
- background-color: color-mix(in oklab, var(--color-neutral-500) 20%, transparent);
-}
-
-.bg-neutral-500.bg-opacity-40 {
- background-color: color-mix(in oklab, var(--color-neutral-500) 40%, transparent);
-}
-
-.hover\:bg-neutral-500.hover\:bg-opacity-20:hover {
- background-color: color-mix(in oklab, var(--color-neutral-500) 20%, transparent);
-}
-
-.group:hover .bg-neutral-500.group-hover\:bg-opacity-30 {
- background-color: color-mix(in oklab, var(--color-neutral-500) 30%, transparent);
-}
-
-.ring-neutral-500.ring-opacity-20 {
- --tw-ring-color: color-mix(in oklab, var(--color-neutral-500) 20%, transparent);
-}
-
-.ring-neutral-300.ring-opacity-40 {
- --tw-ring-color: color-mix(in oklab, var(--color-neutral-300) 40%, transparent);
-}
-
-.bg-stone-500.bg-opacity-10 {
- background-color: color-mix(in oklab, var(--color-stone-500) 10%, transparent);
-}
-
-.bg-stone-500.bg-opacity-20 {
- background-color: color-mix(in oklab, var(--color-stone-500) 20%, transparent);
-}
-
-.bg-stone-500.bg-opacity-40 {
- background-color: color-mix(in oklab, var(--color-stone-500) 40%, transparent);
-}
-
-.hover\:bg-stone-500.hover\:bg-opacity-20:hover {
- background-color: color-mix(in oklab, var(--color-stone-500) 20%, transparent);
-}
-
-.group:hover .bg-stone-500.group-hover\:bg-opacity-30 {
- background-color: color-mix(in oklab, var(--color-stone-500) 30%, transparent);
-}
-
-.ring-stone-500.ring-opacity-20 {
- --tw-ring-color: color-mix(in oklab, var(--color-stone-500) 20%, transparent);
-}
-
-.ring-stone-300.ring-opacity-40 {
- --tw-ring-color: color-mix(in oklab, var(--color-stone-300) 40%, transparent);
-}
-
-.bg-red-500.bg-opacity-10 {
- background-color: color-mix(in oklab, var(--color-red-500) 10%, transparent);
-}
-
-.bg-red-500.bg-opacity-20 {
- background-color: color-mix(in oklab, var(--color-red-500) 20%, transparent);
-}
-
-.bg-red-500.bg-opacity-40 {
- background-color: color-mix(in oklab, var(--color-red-500) 40%, transparent);
-}
-
-.hover\:bg-red-500.hover\:bg-opacity-20:hover {
- background-color: color-mix(in oklab, var(--color-red-500) 20%, transparent);
-}
-
-.group:hover .bg-red-500.group-hover\:bg-opacity-30 {
- background-color: color-mix(in oklab, var(--color-red-500) 30%, transparent);
-}
-
-.ring-red-500.ring-opacity-20 {
- --tw-ring-color: color-mix(in oklab, var(--color-red-500) 20%, transparent);
-}
-
-.ring-red-300.ring-opacity-40 {
- --tw-ring-color: color-mix(in oklab, var(--color-red-300) 40%, transparent);
-}
-
-.bg-orange-500.bg-opacity-10 {
- background-color: color-mix(in oklab, var(--color-orange-500) 10%, transparent);
-}
-
-.bg-orange-500.bg-opacity-20 {
- background-color: color-mix(in oklab, var(--color-orange-500) 20%, transparent);
-}
-
-.bg-orange-500.bg-opacity-40 {
- background-color: color-mix(in oklab, var(--color-orange-500) 40%, transparent);
-}
-
-.hover\:bg-orange-500.hover\:bg-opacity-20:hover {
- background-color: color-mix(in oklab, var(--color-orange-500) 20%, transparent);
-}
-
-.group:hover .bg-orange-500.group-hover\:bg-opacity-30 {
- background-color: color-mix(in oklab, var(--color-orange-500) 30%, transparent);
-}
-
-.ring-orange-500.ring-opacity-20 {
- --tw-ring-color: color-mix(in oklab, var(--color-orange-500) 20%, transparent);
-}
-
-.ring-orange-300.ring-opacity-40 {
- --tw-ring-color: color-mix(in oklab, var(--color-orange-300) 40%, transparent);
-}
-
-.bg-amber-500.bg-opacity-10 {
- background-color: color-mix(in oklab, var(--color-amber-500) 10%, transparent);
-}
-
-.bg-amber-500.bg-opacity-20 {
- background-color: color-mix(in oklab, var(--color-amber-500) 20%, transparent);
-}
-
-.bg-amber-500.bg-opacity-40 {
- background-color: color-mix(in oklab, var(--color-amber-500) 40%, transparent);
-}
-
-.hover\:bg-amber-500.hover\:bg-opacity-20:hover {
- background-color: color-mix(in oklab, var(--color-amber-500) 20%, transparent);
-}
-
-.group:hover .bg-amber-500.group-hover\:bg-opacity-30 {
- background-color: color-mix(in oklab, var(--color-amber-500) 30%, transparent);
-}
-
-.ring-amber-500.ring-opacity-20 {
- --tw-ring-color: color-mix(in oklab, var(--color-amber-500) 20%, transparent);
-}
-
-.ring-amber-300.ring-opacity-40 {
- --tw-ring-color: color-mix(in oklab, var(--color-amber-300) 40%, transparent);
-}
-
-.bg-yellow-500.bg-opacity-10 {
- background-color: color-mix(in oklab, var(--color-yellow-500) 10%, transparent);
-}
-
-.bg-yellow-500.bg-opacity-20 {
- background-color: color-mix(in oklab, var(--color-yellow-500) 20%, transparent);
-}
-
-.bg-yellow-500.bg-opacity-40 {
- background-color: color-mix(in oklab, var(--color-yellow-500) 40%, transparent);
-}
-
-.hover\:bg-yellow-500.hover\:bg-opacity-20:hover {
- background-color: color-mix(in oklab, var(--color-yellow-500) 20%, transparent);
-}
-
-.group:hover .bg-yellow-500.group-hover\:bg-opacity-30 {
- background-color: color-mix(in oklab, var(--color-yellow-500) 30%, transparent);
-}
-
-.ring-yellow-500.ring-opacity-20 {
- --tw-ring-color: color-mix(in oklab, var(--color-yellow-500) 20%, transparent);
-}
-
-.ring-yellow-300.ring-opacity-40 {
- --tw-ring-color: color-mix(in oklab, var(--color-yellow-300) 40%, transparent);
-}
-
-.bg-lime-500.bg-opacity-10 {
- background-color: color-mix(in oklab, var(--color-lime-500) 10%, transparent);
-}
-
-.bg-lime-500.bg-opacity-20 {
- background-color: color-mix(in oklab, var(--color-lime-500) 20%, transparent);
-}
-
-.bg-lime-500.bg-opacity-40 {
- background-color: color-mix(in oklab, var(--color-lime-500) 40%, transparent);
-}
-
-.hover\:bg-lime-500.hover\:bg-opacity-20:hover {
- background-color: color-mix(in oklab, var(--color-lime-500) 20%, transparent);
-}
-
-.group:hover .bg-lime-500.group-hover\:bg-opacity-30 {
- background-color: color-mix(in oklab, var(--color-lime-500) 30%, transparent);
-}
-
-.ring-lime-500.ring-opacity-20 {
- --tw-ring-color: color-mix(in oklab, var(--color-lime-500) 20%, transparent);
-}
-
-.ring-lime-300.ring-opacity-40 {
- --tw-ring-color: color-mix(in oklab, var(--color-lime-300) 40%, transparent);
-}
-
-.bg-green-500.bg-opacity-10 {
- background-color: color-mix(in oklab, var(--color-green-500) 10%, transparent);
-}
-
-.bg-green-500.bg-opacity-20 {
- background-color: color-mix(in oklab, var(--color-green-500) 20%, transparent);
-}
-
-.bg-green-500.bg-opacity-40 {
- background-color: color-mix(in oklab, var(--color-green-500) 40%, transparent);
-}
-
-.hover\:bg-green-500.hover\:bg-opacity-20:hover {
- background-color: color-mix(in oklab, var(--color-green-500) 20%, transparent);
-}
-
-.group:hover .bg-green-500.group-hover\:bg-opacity-30 {
- background-color: color-mix(in oklab, var(--color-green-500) 30%, transparent);
-}
-
-.ring-green-500.ring-opacity-20 {
- --tw-ring-color: color-mix(in oklab, var(--color-green-500) 20%, transparent);
-}
-
-.ring-green-300.ring-opacity-40 {
- --tw-ring-color: color-mix(in oklab, var(--color-green-300) 40%, transparent);
-}
-
-.bg-emerald-500.bg-opacity-10 {
- background-color: color-mix(in oklab, var(--color-emerald-500) 10%, transparent);
-}
-
-.bg-emerald-500.bg-opacity-20 {
- background-color: color-mix(in oklab, var(--color-emerald-500) 20%, transparent);
-}
-
-.bg-emerald-500.bg-opacity-40 {
- background-color: color-mix(in oklab, var(--color-emerald-500) 40%, transparent);
-}
-
-.hover\:bg-emerald-500.hover\:bg-opacity-20:hover {
- background-color: color-mix(in oklab, var(--color-emerald-500) 20%, transparent);
-}
-
-.group:hover .bg-emerald-500.group-hover\:bg-opacity-30 {
- background-color: color-mix(in oklab, var(--color-emerald-500) 30%, transparent);
-}
-
-.ring-emerald-500.ring-opacity-20 {
- --tw-ring-color: color-mix(in oklab, var(--color-emerald-500) 20%, transparent);
-}
-
-.ring-emerald-300.ring-opacity-40 {
- --tw-ring-color: color-mix(in oklab, var(--color-emerald-300) 40%, transparent);
-}
-
-.bg-teal-500.bg-opacity-10 {
- background-color: color-mix(in oklab, var(--color-teal-500) 10%, transparent);
-}
-
-.bg-teal-500.bg-opacity-20 {
- background-color: color-mix(in oklab, var(--color-teal-500) 20%, transparent);
-}
-
-.bg-teal-500.bg-opacity-40 {
- background-color: color-mix(in oklab, var(--color-teal-500) 40%, transparent);
-}
-
-.hover\:bg-teal-500.hover\:bg-opacity-20:hover {
- background-color: color-mix(in oklab, var(--color-teal-500) 20%, transparent);
-}
-
-.group:hover .bg-teal-500.group-hover\:bg-opacity-30 {
- background-color: color-mix(in oklab, var(--color-teal-500) 30%, transparent);
-}
-
-.ring-teal-500.ring-opacity-20 {
- --tw-ring-color: color-mix(in oklab, var(--color-teal-500) 20%, transparent);
-}
-
-.ring-teal-300.ring-opacity-40 {
- --tw-ring-color: color-mix(in oklab, var(--color-teal-300) 40%, transparent);
-}
-
-.bg-cyan-500.bg-opacity-10 {
- background-color: color-mix(in oklab, var(--color-cyan-500) 10%, transparent);
-}
-
-.bg-cyan-500.bg-opacity-20 {
- background-color: color-mix(in oklab, var(--color-cyan-500) 20%, transparent);
-}
-
-.bg-cyan-500.bg-opacity-40 {
- background-color: color-mix(in oklab, var(--color-cyan-500) 40%, transparent);
-}
-
-.hover\:bg-cyan-500.hover\:bg-opacity-20:hover {
- background-color: color-mix(in oklab, var(--color-cyan-500) 20%, transparent);
-}
-
-.group:hover .bg-cyan-500.group-hover\:bg-opacity-30 {
- background-color: color-mix(in oklab, var(--color-cyan-500) 30%, transparent);
-}
-
-.ring-cyan-500.ring-opacity-20 {
- --tw-ring-color: color-mix(in oklab, var(--color-cyan-500) 20%, transparent);
-}
-
-.ring-cyan-300.ring-opacity-40 {
- --tw-ring-color: color-mix(in oklab, var(--color-cyan-300) 40%, transparent);
-}
-
-.bg-sky-500.bg-opacity-10 {
- background-color: color-mix(in oklab, var(--color-sky-500) 10%, transparent);
-}
-
-.bg-sky-500.bg-opacity-20 {
- background-color: color-mix(in oklab, var(--color-sky-500) 20%, transparent);
-}
-
-.bg-sky-500.bg-opacity-40 {
- background-color: color-mix(in oklab, var(--color-sky-500) 40%, transparent);
-}
-
-.hover\:bg-sky-500.hover\:bg-opacity-20:hover {
- background-color: color-mix(in oklab, var(--color-sky-500) 20%, transparent);
-}
-
-.group:hover .bg-sky-500.group-hover\:bg-opacity-30 {
- background-color: color-mix(in oklab, var(--color-sky-500) 30%, transparent);
-}
-
-.ring-sky-500.ring-opacity-20 {
- --tw-ring-color: color-mix(in oklab, var(--color-sky-500) 20%, transparent);
-}
-
-.ring-sky-300.ring-opacity-40 {
- --tw-ring-color: color-mix(in oklab, var(--color-sky-300) 40%, transparent);
-}
-
-.bg-blue-500.bg-opacity-10 {
- background-color: color-mix(in oklab, var(--color-blue-500) 10%, transparent);
-}
-
-.bg-blue-500.bg-opacity-20 {
- background-color: color-mix(in oklab, var(--color-blue-500) 20%, transparent);
-}
-
-.bg-blue-500.bg-opacity-40 {
- background-color: color-mix(in oklab, var(--color-blue-500) 40%, transparent);
-}
-
-.hover\:bg-blue-500.hover\:bg-opacity-20:hover {
- background-color: color-mix(in oklab, var(--color-blue-500) 20%, transparent);
-}
-
-.group:hover .bg-blue-500.group-hover\:bg-opacity-30 {
- background-color: color-mix(in oklab, var(--color-blue-500) 30%, transparent);
-}
-
-.ring-blue-500.ring-opacity-20 {
- --tw-ring-color: color-mix(in oklab, var(--color-blue-500) 20%, transparent);
-}
-
-.ring-blue-300.ring-opacity-40 {
- --tw-ring-color: color-mix(in oklab, var(--color-blue-300) 40%, transparent);
-}
-
-.bg-indigo-500.bg-opacity-10 {
- background-color: color-mix(in oklab, var(--color-indigo-500) 10%, transparent);
-}
-
-.bg-indigo-500.bg-opacity-20 {
- background-color: color-mix(in oklab, var(--color-indigo-500) 20%, transparent);
-}
-
-.bg-indigo-500.bg-opacity-40 {
- background-color: color-mix(in oklab, var(--color-indigo-500) 40%, transparent);
-}
-
-.hover\:bg-indigo-500.hover\:bg-opacity-20:hover {
- background-color: color-mix(in oklab, var(--color-indigo-500) 20%, transparent);
-}
-
-.group:hover .bg-indigo-500.group-hover\:bg-opacity-30 {
- background-color: color-mix(in oklab, var(--color-indigo-500) 30%, transparent);
-}
-
-.ring-indigo-500.ring-opacity-20 {
- --tw-ring-color: color-mix(in oklab, var(--color-indigo-500) 20%, transparent);
-}
-
-.ring-indigo-300.ring-opacity-40 {
- --tw-ring-color: color-mix(in oklab, var(--color-indigo-300) 40%, transparent);
-}
-
-.bg-violet-500.bg-opacity-10 {
- background-color: color-mix(in oklab, var(--color-violet-500) 10%, transparent);
-}
-
-.bg-violet-500.bg-opacity-20 {
- background-color: color-mix(in oklab, var(--color-violet-500) 20%, transparent);
-}
-
-.bg-violet-500.bg-opacity-40 {
- background-color: color-mix(in oklab, var(--color-violet-500) 40%, transparent);
-}
-
-.hover\:bg-violet-500.hover\:bg-opacity-20:hover {
- background-color: color-mix(in oklab, var(--color-violet-500) 20%, transparent);
-}
-
-.group:hover .bg-violet-500.group-hover\:bg-opacity-30 {
- background-color: color-mix(in oklab, var(--color-violet-500) 30%, transparent);
-}
-
-.ring-violet-500.ring-opacity-20 {
- --tw-ring-color: color-mix(in oklab, var(--color-violet-500) 20%, transparent);
-}
-
-.ring-violet-300.ring-opacity-40 {
- --tw-ring-color: color-mix(in oklab, var(--color-violet-300) 40%, transparent);
-}
-
-.bg-purple-500.bg-opacity-10 {
- background-color: color-mix(in oklab, var(--color-purple-500) 10%, transparent);
-}
-
-.bg-purple-500.bg-opacity-20 {
- background-color: color-mix(in oklab, var(--color-purple-500) 20%, transparent);
-}
-
-.bg-purple-500.bg-opacity-40 {
- background-color: color-mix(in oklab, var(--color-purple-500) 40%, transparent);
-}
-
-.hover\:bg-purple-500.hover\:bg-opacity-20:hover {
- background-color: color-mix(in oklab, var(--color-purple-500) 20%, transparent);
-}
-
-.group:hover .bg-purple-500.group-hover\:bg-opacity-30 {
- background-color: color-mix(in oklab, var(--color-purple-500) 30%, transparent);
-}
-
-.ring-purple-500.ring-opacity-20 {
- --tw-ring-color: color-mix(in oklab, var(--color-purple-500) 20%, transparent);
-}
-
-.ring-purple-300.ring-opacity-40 {
- --tw-ring-color: color-mix(in oklab, var(--color-purple-300) 40%, transparent);
-}
-
-.bg-fuchsia-500.bg-opacity-10 {
- background-color: color-mix(in oklab, var(--color-fuchsia-500) 10%, transparent);
-}
-
-.bg-fuchsia-500.bg-opacity-20 {
- background-color: color-mix(in oklab, var(--color-fuchsia-500) 20%, transparent);
-}
-
-.bg-fuchsia-500.bg-opacity-40 {
- background-color: color-mix(in oklab, var(--color-fuchsia-500) 40%, transparent);
-}
-
-.hover\:bg-fuchsia-500.hover\:bg-opacity-20:hover {
- background-color: color-mix(in oklab, var(--color-fuchsia-500) 20%, transparent);
-}
-
-.group:hover .bg-fuchsia-500.group-hover\:bg-opacity-30 {
- background-color: color-mix(in oklab, var(--color-fuchsia-500) 30%, transparent);
-}
-
-.ring-fuchsia-500.ring-opacity-20 {
- --tw-ring-color: color-mix(in oklab, var(--color-fuchsia-500) 20%, transparent);
-}
-
-.ring-fuchsia-300.ring-opacity-40 {
- --tw-ring-color: color-mix(in oklab, var(--color-fuchsia-300) 40%, transparent);
-}
-
-.bg-pink-500.bg-opacity-10 {
- background-color: color-mix(in oklab, var(--color-pink-500) 10%, transparent);
-}
-
-.bg-pink-500.bg-opacity-20 {
- background-color: color-mix(in oklab, var(--color-pink-500) 20%, transparent);
-}
-
-.bg-pink-500.bg-opacity-40 {
- background-color: color-mix(in oklab, var(--color-pink-500) 40%, transparent);
-}
-
-.hover\:bg-pink-500.hover\:bg-opacity-20:hover {
- background-color: color-mix(in oklab, var(--color-pink-500) 20%, transparent);
-}
-
-.group:hover .bg-pink-500.group-hover\:bg-opacity-30 {
- background-color: color-mix(in oklab, var(--color-pink-500) 30%, transparent);
-}
-
-.ring-pink-500.ring-opacity-20 {
- --tw-ring-color: color-mix(in oklab, var(--color-pink-500) 20%, transparent);
-}
-
-.ring-pink-300.ring-opacity-40 {
- --tw-ring-color: color-mix(in oklab, var(--color-pink-300) 40%, transparent);
-}
-
-.bg-rose-500.bg-opacity-10 {
- background-color: color-mix(in oklab, var(--color-rose-500) 10%, transparent);
-}
-
-.bg-rose-500.bg-opacity-20 {
- background-color: color-mix(in oklab, var(--color-rose-500) 20%, transparent);
-}
-
-.bg-rose-500.bg-opacity-40 {
- background-color: color-mix(in oklab, var(--color-rose-500) 40%, transparent);
-}
-
-.hover\:bg-rose-500.hover\:bg-opacity-20:hover {
- background-color: color-mix(in oklab, var(--color-rose-500) 20%, transparent);
-}
-
-.group:hover .bg-rose-500.group-hover\:bg-opacity-30 {
- background-color: color-mix(in oklab, var(--color-rose-500) 30%, transparent);
-}
-
-.ring-rose-500.ring-opacity-20 {
- --tw-ring-color: color-mix(in oklab, var(--color-rose-500) 20%, transparent);
-}
-
-.ring-rose-300.ring-opacity-40 {
- --tw-ring-color: color-mix(in oklab, var(--color-rose-300) 40%, transparent);
-}
diff --git a/ui/litellm-dashboard/src/components/ModelInfoEditForm.tsx b/ui/litellm-dashboard/src/components/ModelInfoEditForm.tsx
new file mode 100644
index 00000000000..fff732fe270
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/ModelInfoEditForm.tsx
@@ -0,0 +1,817 @@
+"use client";
+
+import { zodResolver } from "@hookform/resolvers/zod";
+// eslint-disable-next-line no-restricted-imports -- the dashboard has no shadcn date-time picker; the PTU window fields need one
+import { DatePicker } from "antd";
+import { CircleHelp } from "lucide-react";
+import type { Dayjs } from "dayjs";
+import * as React from "react";
+import { useForm, type Resolver } from "react-hook-form";
+import { z } from "zod/v4";
+
+import { TagsInput } from "@/app/(dashboard)/guardrails/_components/content_filter/TagsInput";
+import { FormField } from "@/components/shared/form/FormField";
+import { Badge } from "@/components/ui/badge";
+import { Button } from "@/components/ui/button";
+import { Input } from "@/components/ui/input";
+import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
+import { Switch } from "@/components/ui/switch";
+import { Textarea } from "@/components/ui/textarea";
+import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
+import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner";
+
+import CacheControlInjectionPoints, {
+ CACHE_CONTROL_LABEL,
+ CACHE_CONTROL_TOOLTIP,
+ type CacheControlInjectionPoint,
+} from "./add_model/cache_control_settings";
+import type { CredentialItem } from "./networking";
+import NumericalInput from "./shared/numerical_input";
+import type { Tag } from "./tag_management/types";
+import VectorStoreSelector from "./vector_store_management/VectorStoreSelector";
+import { formatPtuUtcDisplay, utcIsoToPickerValue } from "../utils/ptuDatetime";
+import { isMaskedSecret } from "../utils/maskedSecretUtils";
+import {
+ MAX_COST_PER_PTU_PER_HOUR,
+ MAX_PTU_COUNT,
+ PTU_COUNT_FIELD,
+ PTU_END_FIELD,
+ PTU_RATE_FIELD,
+ PTU_START_FIELD,
+ isFilledPtuValue,
+ isNonNegativePtuRate,
+ isPositiveWholePtuCount,
+ ptuWindowIsOrdered,
+} from "../utils/ptuValidation";
+
+interface PtuEditField {
+ name: string;
+ label: string;
+ input: "number" | "datetime";
+ placeholder?: string;
+ isCount?: boolean;
+}
+
+const PTU_EDIT_FIELDS: PtuEditField[] = [
+ { name: PTU_COUNT_FIELD, label: "PTU Count", input: "number", placeholder: "e.g. 15", isCount: true },
+ { name: PTU_RATE_FIELD, label: "Cost per PTU / Hour (USD)", input: "number", placeholder: "e.g. 2.00" },
+ { name: PTU_START_FIELD, label: "PTU Effective From (UTC)", input: "datetime" },
+ { name: PTU_END_FIELD, label: "PTU Effective To (UTC)", input: "datetime" },
+];
+
+export type TouchedPricingField = "input_cost" | "output_cost" | "cache_read_cost" | "cache_write_cost";
+
+const PRICING_FIELDS: readonly TouchedPricingField[] = [
+ "input_cost",
+ "output_cost",
+ "cache_read_cost",
+ "cache_write_cost",
+] as const;
+
+const COST_SOURCES: Record = {
+ input_cost: { param: "input_cost_per_token", info: "input_cost_per_token" },
+ output_cost: { param: "output_cost_per_token", info: "output_cost_per_token" },
+ cache_read_cost: { param: "cache_read_input_token_cost", info: "cache_read_input_token_cost" },
+ cache_write_cost: { param: "cache_creation_input_token_cost", info: "cache_creation_input_token_cost" },
+};
+
+export interface ModelEditFormValues {
+ model_name?: string;
+ litellm_model_name?: string;
+ api_base?: string;
+ custom_llm_provider?: string;
+ organization?: string;
+ tpm?: string | number | null;
+ rpm?: string | number | null;
+ max_retries?: string | number | null;
+ timeout?: string | number | null;
+ stream_timeout?: string | number | null;
+ input_cost?: string | number | null;
+ output_cost?: string | number | null;
+ cache_read_cost?: string | number | null;
+ cache_write_cost?: string | number | null;
+ ptu_count?: string | number | null;
+ cost_per_ptu_per_hour?: string | number | null;
+ ptu_effective_from?: Dayjs | null;
+ ptu_effective_to?: Dayjs | null;
+ cache_control?: boolean;
+ cache_control_injection_points?: CacheControlInjectionPoint[];
+ model_access_group?: string[];
+ guardrails?: string[];
+ vector_store_ids?: string[];
+ tags?: string[];
+ health_check_model?: string | null;
+ litellm_credential_name?: string;
+ litellm_extra_params?: string;
+ model_info?: string;
+}
+
+type ModelEditFieldName = keyof ModelEditFormValues;
+
+const scalar = z.union([z.string(), z.number(), z.null()]).optional();
+const textish = z.string().optional();
+
+const modelEditShape = {
+ model_name: textish,
+ litellm_model_name: textish,
+ api_base: textish,
+ custom_llm_provider: textish,
+ organization: textish,
+ tpm: scalar,
+ rpm: scalar,
+ max_retries: scalar,
+ timeout: scalar,
+ stream_timeout: scalar,
+ input_cost: scalar,
+ output_cost: scalar,
+ cache_read_cost: scalar,
+ cache_write_cost: scalar,
+ ptu_count: scalar,
+ cost_per_ptu_per_hour: scalar,
+ ptu_effective_from: z.custom().nullish(),
+ ptu_effective_to: z.custom().nullish(),
+ cache_control: z.boolean().optional(),
+ cache_control_injection_points: z.array(z.custom()).optional(),
+ model_access_group: z.array(z.string()).optional(),
+ guardrails: z.array(z.string()).optional(),
+ vector_store_ids: z.array(z.string()).optional(),
+ tags: z.array(z.string()).optional(),
+ health_check_model: z.string().nullish(),
+ litellm_credential_name: textish,
+ litellm_extra_params: textish,
+ model_info: textish,
+};
+
+const isJson = (value: string): boolean => {
+ try {
+ JSON.parse(value);
+ return true;
+ } catch {
+ return false;
+ }
+};
+
+const buildSchema = (ptuEnabled: boolean, isFieldTouched: (field: TouchedPricingField) => boolean) =>
+ z.object(modelEditShape).superRefine((values, ctx) => {
+ const reject = (path: ModelEditFieldName, message: string) =>
+ ctx.addIssue({ code: "custom", path: [path], message });
+
+ if (values.litellm_extra_params && !isJson(values.litellm_extra_params)) {
+ reject("litellm_extra_params", "Please enter valid JSON");
+ }
+
+ // antd validates only mounted fields, and the PTU block does not render when the flag is off.
+ if (!ptuEnabled) {
+ return;
+ }
+
+ if (!isPositiveWholePtuCount(values.ptu_count)) {
+ reject("ptu_count", `PTU Count must be a whole number between 1 and ${MAX_PTU_COUNT.toLocaleString()}`);
+ }
+ if (!isNonNegativePtuRate(values.cost_per_ptu_per_hour)) {
+ reject(
+ "cost_per_ptu_per_hour",
+ `Cost per PTU / Hour must be between 0 and ${MAX_COST_PER_PTU_PER_HOUR.toLocaleString()}`,
+ );
+ }
+ if (isFilledPtuValue(values.ptu_count) !== isFilledPtuValue(values.cost_per_ptu_per_hour)) {
+ const message = "PTU Count and Cost per PTU / Hour must be set together";
+ reject("ptu_count", message);
+ reject("cost_per_ptu_per_hour", message);
+ }
+ if (isFilledPtuValue(values.ptu_count) && !isFilledPtuValue(values.ptu_effective_from)) {
+ reject("ptu_effective_from", "PTU Effective From is required when PTU Count is set");
+ }
+ if (!ptuWindowIsOrdered(values.ptu_effective_from, values.ptu_effective_to)) {
+ const message = "PTU Effective To must be after PTU Effective From";
+ reject("ptu_effective_from", message);
+ reject("ptu_effective_to", message);
+ }
+
+ for (const field of PRICING_FIELDS) {
+ const value = values[field];
+ if (
+ isFieldTouched(field) &&
+ isFilledPtuValue(values.ptu_count) &&
+ isFilledPtuValue(value) &&
+ Number(value) !== 0
+ ) {
+ reject(field, "A PTU deployment bills by reserved capacity, so this cost must be 0 or blank");
+ }
+ }
+ });
+
+const perMillionTokens = (...rates: (number | null | undefined)[]): number | null => {
+ const rate = rates.find((candidate) => candidate != null);
+ return rate == null ? null : rate * 1_000_000;
+};
+
+export const toModelEditFormValues = (localModelData: any, isWildcardModel: boolean): ModelEditFormValues => ({
+ model_name: localModelData.model_name,
+ litellm_model_name: localModelData.litellm_model_name,
+ api_base: localModelData.litellm_params.api_base,
+ custom_llm_provider: localModelData.litellm_params.custom_llm_provider,
+ organization: localModelData.litellm_params.organization,
+ tpm: localModelData.litellm_params.tpm,
+ rpm: localModelData.litellm_params.rpm,
+ max_retries: localModelData.litellm_params.max_retries,
+ timeout: localModelData.litellm_params.timeout,
+ stream_timeout: localModelData.litellm_params.stream_timeout,
+ input_cost: perMillionTokens(
+ localModelData.litellm_params.input_cost_per_token,
+ localModelData.model_info?.input_cost_per_token,
+ ),
+ output_cost: perMillionTokens(
+ localModelData.litellm_params?.output_cost_per_token,
+ localModelData.model_info?.output_cost_per_token,
+ ),
+ ptu_count: localModelData.model_info?.ptu_count ?? null,
+ cost_per_ptu_per_hour: localModelData.model_info?.cost_per_ptu_per_hour ?? null,
+ ptu_effective_from: utcIsoToPickerValue(localModelData.model_info?.ptu_effective_from),
+ ptu_effective_to: utcIsoToPickerValue(localModelData.model_info?.ptu_effective_to),
+ cache_read_cost: perMillionTokens(
+ localModelData.litellm_params?.cache_read_input_token_cost,
+ localModelData.model_info?.cache_read_input_token_cost,
+ ),
+ cache_write_cost: perMillionTokens(
+ localModelData.litellm_params?.cache_creation_input_token_cost,
+ localModelData.model_info?.cache_creation_input_token_cost,
+ ),
+ cache_control: localModelData.litellm_params?.cache_control_injection_points ? true : false,
+ cache_control_injection_points: localModelData.litellm_params?.cache_control_injection_points || [],
+ model_access_group: Array.isArray(localModelData.model_info?.access_groups)
+ ? localModelData.model_info.access_groups
+ : [],
+ guardrails: Array.isArray(localModelData.litellm_params?.guardrails) ? localModelData.litellm_params.guardrails : [],
+ vector_store_ids:
+ Array.isArray(localModelData.litellm_params?.vector_store_ids) &&
+ localModelData.litellm_params.vector_store_ids.length > 0
+ ? localModelData.litellm_params.vector_store_ids
+ : undefined,
+ tags: Array.isArray(localModelData.litellm_params?.tags) ? localModelData.litellm_params.tags : [],
+ // antd never mounted this field for a non-wildcard model, so the key must be absent, not null.
+ ...(isWildcardModel ? { health_check_model: localModelData.model_info?.health_check_model } : {}),
+ litellm_credential_name: localModelData.litellm_params?.litellm_credential_name || "",
+ litellm_extra_params: JSON.stringify(
+ Object.fromEntries(
+ Object.entries(localModelData.litellm_params || {}).filter(
+ ([key, value]) => key !== "litellm_credential_name" && !isMaskedSecret(value),
+ ),
+ ),
+ null,
+ 2,
+ ),
+});
+
+const displayCost = (localModelData: any, field: TouchedPricingField): string => {
+ const { param, info } = COST_SOURCES[field];
+ const rate = localModelData?.litellm_params?.[param] ?? localModelData?.model_info?.[info];
+ return rate != null ? (Number(rate) * 1_000_000).toFixed(4) : "Not Set";
+};
+
+interface ModelInfoEditFormProps {
+ localModelData: any;
+ modelData: { model_info: { team_id?: string | null } & Record };
+ accessToken: string | null;
+ isEditing: boolean;
+ isSaving: boolean;
+ isWildcardModel: boolean;
+ ptuCostAttributionEnabled: boolean;
+ showCacheControl: boolean;
+ setShowCacheControl: (checked: boolean) => void;
+ onCancel: () => void;
+ onSubmit: (values: ModelEditFormValues, isFieldTouched: (field: TouchedPricingField) => boolean) => Promise;
+ modelAccessGroups: string[] | null;
+ guardrailsList: string[];
+ tagsList: Record;
+ credentialsList: CredentialItem[];
+ healthCheckModelOptions: { value: string; label: string }[];
+}
+
+const Display: React.FC<{ children: React.ReactNode }> = ({ children }) => (
+ {children}
+);
+
+const FieldLabel: React.FC<{ children: React.ReactNode }> = ({ children }) => (
+ {children}
+);
+
+const Hint: React.FC<{ text: string }> = ({ text }) => (
+
+ }
+ />
+ {text}
+
+);
+
+const DocsHint: React.FC<{ text: string; href: string }> = ({ text, href }) => (
+ event.stopPropagation()}>
+
+
+);
+
+const ChipList: React.FC<{ values: unknown; emptyLabel: string }> = ({ values, emptyLabel }) => {
+ if (!values) {
+ return <>Not Set>;
+ }
+ if (!Array.isArray(values)) {
+ return <>{String(values)}>;
+ }
+ if (values.length === 0) {
+ return <>{emptyLabel}>;
+ }
+ return (
+
+ {values.map((entry: string, index: number) => (
+
+ {entry}
+
+ ))}
+
+ );
+};
+
+const ModelInfoEditForm: React.FC = ({
+ localModelData,
+ modelData,
+ accessToken,
+ isEditing,
+ isSaving,
+ isWildcardModel,
+ ptuCostAttributionEnabled,
+ showCacheControl,
+ setShowCacheControl,
+ onCancel,
+ onSubmit,
+ modelAccessGroups,
+ guardrailsList,
+ tagsList,
+ credentialsList,
+ healthCheckModelOptions,
+}) => {
+ // Neither RHF's blur-based touchedFields nor its resettable dirtyFields matches antd's touched-on-change.
+ const touchedRef = React.useRef>(new Set());
+ const isFieldTouched = React.useCallback((field: TouchedPricingField) => touchedRef.current.has(field), []);
+ const markTouched = (field: string) => {
+ touchedRef.current = new Set([...touchedRef.current, field]);
+ };
+
+ // react-hook-form refreshes control._options every render, so this rebuild is what the next submit runs.
+ const resolver: Resolver = (values, context, options) =>
+ zodResolver(buildSchema(ptuCostAttributionEnabled, isFieldTouched))(values, context, options);
+
+ const form = useForm({
+ resolver,
+ defaultValues: toModelEditFormValues(localModelData, isWildcardModel),
+ });
+
+ const submit = (event: React.FormEvent) =>
+ form.handleSubmit(async (values) => {
+ await onSubmit(values, isFieldTouched);
+ })(event);
+
+ const cancel = () => {
+ form.reset(toModelEditFormValues(localModelData, isWildcardModel));
+ touchedRef.current = new Set();
+ onCancel();
+ };
+
+ const textField = (name: ModelEditFieldName, label: string, placeholder: string, stored: unknown) => (
+
+ {label}
+ {isEditing ? (
+
+ {({ value, ...control }) => }
+
+ ) : (
+ {(stored as string) || "Not Set"}
+ )}
+
+ );
+
+ const numberField = (name: ModelEditFieldName, label: string, placeholder: string, stored: unknown) => (
+
+ {label}
+ {isEditing ? (
+
+ {({ value, ...control }) => }
+
+ ) : (
+ {(stored as string) || "Not Set"}
+ )}
+
+ );
+
+ const pricingField = (name: TouchedPricingField, label: string, placeholder: string, description?: string) =>
+ isEditing ? (
+
+ {({ value, onChange, ...control }) => (
+ ) => {
+ markTouched(name);
+ onChange(event);
+ }}
+ />
+ )}
+
+ ) : (
+
+ {label}
+ {displayCost(localModelData, name)}
+
+ );
+
+ const tagsField = (
+ name: "model_access_group" | "guardrails" | "tags",
+ options: { value: string; label: string }[],
+ placeholder: string,
+ ) => (
+
+ {({ id, value, onChange }) => (
+
+ )}
+
+ );
+
+ return (
+
+
+
+
+ {textField("model_name", "Model Name", "Enter model name", localModelData.model_name)}
+ {textField(
+ "litellm_model_name",
+ "LiteLLM Model Name",
+ "Enter LiteLLM model name",
+ localModelData.litellm_model_name,
+ )}
+
+ {pricingField("input_cost", "Input Cost (per 1M tokens)", "Enter input cost")}
+ {pricingField("output_cost", "Output Cost (per 1M tokens)", "Enter output cost")}
+
+ {ptuCostAttributionEnabled &&
+ PTU_EDIT_FIELDS.map((ptuField) => (
+
+ {ptuField.label}
+ {isEditing ? (
+
+ {({ value, onChange, ...control }) =>
+ ptuField.input === "number" ? (
+
+ ) : (
+
+ )
+ }
+
+ ) : (
+
+ {(ptuField.input === "datetime"
+ ? formatPtuUtcDisplay(localModelData?.model_info?.[ptuField.name])
+ : localModelData?.model_info?.[ptuField.name]) ?? "Not Set"}
+
+ )}
+
+ ))}
+
+ {pricingField(
+ "cache_read_cost",
+ "Cache Read Cost (per 1M tokens)",
+ "Defaults to Input Cost if blank",
+ "If left blank on save, defaults to Input Cost.",
+ )}
+ {pricingField(
+ "cache_write_cost",
+ "Cache Write Cost (per 1M tokens)",
+ "Defaults to Input Cost if blank",
+ "If left blank on save, defaults to Input Cost (backend falls back to input_cost_per_token).",
+ )}
+
+ {textField("api_base", "API Base", "Enter API base", localModelData.litellm_params?.api_base)}
+ {textField(
+ "custom_llm_provider",
+ "Custom LLM Provider",
+ "Enter custom LLM provider",
+ localModelData.litellm_params?.custom_llm_provider,
+ )}
+ {textField(
+ "organization",
+ "Organization",
+ "Enter organization",
+ localModelData.litellm_params?.organization,
+ )}
+
+ {numberField("tpm", "TPM (Tokens per Minute)", "Enter TPM", localModelData.litellm_params?.tpm)}
+ {numberField("rpm", "RPM (Requests per Minute)", "Enter RPM", localModelData.litellm_params?.rpm)}
+ {numberField("max_retries", "Max Retries", "Enter max retries", localModelData.litellm_params?.max_retries)}
+ {numberField("timeout", "Timeout (seconds)", "Enter timeout", localModelData.litellm_params?.timeout)}
+ {numberField(
+ "stream_timeout",
+ "Stream Timeout (seconds)",
+ "Enter stream timeout",
+ localModelData.litellm_params?.stream_timeout,
+ )}
+
+
+ Model Access Groups
+ {isEditing ? (
+ tagsField(
+ "model_access_group",
+ (modelAccessGroups ?? []).map((group) => ({ value: group, label: group })),
+ "Select existing groups or type to create new ones",
+ )
+ ) : (
+
+
+
+ )}
+
+
+
+
+ Guardrails
+
+
+ {isEditing ? (
+ tagsField(
+ "guardrails",
+ guardrailsList.map((name) => ({ value: name, label: name })),
+ "Select existing guardrails or type to create new ones",
+ )
+ ) : (
+
+
+
+ )}
+
+
+
+
+ Attached Knowledge Bases (RAG)
+
+
+ {isEditing ? (
+
+ {({ value, onChange }) => (
+
+ )}
+
+ ) : (
+
+
+
+ )}
+
+
+
+ Tags
+ {isEditing ? (
+ tagsField(
+ "tags",
+ Object.values(tagsList).map((tag: Tag) => ({ value: tag.name, label: tag.name })),
+ "Select existing tags or type to create new ones",
+ )
+ ) : (
+
+
+
+ )}
+
+
+
+ Existing Credentials
+ {isEditing ? (
+
+ {({ id, value, onChange, onBlur }) => {
+ const items = [
+ { value: "", label: "None" },
+ ...credentialsList.map((credential) => ({
+ value: credential.credential_name,
+ label: credential.credential_name,
+ })),
+ ];
+ return (
+ onChange(selected ?? "")}
+ >
+
+
+
+
+ {items.map((item) => (
+
+ {item.label}
+
+ ))}
+
+
+ );
+ }}
+
+ ) : (
+ {localModelData.litellm_params?.litellm_credential_name || "Manual"}
+ )}
+
+
+ {isWildcardModel && (
+
+ Health Check Model
+ {isEditing ? (
+
+ {({ id, value, onChange, onBlur }) => (
+
+
+
+
+
+ None
+ {healthCheckModelOptions.map((option) => (
+
+ {option.label}
+
+ ))}
+
+
+ )}
+
+ ) : (
+ {localModelData.model_info?.health_check_model || "Not Set"}
+ )}
+
+ )}
+
+ {isEditing ? (
+ <>
+
+ {CACHE_CONTROL_LABEL}
+
+ >
+ }
+ orientation="horizontal"
+ >
+ {({ id, value, onChange, onBlur }) => (
+ {
+ onChange(checked);
+ setShowCacheControl(checked);
+ }}
+ />
+ )}
+
+ {showCacheControl && (
+
+ {({ value, onChange }) => (
+
+ )}
+
+ )}
+ >
+ ) : (
+
+
Cache Control
+
+ {localModelData.litellm_params?.cache_control_injection_points ? (
+
+
Enabled
+
+ {localModelData.litellm_params.cache_control_injection_points.map((point: any, i: number) => (
+
+ Location: {point.location},{point.role && Role: {point.role} }
+ {point.index !== undefined && Index: {point.index} }
+
+ ))}
+
+
+ ) : (
+ "Disabled"
+ )}
+
+
+ )}
+
+
+
Model Info
+ {isEditing ? (
+
+ {({ value, ...control }) => (
+
+ )}
+
+ ) : (
+
+
+ {JSON.stringify(localModelData.model_info, null, 2)}
+
+
+ )}
+
+
+
+
+ LiteLLM Params
+
+
+ {isEditing ? (
+
+ {({ value, ...control }) => (
+
+ )}
+
+ ) : (
+
+
+ {JSON.stringify(localModelData.litellm_params, null, 2)}
+
+
+ )}
+
+
+
+ Team ID
+ {modelData.model_info.team_id || "Not Set"}
+
+
+
+ {isEditing && (
+
+
+ Cancel
+
+
+ {isSaving && }
+ Save Changes
+
+
+ )}
+
+
+
+ );
+};
+
+export default ModelInfoEditForm;
diff --git a/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.tsx b/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.tsx
index d3ede5f4318..c3936cc456d 100644
--- a/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.tsx
+++ b/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.tsx
@@ -40,6 +40,7 @@ export const MODEL_SENTINEL_OPTIONS = [
const MAX_VISIBLE_MODEL_CHIPS = 5;
export interface ModelSelectProps {
+ id?: string;
teamID?: string;
organizationID?: string;
options?: {
@@ -122,7 +123,7 @@ const filterModels = (
export const ModelSelect = (props: ModelSelectProps) => {
const anchor = useComboboxAnchor();
- const { teamID, organizationID, options, context, dataTestId, value = [], onChange, style } = props;
+ const { id, teamID, organizationID, options, context, dataTestId, value = [], onChange, style } = props;
const { showAllProxyModelsOverride, includeSpecialOptions } = options || {};
const { data: allProxyModels, isLoading: isLoadingAllProxyModels } = useAllProxyModels();
const { data: team, isLoading: isLoadingTeam } = useTeam(teamID);
@@ -256,7 +257,7 @@ export const ModelSelect = (props: ModelSelectProps) => {
>
)}
-
+
No models found
diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx
index 381818a53f5..e3a36f4fbf2 100644
--- a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx
+++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx
@@ -3,7 +3,8 @@
* Handles primary model selection and fallback chain configuration
*/
-import { Select, Tooltip } from "antd";
+import { SimpleTooltip } from "@/components/ui/tooltip";
+import { Select } from "antd";
import { AlertCircle, ArrowDown, X } from "lucide-react";
import React from "react";
@@ -142,12 +143,9 @@ export function FallbackGroupConfig({
}}
maxTagCount="responsive"
maxTagPlaceholder={(omittedValues) => (
- value).join(", ")}
- >
+ value).join(", ")}>
+{omittedValues.length} more
-
+
)}
showSearch
filterOption={(input, option) => (option?.label ?? "").toLowerCase().includes(input.toLowerCase())}
diff --git a/ui/litellm-dashboard/src/components/Teams.test.tsx b/ui/litellm-dashboard/src/components/Teams.test.tsx
index 0ef4357e73f..d0d41837dce 100644
--- a/ui/litellm-dashboard/src/components/Teams.test.tsx
+++ b/ui/litellm-dashboard/src/components/Teams.test.tsx
@@ -1223,3 +1223,261 @@ describe("Teams - which fields reach the create payload depends on the open sect
expect(payload.team_id).toBe("tid-kept");
});
});
+
+describe("Teams - the exact bytes the create call sends", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ can.mockReturnValue(true);
+ vi.mocked(fetchAvailableModelsForTeamOrKey).mockResolvedValue(["gpt-4"]);
+ vi.mocked(fetchMCPAccessGroups).mockResolvedValue([]);
+ vi.mocked(getGuardrailsList).mockResolvedValue({ guardrails: [] });
+ vi.mocked(getPoliciesList).mockResolvedValue({ policies: [] });
+ vi.mocked(getDefaultTeamSettings).mockResolvedValue({ values: {} });
+ vi.mocked(teamCreateCall).mockResolvedValue({ team_id: "new-team-1" });
+ vi.mocked(useTeamMetadataSchema).mockReturnValue({ data: [], isLoading: false } as any);
+ mockUseOrganizations.mockReturnValue({ data: null });
+ });
+
+ const openCreateModal = async (options?: { premiumUser?: boolean }) => {
+ renderWithQueryClient(
+ ,
+ );
+ act(() => {
+ fireEvent.click(screen.getAllByRole("button", { name: /create team/i })[0]);
+ });
+ await waitFor(() => {
+ expect(screen.getByLabelText(/team name/i)).toBeInTheDocument();
+ });
+ fireEvent.change(screen.getByLabelText(/team name/i), { target: { value: "Byte Contract Team" } });
+ };
+
+ const submit = async () => {
+ const buttons = screen.getAllByRole("button", { name: /create team/i });
+ fireEvent.click(buttons[buttons.length - 1]);
+ await waitFor(() => {
+ expect(teamCreateCall).toHaveBeenCalled();
+ });
+ return vi.mocked(teamCreateCall).mock.calls[0][1] as Record;
+ };
+
+ const wireBody = (payload: Record) => JSON.parse(JSON.stringify(payload)) as Record;
+
+ const openSection = async (title: string, mountedProbe: RegExp | string) => {
+ fireEvent.click(screen.getByText(title));
+ await waitFor(() => {
+ expect(screen.getAllByText(mountedProbe).length).toBeGreaterThan(0);
+ });
+ };
+
+ it("sends three keys and nothing else when every section is left closed", async () => {
+ await openCreateModal();
+
+ const payload = await submit();
+
+ expect(payload).toStrictEqual({
+ team_alias: "Byte Contract Team",
+ organization_id: null,
+ models: ["no-default-models"],
+ max_budget: undefined,
+ budget_duration: undefined,
+ tpm_limit: undefined,
+ rpm_limit: undefined,
+ metadata: undefined,
+ });
+ expect(wireBody(payload)).toStrictEqual({
+ team_alias: "Byte Contract Team",
+ organization_id: null,
+ models: ["no-default-models"],
+ });
+ });
+
+ it("keeps every newly mounted but untouched field out of the request body", async () => {
+ await openCreateModal();
+
+ await openSection("Additional Settings", /Team Member Key Duration/);
+ await openSection("MCP Settings", /Allowed MCP Servers/);
+ await openSection("Agent Settings", /Allowed Agents/);
+ await openSection("Search Tool Settings", /Allowed Search Tools/);
+
+ const payload = await submit();
+
+ expect(payload).toStrictEqual({
+ team_alias: "Byte Contract Team",
+ organization_id: null,
+ models: ["no-default-models"],
+ max_budget: undefined,
+ budget_duration: undefined,
+ tpm_limit: undefined,
+ rpm_limit: undefined,
+ metadata: undefined,
+ team_id: undefined,
+ team_member_budget: undefined,
+ team_member_key_duration: undefined,
+ team_member_rpm_limit: undefined,
+ team_member_tpm_limit: undefined,
+ secret_manager_settings: undefined,
+ guardrails: undefined,
+ disable_global_guardrails: undefined,
+ policies: undefined,
+ access_group_ids: undefined,
+ allowed_vector_store_ids: undefined,
+ allowed_passthrough_routes: undefined,
+ allowed_mcp_servers_and_groups: undefined,
+ mcp_tool_permissions: {},
+ allowed_agents_and_groups: undefined,
+ object_permission_search_tools: undefined,
+ });
+ expect(wireBody(payload)).toStrictEqual({
+ team_alias: "Byte Contract Team",
+ organization_id: null,
+ models: ["no-default-models"],
+ mcp_tool_permissions: {},
+ });
+ });
+
+ it.each([
+ ["MCP Settings", /Allowed MCP Servers/, ["allowed_mcp_servers_and_groups", "mcp_tool_permissions"]],
+ ["Agent Settings", /Allowed Agents/, ["allowed_agents_and_groups"]],
+ ["Search Tool Settings", /Allowed Search Tools/, ["object_permission_search_tools"]],
+ ])("registers %s fields only while that one section is open", async (title, probe, keys) => {
+ await openCreateModal();
+
+ const closedPayload = await submit();
+ for (const key of keys as string[]) {
+ expect(closedPayload).not.toHaveProperty(key);
+ }
+ });
+
+ it("carries every typed value to the payload at the type antd sends today", async () => {
+ await openCreateModal();
+
+ fireEvent.change(screen.getByLabelText("Max Budget (USD)"), { target: { value: "150.75" } });
+ fireEvent.change(screen.getByLabelText("Tokens per minute Limit (TPM)"), { target: { value: "900" } });
+ fireEvent.change(screen.getByLabelText("Requests per minute Limit (RPM)"), { target: { value: "800" } });
+
+ await openSection("Additional Settings", /Team Member Key Duration/);
+
+ fireEvent.change(screen.getByLabelText("Team ID"), { target: { value: "tid-1" } });
+ fireEvent.change(screen.getByLabelText("Team Member Budget (USD)"), { target: { value: "12.5" } });
+ fireEvent.change(screen.getByLabelText(/Team Member Key Duration/), { target: { value: "30d" } });
+ fireEvent.change(screen.getByLabelText("Team Member RPM Limit"), { target: { value: "7" } });
+ fireEvent.change(screen.getByLabelText("Team Member TPM Limit"), { target: { value: "8" } });
+ fireEvent.change(screen.getByLabelText("Secret Manager Settings"), {
+ target: { value: '{"namespace":"admin"}' },
+ });
+
+ const payload = await submit();
+
+ expect(payload.max_budget).toBe("150.75");
+ expect(payload.tpm_limit).toBe("900");
+ expect(payload.rpm_limit).toBe("800");
+ expect(payload.team_id).toBe("tid-1");
+ expect(payload.team_member_budget).toBe(12.5);
+ expect(payload.team_member_key_duration).toBe("30d");
+ expect(payload.team_member_rpm_limit).toBe("7");
+ expect(payload.team_member_tpm_limit).toBe("8");
+ expect(payload.secret_manager_settings).toStrictEqual({ namespace: "admin" });
+ });
+
+ it("blocks the create on an invalid secret manager config, with the rule message suppressed by help", async () => {
+ await openCreateModal();
+ await openSection("Additional Settings", /Team Member Key Duration/);
+
+ fireEvent.change(screen.getByLabelText("Secret Manager Settings"), { target: { value: " " } });
+
+ const buttons = screen.getAllByRole("button", { name: /create team/i });
+ fireEvent.click(buttons[buttons.length - 1]);
+
+ await waitFor(() => {
+ expect(screen.getByLabelText("Secret Manager Settings")).toHaveAttribute("aria-invalid", "true");
+ });
+ expect(teamCreateCall).not.toHaveBeenCalled();
+ expect(screen.queryByText("Please enter valid JSON")).not.toBeInTheDocument();
+ });
+
+ it("turns the disable-global-guardrails switch into a boolean for a premium user", async () => {
+ await openCreateModal({ premiumUser: true });
+ await openSection("Additional Settings", /Team Member Key Duration/);
+
+ const switches = screen.getAllByRole("switch");
+ fireEvent.click(switches[switches.length - 1]);
+
+ const payload = await submit();
+
+ expect(payload.disable_global_guardrails).toBe(true);
+ });
+
+ it("leaves the disable-global-guardrails switch inert for a non-premium user", async () => {
+ await openCreateModal();
+ await openSection("Additional Settings", /Team Member Key Duration/);
+
+ const switches = screen.getAllByRole("switch");
+ fireEvent.click(switches[switches.length - 1]);
+
+ const payload = await submit();
+
+ expect(payload.disable_global_guardrails).toBeUndefined();
+ });
+
+ it.each([
+ ["MCP Settings", /Allowed MCP Servers/, ["allowed_mcp_servers_and_groups", "mcp_tool_permissions"]],
+ ["Agent Settings", /Allowed Agents/, ["allowed_agents_and_groups"]],
+ ["Search Tool Settings", /Allowed Search Tools/, ["object_permission_search_tools"]],
+ ])("adds the %s keys as soon as that one section is opened", async (title, probe, keys) => {
+ await openCreateModal();
+
+ await openSection(title as string, probe as RegExp);
+ const payload = await submit();
+
+ for (const key of keys as string[]) {
+ expect(payload).toHaveProperty(key);
+ }
+ });
+
+ it("leaves policies out of the request body for a caller without the viewPolicies capability", async () => {
+ can.mockReturnValue(false);
+
+ await openCreateModal();
+ await openSection("Additional Settings", /Team Member Key Duration/);
+
+ const payload = await submit();
+
+ expect(payload).toStrictEqual({
+ team_alias: "Byte Contract Team",
+ organization_id: null,
+ models: ["no-default-models"],
+ max_budget: undefined,
+ budget_duration: undefined,
+ tpm_limit: undefined,
+ rpm_limit: undefined,
+ metadata: undefined,
+ team_id: undefined,
+ team_member_budget: undefined,
+ team_member_key_duration: undefined,
+ team_member_rpm_limit: undefined,
+ team_member_tpm_limit: undefined,
+ secret_manager_settings: undefined,
+ guardrails: undefined,
+ disable_global_guardrails: undefined,
+ access_group_ids: undefined,
+ allowed_vector_store_ids: undefined,
+ allowed_passthrough_routes: undefined,
+ });
+ });
+
+ it("blocks the create on an empty team name and names the rule", async () => {
+ renderWithQueryClient( );
+ act(() => {
+ fireEvent.click(screen.getAllByRole("button", { name: /create team/i })[0]);
+ });
+ await waitFor(() => {
+ expect(screen.getByLabelText(/team name/i)).toBeInTheDocument();
+ });
+
+ const buttons = screen.getAllByRole("button", { name: /create team/i });
+ fireEvent.click(buttons[buttons.length - 1]);
+
+ expect(await screen.findByText("Please input a team name")).toBeInTheDocument();
+ expect(teamCreateCall).not.toHaveBeenCalled();
+ });
+});
diff --git a/ui/litellm-dashboard/src/components/Teams.tsx b/ui/litellm-dashboard/src/components/Teams.tsx
index b1f96f51028..94e84c6e721 100644
--- a/ui/litellm-dashboard/src/components/Teams.tsx
+++ b/ui/litellm-dashboard/src/components/Teams.tsx
@@ -4,12 +4,21 @@ import AvailableTeamsPanel from "@/components/team/AvailableTeamsPanel";
import TeamInfoView from "@/components/team/TeamInfo";
import TeamSSOSettings from "@/components/TeamSSOSettings";
import { isProxyAdminRole } from "@/utils/roles";
-import { InfoCircleOutlined } from "@ant-design/icons";
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible";
import { Input as UIInput } from "@/components/ui/input";
-import { Button, Form, Input, Layout, Modal, Select, Switch, Tabs, theme, Tooltip, Typography } from "antd";
+import { Switch } from "@/components/ui/switch";
+import { Textarea } from "@/components/ui/textarea";
+import { TooltipProvider } from "@/components/ui/tooltip";
+import { Field, FieldDescription, FieldGroup, FieldLabel } from "@/components/shared/form/field";
+import { FormField } from "@/components/shared/form/FormField";
+import { SearchSelect } from "@/components/shared/SearchSelect";
+import { labelWithDocsHint, labelWithHint } from "@/components/shared/form/LabelWithHint";
+import { useZodForm } from "@/lib/forms/useZodForm";
+import { TagsInput } from "@/app/(dashboard)/guardrails/_components/content_filter/TagsInput";
+import { Layout, Modal, Tabs, theme } from "antd";
import { ChevronDown, Plus, Users } from "lucide-react";
-import React, { useEffect, useState } from "react";
+import React, { useEffect, useMemo, useState } from "react";
+import { z } from "zod/v4";
import { useQuery, useQueryClient } from "@tanstack/react-query";
import { PageHeader } from "@/components/shared/PageHeader";
import { Button as UIButton } from "@/components/ui/button";
@@ -17,7 +26,10 @@ import { teamsTableKeys } from "@/app/(dashboard)/hooks/teams/useTeams";
import { parseAsString, useQueryState } from "nuqs";
import { TeamsTable } from "./TeamsPage/TeamsTable";
import AccessGroupSelector from "./common_components/AccessGroupSelector";
-import MetadataKeyValueFields, { metadataPairsToObject } from "./common_components/MetadataKeyValueFields";
+import MetadataKeyValueFields, {
+ metadataPairsSchema,
+ metadataPairsToObject,
+} from "./common_components/MetadataKeyValueFields";
import { useTeamMetadataSchema } from "@/app/(dashboard)/hooks/teams/useTeamMetadataSchema";
import PassThroughRoutesSelector from "./common_components/PassThroughRoutesSelector";
import AgentSelector from "./agent_management/AgentSelector";
@@ -51,6 +63,102 @@ import { teamCreateCall } from "./networking";
import { normalizeTeamModelSelection } from "./team/teamModelAccess";
import { ModelSelect } from "./ModelSelect/ModelSelect";
+const SUPPRESSED_BY_DESCRIPTION = "";
+
+const numericInputSchema = z.union([z.string(), z.number()]).optional();
+
+const teamCreateFieldsSchema = z.object({
+ team_alias: z.string().min(1, "Please input a team name"),
+ organization_id: z.string().nullish(),
+ models: z.array(z.string()).optional(),
+ max_budget: numericInputSchema,
+ budget_duration: z.string().nullish(),
+ tpm_limit: numericInputSchema,
+ rpm_limit: numericInputSchema,
+ metadata: metadataPairsSchema.optional(),
+ team_id: z.string().optional(),
+ team_member_budget: z.number().optional(),
+ team_member_key_duration: z.string().optional(),
+ team_member_rpm_limit: numericInputSchema,
+ team_member_tpm_limit: numericInputSchema,
+ secret_manager_settings: z.string().optional(),
+ guardrails: z.array(z.string()).optional(),
+ disable_global_guardrails: z.boolean().optional(),
+ policies: z.array(z.string()).optional(),
+ access_group_ids: z.array(z.string()).optional(),
+ allowed_vector_store_ids: z.array(z.string()).optional(),
+ allowed_passthrough_routes: z.array(z.string()).optional(),
+ allowed_mcp_servers_and_groups: z
+ .object({
+ servers: z.array(z.string()),
+ accessGroups: z.array(z.string()),
+ toolsets: z.array(z.string()).optional(),
+ })
+ .optional(),
+ mcp_tool_permissions: z.record(z.string(), z.array(z.string())).optional(),
+ allowed_agents_and_groups: z.object({ agents: z.array(z.string()), accessGroups: z.array(z.string()) }).optional(),
+ object_permission_search_tools: z.array(z.string()).optional(),
+});
+
+type TeamCreateFormValues = z.infer;
+
+const EMPTY_TEAM_CREATE_VALUES: TeamCreateFormValues = {
+ team_alias: "",
+ organization_id: null,
+ models: [],
+ max_budget: undefined,
+ budget_duration: undefined,
+ tpm_limit: undefined,
+ rpm_limit: undefined,
+ metadata: [],
+ team_id: undefined,
+ team_member_budget: undefined,
+ team_member_key_duration: undefined,
+ team_member_rpm_limit: undefined,
+ team_member_tpm_limit: undefined,
+ secret_manager_settings: undefined,
+ guardrails: undefined,
+ disable_global_guardrails: undefined,
+ policies: undefined,
+ access_group_ids: undefined,
+ allowed_vector_store_ids: undefined,
+ allowed_passthrough_routes: undefined,
+ allowed_mcp_servers_and_groups: undefined,
+ mcp_tool_permissions: {},
+ allowed_agents_and_groups: undefined,
+ object_permission_search_tools: undefined,
+};
+
+const ADDITIONAL_SETTINGS_FIELDS = [
+ "team_id",
+ "team_member_budget",
+ "team_member_key_duration",
+ "team_member_rpm_limit",
+ "team_member_tpm_limit",
+ "secret_manager_settings",
+ "guardrails",
+ "disable_global_guardrails",
+ "policies",
+ "access_group_ids",
+ "allowed_vector_store_ids",
+ "allowed_passthrough_routes",
+] as const;
+const MCP_SETTINGS_FIELDS = ["allowed_mcp_servers_and_groups", "mcp_tool_permissions"] as const;
+const AGENT_SETTINGS_FIELDS = ["allowed_agents_and_groups"] as const;
+const SEARCH_TOOL_SETTINGS_FIELDS = ["object_permission_search_tools"] as const;
+
+const isParsableJson = (value: string | undefined): boolean => {
+ if (!value) {
+ return true;
+ }
+ try {
+ JSON.parse(value);
+ return true;
+ } catch {
+ return false;
+ }
+};
+
const canCreateOrManageTeams = (
userRole: string | null,
userID: string | null,
@@ -101,7 +209,29 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser
const [currentOrg] = useState(null);
const [currentOrgForCreateTeam, setCurrentOrgForCreateTeam] = useState(null);
- const [form] = Form.useForm();
+ const isOrgAdmin = userRole !== "Admin";
+ const [additionalSettingsOpen, setAdditionalSettingsOpen] = useState(false);
+ const [mcpSettingsOpen, setMcpSettingsOpen] = useState(false);
+ const [agentSettingsOpen, setAgentSettingsOpen] = useState(false);
+ const [searchToolSettingsOpen, setSearchToolSettingsOpen] = useState(false);
+
+ const teamCreateSchema = useMemo(
+ () =>
+ teamCreateFieldsSchema.superRefine((values, ctx) => {
+ if (isOrgAdmin && !values.organization_id) {
+ ctx.addIssue({ code: "custom", message: SUPPRESSED_BY_DESCRIPTION, path: ["organization_id"] });
+ }
+ if (additionalSettingsOpen && !isParsableJson(values.secret_manager_settings)) {
+ ctx.addIssue({ code: "custom", message: SUPPRESSED_BY_DESCRIPTION, path: ["secret_manager_settings"] });
+ }
+ }),
+ [isOrgAdmin, additionalSettingsOpen],
+ );
+
+ const form = useZodForm(teamCreateSchema, { defaultValues: EMPTY_TEAM_CREATE_VALUES });
+ const watchedOrganizationId = form.watch("organization_id");
+ const watchedMcpSelection = form.watch("allowed_mcp_servers_and_groups");
+ const watchedToolPermissions = form.watch("mcp_tool_permissions");
const [selectedTeam, setSelectedTeam] = useState(null);
const [selectedTeamId, setSelectedTeamId] = useQueryState("team", parseAsString.withOptions({ history: "push" }));
@@ -134,27 +264,26 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser
: "n/a";
useEffect(() => {
- form.setFieldValue("models", []);
+ form.setValue("models", []);
}, [currentOrgForCreateTeam, userModels]);
// Handle organization preselection when modal opens
useEffect(() => {
if (isTeamModalVisible) {
const adminOrgs = getAdminOrganizations(userRole, userID, organizations);
- const isOrgAdmin = userRole !== "Admin";
// Org admins must scope a team to an org, so with exactly one we preselect it.
// Proxy admins can create org-less teams, so the field stays optional regardless of org count.
if (isOrgAdmin && adminOrgs.length === 1) {
const org = adminOrgs[0];
- form.setFieldValue("organization_id", org.organization_id);
+ form.setValue("organization_id", org.organization_id);
setCurrentOrgForCreateTeam(org);
} else {
- form.setFieldValue("organization_id", currentOrg?.organization_id || null);
+ form.setValue("organization_id", currentOrg?.organization_id || null);
setCurrentOrgForCreateTeam(currentOrg);
}
}
- }, [isTeamModalVisible, userRole, userID, organizations, currentOrg]);
+ }, [isTeamModalVisible, isOrgAdmin, userRole, userID, organizations, currentOrg]);
// Add this useEffect to fetch guardrails
useEffect(() => {
@@ -190,22 +319,26 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser
if (canViewPolicies) fetchPolicies();
}, [accessToken, canViewPolicies]);
- const handleOk = () => {
- setIsTeamModalVisible(false);
- form.resetFields();
+ const resetCreateForm = () => {
+ form.reset(EMPTY_TEAM_CREATE_VALUES);
+ setAdditionalSettingsOpen(false);
+ setMcpSettingsOpen(false);
+ setAgentSettingsOpen(false);
+ setSearchToolSettingsOpen(false);
setLoggingSettings([]);
setModelAliases({});
setRouterSettings(null);
setRouterSettingsKey((prev) => prev + 1);
};
+ const handleOk = () => {
+ setIsTeamModalVisible(false);
+ resetCreateForm();
+ };
+
const handleCancel = () => {
setIsTeamModalVisible(false);
- form.resetFields();
- setLoggingSettings([]);
- setModelAliases({});
- setRouterSettings(null);
- setRouterSettingsKey((prev) => prev + 1);
+ resetCreateForm();
};
const handleDelete = async (team: Team) => {
@@ -378,11 +511,7 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser
await teamCreateCall(accessToken, { ...formValues, models: normalizeTeamModelSelection(formValues.models) });
toast.success("Team created");
await refreshTeams();
- form.resetFields();
- setLoggingSettings([]);
- setModelAliases({});
- setRouterSettings(null);
- setRouterSettingsKey((prev) => prev + 1);
+ resetCreateForm();
setIsTeamModalVisible(false);
}
} catch (error) {
@@ -391,6 +520,19 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser
}
};
+ const mountedCreateValues = (values: TeamCreateFormValues): Record => {
+ const unmounted = new Set([
+ ...(additionalSettingsOpen ? [] : ADDITIONAL_SETTINGS_FIELDS),
+ ...(additionalSettingsOpen && canViewPolicies ? [] : ["policies"]),
+ ...(mcpSettingsOpen ? [] : MCP_SETTINGS_FIELDS),
+ ...(agentSettingsOpen ? [] : AGENT_SETTINGS_FIELDS),
+ ...(searchToolSettingsOpen ? [] : SEARCH_TOOL_SETTINGS_FIELDS),
+ ]);
+ return Object.fromEntries(Object.entries(values).filter(([key]) => !unmounted.has(key)));
+ };
+
+ const onCreateSubmit = (values: TeamCreateFormValues) => handleCreate(mountedCreateValues(values));
+
const is_team_admin = (team: any) => {
if (team == null || team.members_with_roles == null) {
return false;
@@ -405,7 +547,6 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser
};
const { token } = theme.useToken();
- const { Text } = Typography;
const { Content } = Layout;
const tabItems = [
@@ -531,556 +672,538 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser
onCancel={handleCancel}
destroyOnHidden
>
-
- <>
-
-
-
- {(() => {
- const adminOrgs = getAdminOrganizations(userRole, userID, organizations);
- const isOrgAdmin = userRole !== "Admin";
- const isSingleOrg = adminOrgs.length === 1;
- const hasNoOrgs = adminOrgs.length === 0;
-
- return (
- <>
-
- Organization{" "}
-
- Organizations can have multiple teams. Learn more about{" "}
- e.stopPropagation()}
- >
- user management hierarchy
-
-
- }
- >
-
-
-
- }
- name="organization_id"
- initialValue={currentOrg ? currentOrg.organization_id : null}
- className="mt-8"
- rules={
- isOrgAdmin
- ? [
- {
- required: true,
- message: "Please select an organization",
- },
- ]
- : []
- }
- help={
- isOrgAdmin && isSingleOrg
- ? "You can only create teams within this organization"
- : isOrgAdmin
- ? "required"
- : ""
- }
- >
- {
- form.setFieldValue("organization_id", value);
- setCurrentOrgForCreateTeam(adminOrgs?.find((org) => org.organization_id === value) || null);
- }}
- filterOption={(input, option) => {
- if (!option) return false;
- const optionValue = option.children?.toString() || "";
- return optionValue.toLowerCase().includes(input.toLowerCase());
- }}
- optionFilterProp="children"
- >
- {adminOrgs?.map((org) => (
-
- {org.organization_alias} {" "}
- ({org.organization_id})
-
- ))}
-
-
-
- {/* Show message when org admin needs to select organization */}
- {isOrgAdmin && !isSingleOrg && adminOrgs.length > 1 && (
-
-
- Please select an organization to create a team for. You can only create teams within
- organizations where you are an admin.
-
-
- )}
- >
- );
- })()}
-
- Models{" "}
-
-
-
-
- }
- name="models"
- >
- form.setFieldValue("models", values)}
- organizationID={form.getFieldValue("organization_id")}
- options={{
- includeSpecialOptions: true,
- showAllProxyModelsOverride: !form.getFieldValue("organization_id"),
- }}
- context="team"
- dataTestId="create-team-models-select"
- />
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Additional Settings
-
-
-
-
- {
- e.target.value = e.target.value.trim();
- }}
- />
-
- (value ? Number(value) : undefined)}
- tooltip="This is the individual budget for a user in the team."
- >
-
-
-
-
-
-
-
-
-
-
-
- {
- if (!value) {
- return Promise.resolve();
- }
- try {
- JSON.parse(value);
- return Promise.resolve();
- } catch (error) {
- return Promise.reject(new Error("Please enter valid JSON"));
- }
- },
- },
- ]}
- >
-
-
-
- Guardrails{" "}
-
- e.stopPropagation()}
- >
-
-
-
-
- }
- name="guardrails"
- className="mt-8"
- help="Select existing guardrails or enter new ones"
- >
- ({
- value: name,
- label: name,
- }))}
- />
-
-
- Disable Global Guardrails{" "}
-
-
-
-
- }
- name="disable_global_guardrails"
- className="mt-4"
- valuePropName="checked"
- help="Bypass global guardrails for this team"
- >
-
-
- {canViewPolicies && (
-
- Policies{" "}
-
- e.stopPropagation()}
- >
-
-
-
-
- }
- name="policies"
- className="mt-8"
- help="Select existing policies or enter new ones"
- >
- ({
- value: name,
- label: name,
- }))}
- />
-
+
+
+
+
+ {({ ref, value, ...field }) => (
+
)}
-
- Access Groups{" "}
-
-
-
-
- }
- name="access_group_ids"
- className="mt-8"
- help="Select access groups to assign to this team"
- >
-
-
-
- Allowed Vector Stores{" "}
-
-
-
-
- }
- name="allowed_vector_store_ids"
- className="mt-8"
- help="Select vector stores this team can access. Leave empty for access to all vector stores"
- >
- form.setFieldValue("allowed_vector_store_ids", values)}
- value={form.getFieldValue("allowed_vector_store_ids")}
- accessToken={accessToken || ""}
- placeholder="Select vector stores (optional)"
- />
-
-
-
-
-
-
+
+ {(() => {
+ const adminOrgs = getAdminOrganizations(userRole, userID, organizations);
+ const isSingleOrg = adminOrgs.length === 1;
+ const hasNoOrgs = adminOrgs.length === 0;
-
-
- MCP Settings
-
-
-
-
- Allowed MCP Servers{" "}
-
-
-
-
- }
- name="allowed_mcp_servers_and_groups"
- className="mt-4"
- help="Select MCP servers or access groups this team can access"
- >
- form.setFieldValue("allowed_mcp_servers_and_groups", val)}
- value={form.getFieldValue("allowed_mcp_servers_and_groups")}
- accessToken={accessToken || ""}
- placeholder="Select MCP servers or access groups (optional)"
- allowAllProxyMcpServers={isProxyAdminRole(userRole || "")}
+ return (
+ <>
+
+ {({ id, value, onChange }) => (
+ ({
+ value: org.organization_id ?? "",
+ label: org.organization_alias ?? "",
+ sublabel: org.organization_id ?? "",
+ }))}
+ disabled={isOrgAdmin && isSingleOrg}
+ allowClear={!isOrgAdmin}
+ placeholder={hasNoOrgs ? "No organizations available" : "Search or select an Organization"}
+ emptyText="No organizations available"
+ onValueChange={(next) => {
+ onChange(next === "" ? null : next);
+ setCurrentOrgForCreateTeam(adminOrgs.find((org) => org.organization_id === next) ?? null);
+ }}
+ />
+ )}
+
+
+ {isOrgAdmin && !isSingleOrg && adminOrgs.length > 1 && (
+
+
+ Please select an organization to create a team for. You can only create teams within
+ organizations where you are an admin.
+
+
+ )}
+ >
+ );
+ })()}
+
+ {({ id, value, onChange }) => (
+
-
+ )}
+
- {/* Hidden field to register mcp_tool_permissions with the form */}
-
-
-
+
+ {({ ref, value, ...field }) => (
+
+ )}
+
+
+ {({ id, value, onChange }) => (
+
+ )}
+
+
+ {({ ref, value, ...field }) => (
+
+ )}
+
+
+ {({ ref, value, ...field }) => (
+
+ )}
+
+
+ Metadata
+
+
+ Values are saved as text. Enter JSON for typed values, e.g. 3, true, or {'{"region": "us"}'}.
+
+
-
- prevValues.allowed_mcp_servers_and_groups !== currentValues.allowed_mcp_servers_and_groups ||
- prevValues.mcp_tool_permissions !== currentValues.mcp_tool_permissions
- }
- >
- {() => (
-
-
+
+ Additional Settings
+
+
+
+
+
+ {({ ref, value, ...field }) => }
+
+
+ {({ ref, value, onChange, ...field }) => (
+ ) =>
+ onChange(event.target.value ? Number(event.target.value) : undefined)
+ }
+ step={0.01}
+ precision={2}
+ width={200}
+ />
+ )}
+
+
+ {({ ref, value, ...field }) => (
+
+ )}
+
+
+ {({ ref, value, ...field }) => (
+
+ )}
+
+
+ {({ ref, value, ...field }) => (
+
+ )}
+
+
+ {({ ref, value, ...field }) => (
+
+ )}
+
+
+ {({ id, value, onChange }) => (
+ ({ value: name, label: name }))}
+ placeholder="Select or enter guardrails"
+ />
+ )}
+
+
+ {({ id, value, onChange }) => (
+
+ )}
+
+ {canViewPolicies && (
+
+ {({ id, value, onChange }) => (
+ ({ value: name, label: name }))}
+ placeholder="Select or enter policies"
+ />
+ )}
+
+ )}
+
+ {({ value, onChange }) => (
+
+ )}
+
+
+ {({ value, onChange }) => (
+
+ )}
+
+
+ {({ value, onChange }) => (
+
+ )}
+
+
+
+
+
+
+
+ MCP Settings
+
+
+
+
+ {({ value, onChange }) => (
+ form.setFieldsValue({ mcp_tool_permissions: toolPerms })}
+ placeholder="Select MCP servers or access groups (optional)"
+ allowAllProxyMcpServers={isProxyAdminRole(userRole || "")}
/>
-
- )}
-
-
-
+ )}
+
-
-
- Agent Settings
-
-
-
-
- Allowed Agents{" "}
-
-
-
-
- }
- name="allowed_agents_and_groups"
- className="mt-4"
- help="Select agents or access groups this team can access"
- >
- form.setFieldValue("allowed_agents_and_groups", val)}
- value={form.getFieldValue("allowed_agents_and_groups")}
- accessToken={accessToken || ""}
- placeholder="Select agents or access groups (optional)"
- />
-
-
-
+
+ form.setValue("mcp_tool_permissions", toolPerms)}
+ />
+
+
+
-
-
- Search Tool Settings
-
-
-
-
- Allowed Search Tools{" "}
-
-
-
-
- }
- name="object_permission_search_tools"
- className="mt-4"
- help="Restrict which configured search tools keys on this team may call."
- >
- form.setFieldValue("object_permission_search_tools", vals)}
- value={form.getFieldValue("object_permission_search_tools")}
- accessToken={accessToken || ""}
- placeholder="Select search tools (optional, empty = all allowed)"
- />
-
-
-
+
+
+ Agent Settings
+
+
+
+
+ {({ value, onChange }) => (
+
+ )}
+
+
+
-
-
- Logging Settings
-
-
-
-
-
-
+
+
+ Search Tool Settings
+
+
+
+
+ {({ value, onChange }) => (
+
+ )}
+
+
+
-
-
- Router Settings
-
-
-
-
- 0 ? { data: userModels.map((model) => ({ model_name: model })) } : undefined
- }
- />
-
-
-
+
+
+ Logging Settings
+
+
+
+
+
+
-
-
- Model Aliases
-
-
-
-
-
- Create custom aliases for models that can be used by team members in API calls. This allows you to
- create shortcuts for specific models.
-
-
-
-
-
- >
-
-
- Create Team
-
-
-
+
+
+ Router Settings
+
+
+
+
+ 0
+ ? { data: userModels.map((model) => ({ model_name: model })) }
+ : undefined
+ }
+ />
+
+
+
+
+
+
+ Model Aliases
+
+
+
+
+
+ Create custom aliases for models that can be used by team members in API calls. This allows you
+ to create shortcuts for specific models.
+
+
+
+
+
+
+
+
+ Create Team
+
+
+
+
)}
diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopKeyView.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopKeyView.tsx
index 2993f549c65..b3c1be62e88 100644
--- a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopKeyView.tsx
+++ b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopKeyView.tsx
@@ -3,7 +3,8 @@ import { BarChart } from "@/components/shared/charts";
import { DataTable } from "@/components/shared/DataTable";
import { IdCell, MoneyCell } from "@/components/shared/table_cells";
import { ChevronDownIcon, ChevronUpIcon } from "@heroicons/react/outline";
-import { Segmented, Tooltip } from "antd";
+import { SimpleTooltip } from "@/components/ui/tooltip";
+import { Segmented } from "antd";
import React, { useState } from "react";
import { formatNumberWithCommas } from "../../../../utils/dataUtils";
import { transformKeyInfo } from "../../../key_team_helpers/transform_key_info";
@@ -113,9 +114,9 @@ const TopKeyView: React.FC = ({ topKeys, teams, showTags = fals
{displayTags.map((tag, index) => (
-
Tag Name: {tag.tag}
@@ -128,7 +129,7 @@ const TopKeyView: React.FC
= ({ topKeys, teams, showTags = fals
}
>
{tag.tag.slice(0, 7)}...
-
+
))}
{hasMoreTags && (
({
@@ -131,8 +132,12 @@ const testTeam: Team = {
};
const createTestProps = (userRole = "proxy_admin", userId = "user-1", isTeamAdmin = false) => {
- const { result } = renderHook(() => Form.useForm());
- const [form] = result.current;
+ const { result } = renderHook(() => {
+ const form = useForm({ mode: "onChange" });
+ const registry = useMountRegistry();
+ return { form, registry };
+ });
+ const { form, registry } = result.current;
const teams = [
{
@@ -159,7 +164,9 @@ const createTestProps = (userRole = "proxy_admin", userId = "user-1", isTeamAdmi
return {
form,
- handleOk: vi.fn(),
+ registry,
+ mountedValues: () => projectMountedValues(registry, form.getValues),
+ handleOk: vi.fn().mockResolvedValue(true),
setSelectedProvider: vi.fn(),
setProviderModelsFn: vi.fn(),
getPlaceholder: vi.fn((provider: Providers) => `Enter ${provider} model name`),
@@ -310,4 +317,81 @@ describe("AddModelForm", () => {
expect((await screen.findAllByRole("button", { name: "Test Connect" })).length).toBeGreaterThan(0);
expect(await screen.findByRole("button", { name: "Add Model" })).toBeInTheDocument();
});
+
+ describe("cache control bindings reach the parent form store", () => {
+ const renderWithForm = async () => {
+ const mockUseAuthorized = vi.mocked(await import("@/app/(dashboard)/hooks/useAuthorized"));
+ mockUseAuthorized.default.mockReturnValue(mockAuthorizedUser("proxy_admin", "user-1", true));
+ const props = createTestProps();
+ const user = userEvent.setup({ pointerEventsCheck: PointerEventsCheckLevel.Never });
+ renderWithProviders( );
+ await screen.findByText("Provider");
+
+ return {
+ user,
+ openCacheControl: async () => {
+ await user.click(await screen.findByText("Advanced Settings"));
+ await user.click(screen.getByLabelText("Cache Control Injection Points"));
+ await screen.findByText("Add Injection Point");
+ },
+ closeCacheControl: async () => {
+ await user.click(screen.getByLabelText("Cache Control Injection Points"));
+ await waitFor(() => expect(screen.queryByText("Add Injection Point")).not.toBeInTheDocument());
+ },
+ mountedValues: async (): Promise> => props.mountedValues(),
+ };
+ };
+
+ it("omits both cache control keys while the section is untouched", async () => {
+ const { mountedValues } = await renderWithForm();
+ const values = await mountedValues();
+ expect(values).not.toHaveProperty("cache_control_injection_points");
+ expect(values.cache_control).toBeUndefined();
+ });
+
+ it("sends the seeded injection point once the toggle is on", async () => {
+ const { openCacheControl, mountedValues } = await renderWithForm();
+ await openCacheControl();
+ const values = await mountedValues();
+ expect(values.cache_control).toBe(true);
+ expect(values.cache_control_injection_points).toEqual([{ location: "message" }]);
+ });
+
+ it("carries an edited role and keeps the index a string, as the antd control did", async () => {
+ const { user, openCacheControl, mountedValues } = await renderWithForm();
+ await openCacheControl();
+
+ await user.click(screen.getByText("Select a role"));
+ await user.click(await screen.findByText("System"));
+ await user.type(screen.getByPlaceholderText("Optional"), "3");
+
+ const values = await mountedValues();
+ expect(values.cache_control_injection_points).toEqual([{ location: "message", role: "system", index: "3" }]);
+ });
+
+ it("adds a second injection point row", async () => {
+ const { user, openCacheControl, mountedValues } = await renderWithForm();
+ await openCacheControl();
+
+ await user.click(screen.getByText("Add Injection Point"));
+ await waitFor(() => expect(screen.getAllByPlaceholderText("Optional")).toHaveLength(2));
+ await user.type(screen.getAllByPlaceholderText("Optional")[1], "7");
+
+ const values = await mountedValues();
+ expect(values.cache_control_injection_points).toEqual([
+ { location: "message" },
+ { location: "message", index: "7" },
+ ]);
+ });
+
+ it("drops the injection points again when the toggle goes back off", async () => {
+ const { openCacheControl, closeCacheControl, mountedValues } = await renderWithForm();
+ await openCacheControl();
+ await closeCacheControl();
+
+ const values = await mountedValues();
+ expect(values.cache_control).toBe(false);
+ expect(values).not.toHaveProperty("cache_control_injection_points");
+ });
+ });
});
diff --git a/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx b/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx
index de159207078..4d339737ab0 100644
--- a/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx
@@ -4,11 +4,20 @@ import { useTags } from "@/app/(dashboard)/hooks/tags/useTags";
import { all_admin_roles, isUserTeamAdminForAnyTeam } from "@/utils/roles";
import { modelCreationScope } from "@/utils/modelPermissions";
import { Switch } from "@/components/ui/switch";
-import type { FormInstance } from "antd";
-import { Select as AntdSelect, Button, Card, Col, Form, Modal, Row, Tooltip, Typography, Alert } from "antd";
+import { Field, FieldLabel } from "@/components/shared/form/field";
+import { Select as AntdSelect, Button, Card, Col, Modal, Row, Tooltip, Typography, Alert } from "antd";
import type { UploadProps } from "antd/es/upload";
import React, { useEffect, useMemo, useState } from "react";
+import { FormProvider, useWatch, type UseFormReturn } from "react-hook-form";
import TeamDropdown from "../common_components/team_dropdown";
+import { antdRequired } from "../common_components/antdFormRules";
+import { labelWithHint } from "@/components/shared/form/LabelWithHint";
+import {
+ MountedFormField,
+ MountedFormProvider,
+ type MountRegistry,
+ type MountedFormValues,
+} from "../common_components/MountedFormField";
import type { Team } from "../key_team_helpers/key_list";
import { type CredentialItem, type ProviderCreateInfo, modelAvailableCall } from "../networking";
import { Providers } from "../provider_info_helpers";
@@ -22,8 +31,10 @@ import { TEST_MODES } from "./add_model_modes";
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
interface AddModelFormProps {
- form: FormInstance; // For the Add Model tab
- handleOk: () => Promise;
+ form: UseFormReturn; // For the Add Model tab
+ registry: MountRegistry;
+ mountedValues: () => MountedFormValues;
+ handleOk: () => Promise;
selectedProvider: Providers;
setSelectedProvider: (provider: Providers) => void;
providerModels: string[];
@@ -36,10 +47,20 @@ interface AddModelFormProps {
credentials: CredentialItem[];
}
+const connectionTestModelName = (values: MountedFormValues): string | undefined => {
+ const named = values.model_name || values.model;
+ if (Array.isArray(named)) {
+ return named.join(", ");
+ }
+ return typeof named === "string" ? named : undefined;
+};
+
const { Title, Link } = Typography;
const AddModelForm: React.FC = ({
form,
+ registry,
+ mountedValues,
handleOk,
selectedProvider,
setSelectedProvider,
@@ -67,6 +88,7 @@ const AddModelForm: React.FC = ({
const { data: guardrailsData } = useGuardrails();
const guardrailsList = guardrailsData?.guardrails.map((g) => g.guardrail_name);
const { data: tagsList } = useTags();
+ const selectedCredentialName = useWatch({ control: form.control, name: "litellm_credential_name" });
const handleTestConnection = async () => {
setIsTestingConnection(true);
@@ -112,274 +134,302 @@ const AddModelForm: React.FC = ({
Add Model
- {
- await handleOk().then(() => {
- setTeamAdminSelectedTeam(null);
- });
- }}
- onFinishFailed={(errorInfo) => {}}
- labelCol={{ span: 10 }}
- wrapperCol={{ span: 16 }}
- labelAlign="left"
- >
- <>
- {requiresTeamScope && (
- <>
-
- {
- setTeamAdminSelectedTeam(value);
- }}
- />
-
- {!teamAdminSelectedTeam && (
-
- )}
- >
- )}
- {(isAdmin || (isTeamAdmin && teamAdminSelectedTeam)) && (
- <>
-
- {
- setSelectedProvider(value as Providers);
- setProviderModelsFn(value as Providers);
- form.setFieldsValue({
- custom_llm_provider: value,
- });
- form.setFieldsValue({
- model: [],
- model_name: undefined,
- });
- }}
- >
- {providerMetadataErrorText && sortedProviderMetadata.length === 0 && (
-
- {providerMetadataErrorText}
-
- )}
- {sortedProviderMetadata.map((providerInfo) => {
- const displayName = providerInfo.provider_display_name;
- const providerKey = providerInfo.provider;
-
- return (
-
-
-
- );
- })}
-
-
-
-
- {/* Conditionally Render "Public Model Name" */}
-
-
- {/* Select Mode */}
-
- setTestMode(value)}
- options={TEST_MODES}
- />
-
-
-
-
-
- Optional - LiteLLM endpoint to use when health checking this model{" "}
-
- Learn more
-
-
-
-
-
- {/* Credentials */}
-
-
- Either select existing credentials OR enter new provider credentials below
-
-
-
-
- (option?.label ?? "").toLowerCase().includes(input.toLowerCase())}
- options={[
- { value: null, label: "None" },
- ...credentials.map((credential) => ({
- value: credential.credential_name,
- label: credential.credential_name,
- })),
- ]}
- allowClear
- />
-
-
-
- prevValues.litellm_credential_name !== currentValues.litellm_credential_name ||
- prevValues.provider !== currentValues.provider
+
+
+ {
+ event.preventDefault();
+ void handleOk().then((submitted) => {
+ if (submitted) {
+ setTeamAdminSelectedTeam(null);
}
- >
- {({ getFieldValue }) => {
- const credentialName = getFieldValue("litellm_credential_name");
- // Only show provider specific fields if no credentials selected
- if (!credentialName) {
- return (
- <>
-
-
- >
- );
- }
- return null;
- }}
-
-
-
-
Additional Model Info Settings
-
-
- {/* Team-only Model Switch - Only show for proxy admins, not team admins */}
- {(isAdmin || !isTeamAdmin) && (
-
-
-
- {
- setIsTeamOnly(checked);
- if (!checked) {
- form.setFieldValue("team_id", undefined);
- }
- }}
- disabled={!premiumUser}
- aria-label="Team-BYOK Model"
- />
-
-
-
- )}
-
- {/* Conditional Team Selection */}
- {isTeamOnly && !requiresTeamScope && (
-
-
-
- )}
- {isAdmin && (
+ });
+ }}
+ >
+ <>
+ {requiresTeamScope && (
<>
-
- ({
- value: group,
- label: group,
- }))}
- maxTagCount="responsive"
- allowClear
+ {(control) => (
+ {
+ control.onChange(value);
+ setTeamAdminSelectedTeam(value);
+ }}
+ />
+ )}
+
+ {!teamAdminSelectedTeam && (
+
-
+ )}
>
)}
-
+ {(isAdmin || (isTeamAdmin && teamAdminSelectedTeam)) && (
+ <>
+
+ {(control) => (
+ {
+ control.onChange(value);
+ setSelectedProvider(value as Providers);
+ setProviderModelsFn(value as Providers);
+ form.setValue("model", []);
+ form.setValue("model_name", undefined);
+ }}
+ >
+ {providerMetadataErrorText && sortedProviderMetadata.length === 0 && (
+
+ {providerMetadataErrorText}
+
+ )}
+ {sortedProviderMetadata.map((providerInfo) => {
+ const displayName = providerInfo.provider_display_name;
+ const providerKey = providerInfo.provider;
+
+ return (
+
+
+
+ );
+ })}
+
+ )}
+
+
+
+ {/* Conditionally Render "Public Model Name" */}
+
+
+ {/* Select Mode */}
+
+ {(control) => (
+ {
+ control.onChange(value);
+ setTestMode(value);
+ }}
+ options={TEST_MODES}
+ />
+ )}
+
+
+
+
+
+ Optional - LiteLLM endpoint to use when health checking this model{" "}
+
+ Learn more
+
+
+
+
+
+ {/* Credentials */}
+
+
+ Either select existing credentials OR enter new provider credentials below
+
+
+
+
+ {(control) => (
+
+ (option?.label ?? "").toLowerCase().includes(input.toLowerCase())
+ }
+ value={control.value as string | null | undefined}
+ onChange={control.onChange}
+ onBlur={control.onBlur}
+ options={[
+ { value: null, label: "None" },
+ ...credentials.map((credential) => ({
+ value: credential.credential_name,
+ label: credential.credential_name,
+ })),
+ ]}
+ allowClear
+ />
+ )}
+
+
+ {/* Only show provider specific fields if no credentials selected */}
+ {!selectedCredentialName && (
+ <>
+
+
+ >
+ )}
+
+
+
Additional Model Info Settings
+
+
+ {/* Team-only Model Switch - Only show for proxy admins, not team admins */}
+ {(isAdmin || !isTeamAdmin) && (
+
+
+ {labelWithHint(
+ "Team-BYOK Model",
+ "Only use this model + credential combination for this team. Useful when teams want to onboard their own OpenAI keys.",
+ )}
+
+
+
+ {
+ setIsTeamOnly(checked);
+ if (!checked) {
+ form.setValue("team_id", undefined);
+ }
+ }}
+ disabled={!premiumUser}
+ aria-label="Team-BYOK Model"
+ />
+
+
+
+ )}
+
+ {/* Conditional Team Selection */}
+ {isTeamOnly && !requiresTeamScope && (
+
+ {(control) => (
+
+ )}
+
+ )}
+ {isAdmin && (
+ <>
+
+ {(control) => (
+ ({
+ value: group,
+ label: group,
+ }))}
+ maxTagCount="responsive"
+ allowClear
+ />
+ )}
+
+ >
+ )}
+
+ >
+ )}
+
+
+ Need Help?
+
+
+
+ Test Connect
+
+
+ Add Model
+
+
+
>
- )}
-
-
- Need Help?
-
-
-
- Test Connect
-
-
- Add Model
-
-
-
- >
-
+
+
+
{/* Test Connection Results Modal */}
@@ -408,10 +458,10 @@ const AddModelForm: React.FC = ({
{
setIsResultModalVisible(false);
setIsTestingConnection(false);
diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx
index 03d8fbc2394..8d42b34a14b 100644
--- a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx
@@ -1,5 +1,6 @@
import { InfoCircleOutlined } from "@ant-design/icons";
-import { Select as AntdSelect, Card, InputNumber, Radio, Space, Switch, Tooltip, Typography } from "antd";
+import { SimpleTooltip } from "@/components/ui/tooltip";
+import { Select as AntdSelect, Card, InputNumber, Radio, Space, Switch, Typography } from "antd";
import React from "react";
import ClassifierPromptEditor from "./ClassifierPromptEditor";
import HeuristicScoringConfig from "./HeuristicScoringConfig";
@@ -286,11 +287,14 @@ const ClassificationMethodConfig: React.FC = ({
Classification Rubric
-
+
-
+
-
+
= ({
label: CLASSIFICATION_RUBRIC_DESCRIPTIONS[preset].label,
}))}
/>
-
+
{usesCustomPrompt
? "Not in use: the custom prompt below is the classifier's entire rubric."
@@ -335,8 +339,8 @@ const ClassificationMethodConfig: React.FC = ({
— right when the classifier grades complexity too
- = ({
Route to the default model{defaultModel ? ` (${defaultModel})` : ""} {" "}
— right when your prompt grades something other than complexity
-
+
@@ -393,9 +397,9 @@ const ClassificationMethodConfig: React.FC = ({
aria-label="Include Assistant Turns"
/>
Include Assistant Turns
-
+
-
+
Let the classifier read the assistant's replies, so difficulty the model stated rather than the user
@@ -411,9 +415,9 @@ const ClassificationMethodConfig: React.FC = ({
Custom Technical Keywords
-
+
-
+
Optional: Add terms to the built-in list to improve classification accuracy on the technical dimension.
diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
index 3d4845ecf39..9f1108639ac 100644
--- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
@@ -1,5 +1,6 @@
import { InfoCircleOutlined } from "@ant-design/icons";
-import { Select as AntdSelect, Card, Collapse, Divider, Input, Space, Switch, Tooltip, Typography } from "antd";
+import { SimpleTooltip } from "@/components/ui/tooltip";
+import { Select as AntdSelect, Card, Collapse, Divider, Input, Space, Switch, Typography } from "antd";
import React from "react";
import { ModelGroup } from "@/components/llm_calls/fetch_models";
import AdaptiveRoutingConfig from "./AdaptiveRoutingConfig";
@@ -249,9 +250,9 @@ const ComplexityRouterConfig: React.FC = ({
Complexity Tier Configuration
-
+
-
+
@@ -279,9 +280,9 @@ const ComplexityRouterConfig: React.FC = ({
{label} Tier
-
+
-
+
Tier {index + 1} of {TIER_KEYS.length} · {tier}
@@ -329,9 +330,9 @@ const ComplexityRouterConfig: React.FC = ({
Default Model
-
+
-
+
= ({ keywords, onCha
Escalation Keywords
-
+
-
+
Optional: when a user message contains one of these phrases, the request is bumped one tier higher than it would
diff --git a/ui/litellm-dashboard/src/components/add_model/KeywordTierRules.tsx b/ui/litellm-dashboard/src/components/add_model/KeywordTierRules.tsx
index 72d6e9f4fef..49fea3d30e7 100644
--- a/ui/litellm-dashboard/src/components/add_model/KeywordTierRules.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/KeywordTierRules.tsx
@@ -1,5 +1,6 @@
import { DeleteOutlined, InfoCircleOutlined, PlusOutlined } from "@ant-design/icons";
-import { Button, Card, Empty, Select as AntdSelect, Tooltip, Typography } from "antd";
+import { SimpleTooltip } from "@/components/ui/tooltip";
+import { Button, Card, Empty, Select as AntdSelect, Typography } from "antd";
import React from "react";
import { emptyKeywordTierRuleIndexes } from "./complexity_router_keywords";
@@ -70,9 +71,9 @@ const KeywordTierRules: React.FC = ({ rules, onChange, ti
Keyword Tier Overrides
-
+
-
+
} onClick={addRule}>
Add keyword rule
diff --git a/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.tsx b/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.tsx
index c2252843b29..867f6a901e8 100644
--- a/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.tsx
@@ -1,5 +1,6 @@
import { InfoCircleOutlined } from "@ant-design/icons";
-import { InputNumber, Select as AntdSelect, Switch, Tooltip, Typography } from "antd";
+import { SimpleTooltip } from "@/components/ui/tooltip";
+import { InputNumber, Select as AntdSelect, Switch, Typography } from "antd";
import React from "react";
import { ModelGroup } from "@/components/llm_calls/fetch_models";
@@ -41,9 +42,9 @@ const SemanticKeywordMatching: React.FC
= ({
Semantic keyword matching
-
+
-
+
Uses same keyword-tier pairs as above and overrides direct keyword matching. Adds latency based on embedding
diff --git a/ui/litellm-dashboard/src/components/add_model/advanced_settings.test.tsx b/ui/litellm-dashboard/src/components/add_model/advanced_settings.test.tsx
index 4e5c5f25374..01e00d903aa 100644
--- a/ui/litellm-dashboard/src/components/add_model/advanced_settings.test.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/advanced_settings.test.tsx
@@ -1,5 +1,6 @@
import { act, fireEvent, render, waitFor } from "@testing-library/react";
import { beforeEach, describe, expect, it, vi } from "vitest";
+import { MountedFormHost } from "../../../tests/mounted-form-host";
import AdvancedSettings from "./advanced_settings";
const mockUsePtuCostAttributionEnabled = vi.fn();
@@ -12,13 +13,15 @@ const PTU_LABELS = ["PTU Count", "Calculated Cost per PTU / Hour (USD)", "PTU Ef
const renderAdvancedSettings = () =>
render(
- {}}
- guardrailsList={[]}
- tagsList={{}}
- accessToken="test-token"
- />,
+
+ {}}
+ guardrailsList={[]}
+ tagsList={{}}
+ accessToken="test-token"
+ />
+ ,
);
describe("AdvancedSettings", () => {
diff --git a/ui/litellm-dashboard/src/components/add_model/advanced_settings.tsx b/ui/litellm-dashboard/src/components/add_model/advanced_settings.tsx
index 00b4c000a43..140b4363327 100644
--- a/ui/litellm-dashboard/src/components/add_model/advanced_settings.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/advanced_settings.tsx
@@ -1,5 +1,5 @@
import React from "react";
-import { Form, Switch, Select, Tooltip, DatePicker } from "antd";
+import { Switch, Select, Tooltip, DatePicker } from "antd";
import { ChevronDown } from "lucide-react";
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible";
import { Input } from "@/components/ui/input";
@@ -7,7 +7,14 @@ import { Row, Col, Typography } from "antd";
import TextArea from "antd/es/input/TextArea";
import { InfoCircleOutlined } from "@ant-design/icons";
import { Team } from "../key_team_helpers/key_list";
-import CacheControlSettings from "./cache_control_settings";
+import { antdRules } from "../common_components/antdFormRules";
+import { labelWithHint } from "@/components/shared/form/LabelWithHint";
+import { MountedFormField } from "../common_components/MountedFormField";
+import CacheControlInjectionPoints, {
+ CACHE_CONTROL_LABEL,
+ CACHE_CONTROL_TOOLTIP,
+ NEW_CACHE_CONTROL_POINT,
+} from "./cache_control_settings";
import VectorStoreSelector from "../vector_store_management/VectorStoreSelector";
import { Tag } from "../tag_management/types";
import { formItemValidateJSON } from "../../utils/textUtils";
@@ -35,6 +42,31 @@ interface AdvancedSettingsProps {
accessToken: string;
}
+const USAGE_COST_FIELDS = [
+ "input_cost_per_token",
+ "output_cost_per_token",
+ "cache_read_input_token_cost",
+ "cache_creation_input_token_cost",
+ "input_cost_per_second",
+];
+
+const REVALIDATED_WHEN_PTU_COUNT_CHANGES = [PTU_RATE_FIELD, PTU_START_FIELD, ...USAGE_COST_FIELDS];
+
+const validateNumber = (_: unknown, value: unknown) => {
+ if (!value) {
+ return Promise.resolve();
+ }
+ if (isNaN(Number(value)) || Number(value) < 0) {
+ return Promise.reject("Please enter a valid positive number");
+ }
+ return Promise.resolve();
+};
+
+const usageCostRules = {
+ deps: [PTU_COUNT_FIELD],
+ validate: antdRules({ validator: validateNumber }, ptuNoUsageCostRule(PTU_COUNT_FIELD)),
+};
+
const AdvancedSettings: React.FC = ({
showAdvancedSettings,
setShowAdvancedSettings,
@@ -43,95 +75,36 @@ const AdvancedSettings: React.FC = ({
tagsList,
accessToken,
}) => {
- const [form] = Form.useForm();
const [customPricing, setCustomPricing] = React.useState(false);
const [pricingModel, setPricingModel] = React.useState<"per_token" | "per_second">("per_token");
const [showCacheControl, setShowCacheControl] = React.useState(false);
const ptuCostAttributionEnabled = usePtuCostAttributionEnabled();
- // Add validation function for numbers
- const validateNumber = (_: any, value: string) => {
- if (!value) {
- return Promise.resolve();
- }
- if (isNaN(Number(value)) || Number(value) < 0) {
- return Promise.reject("Please enter a valid positive number");
- }
- return Promise.resolve();
- };
-
- // Handle custom pricing changes
- const handleCustomPricingChange = (checked: boolean) => {
- setCustomPricing(checked);
- if (!checked) {
- // Clear pricing fields when disabled
- form.setFieldsValue({
- input_cost_per_token: undefined,
- output_cost_per_token: undefined,
- cache_read_input_token_cost: undefined,
- cache_creation_input_token_cost: undefined,
- input_cost_per_second: undefined,
- });
- }
- };
-
- const handlePassThroughChange = (checked: boolean) => {
- const currentParams = form.getFieldValue("litellm_extra_params");
- try {
- let paramsObj = currentParams ? JSON.parse(currentParams) : {};
- if (checked) {
- paramsObj.use_in_pass_through = true;
- } else {
- delete paramsObj.use_in_pass_through;
- }
- // Only set the field value if there are remaining parameters
- if (Object.keys(paramsObj).length > 0) {
- form.setFieldValue("litellm_extra_params", JSON.stringify(paramsObj, null, 2));
- } else {
- form.setFieldValue("litellm_extra_params", "");
- }
- } catch (error) {
- // If JSON parsing fails, only create new object if checked is true
- if (checked) {
- form.setFieldValue("litellm_extra_params", JSON.stringify({ use_in_pass_through: true }, null, 2));
- } else {
- form.setFieldValue("litellm_extra_params", "");
- }
- }
- };
-
- const handleCacheControlChange = (checked: boolean) => {
- setShowCacheControl(checked);
- if (!checked) {
- const currentParams = form.getFieldValue("litellm_extra_params");
- try {
- let paramsObj = currentParams ? JSON.parse(currentParams) : {};
- delete paramsObj.cache_control_injection_points;
- if (Object.keys(paramsObj).length > 0) {
- form.setFieldValue("litellm_extra_params", JSON.stringify(paramsObj, null, 2));
- } else {
- form.setFieldValue("litellm_extra_params", "");
- }
- } catch (error) {
- form.setFieldValue("litellm_extra_params", "");
- }
- }
- };
-
return (
<>
Advanced Settings
-
+
-
-
-
-
+
+
+ {(control) => (
+ {
+ control.onChange(checked);
+ setCustomPricing(checked);
+ }}
+ className="bg-gray-600"
+ />
+ )}
+
-
Attached Knowledge Bases (RAG){" "}
@@ -147,18 +120,21 @@ const AdvancedSettings: React.FC = ({
}
- name="vector_store_ids"
className="mt-4"
help="Select vector stores to attach. Requests to this model will automatically use these for RAG. Set up vector stores in Tools > Vector Stores."
>
- {}}
- accessToken={accessToken}
- placeholder="Select knowledge bases (optional)"
- />
-
+ {(control) => (
+
+ )}
+
-
Guardrails{" "}
@@ -174,189 +150,331 @@ const AdvancedSettings: React.FC = ({
}
- name="guardrails"
className="mt-4"
help="Select existing guardrails. Go to 'Guardrails' tab to create new guardrails."
>
- ({ value: name, label: name }))}
- />
-
+ {(control) => (
+
({ value: name, label: name }))}
+ />
+ )}
+
-
- ({
- value: tag.name,
- label: tag.name,
- title: tag.description || tag.name,
- }))}
- />
-
+
+ {(control) => (
+ ({
+ value: tag.name,
+ label: tag.name,
+ title: tag.description || tag.name,
+ }))}
+ />
+ )}
+
{ptuCostAttributionEnabled && (
<>
-
-
-
+ {(control) => (
+
+ )}
+
-
-
-
+ {(control) => (
+
+ )}
+
-
-
-
+ {(control) => (
+
+ )}
+
-
-
-
+ {(control) => (
+
+ )}
+
>
)}
{customPricing && (
-
-
- setPricingModel(value)}
- options={[
- { value: "per_token", label: "Per Million Tokens" },
- { value: "per_second", label: "Per Second" },
- ]}
- />
-
+
+
+ {(control) => (
+ {
+ control.onChange(value);
+ setPricingModel(value);
+ }}
+ options={[
+ { value: "per_token", label: "Per Million Tokens" },
+ { value: "per_second", label: "Per Second" },
+ ]}
+ />
+ )}
+
{pricingModel === "per_token" ? (
<>
-
-
-
- (
+
+ )}
+
+
-
-
- (
+
+ )}
+
+
-
-
- (
+
+ )}
+
+
-
-
+ {(control) => (
+
+ )}
+
>
) : (
-
-
-
+ {(control) => (
+
+ )}
+
)}
)}
-
Allow using these credentials in pass through routes.{" "}
Learn more
-
- }
- >
-
-
-
-
-
,
+ )}
className="mb-4 mt-4"
- rules={[{ validator: formItemValidateJSON }]}
>
-
-
+ />
+ )}
+
-
+
Pass JSON of litellm supported params{" "}
litellm.completion() call
@@ -364,20 +482,28 @@ const AdvancedSettings: React.FC = ({
-
-
-
+ />
+ )}
+
diff --git a/ui/litellm-dashboard/src/components/add_model/cache_control_settings.test.tsx b/ui/litellm-dashboard/src/components/add_model/cache_control_settings.test.tsx
new file mode 100644
index 00000000000..2c403f7d8b2
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/add_model/cache_control_settings.test.tsx
@@ -0,0 +1,83 @@
+import { render, screen } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
+import { describe, expect, it, vi } from "vitest";
+import CacheControlInjectionPoints, { type CacheControlInjectionPoint } from "./cache_control_settings";
+
+const ROLE_HINT = "LiteLLM will mark all messages of this role as cacheable";
+const INDEX_HINT = "(Optional) If set litellm will mark the message at this index as cacheable";
+
+const ONE_POINT: CacheControlInjectionPoint[] = [{ location: "message" }];
+
+const tabTo = async (user: ReturnType, name: string): Promise => {
+ for (let step = 0; step < 8; step++) {
+ await user.tab();
+ if (document.activeElement === screen.getByRole("button", { name })) {
+ return;
+ }
+ }
+ throw new Error(`${name} is not reachable by keyboard`);
+};
+
+describe("CacheControlInjectionPoints field hints", () => {
+ it("explains on the Role field that the role marks every message of that role cacheable", async () => {
+ const user = userEvent.setup();
+ render( );
+
+ await user.hover(screen.getByRole("button", { name: "Role help" }));
+
+ expect(await screen.findByText(ROLE_HINT)).toBeInTheDocument();
+ });
+
+ it("explains on the Index field that it is optional and marks that message cacheable", async () => {
+ const user = userEvent.setup();
+ render( );
+
+ await user.hover(screen.getByRole("button", { name: "Index help" }));
+
+ expect(await screen.findByText(INDEX_HINT)).toBeInTheDocument();
+ });
+
+ it("reveals the Role hint on keyboard focus, so it is reachable without a pointer", async () => {
+ const user = userEvent.setup();
+ render( );
+
+ await tabTo(user, "Role help");
+
+ expect(await screen.findByText(ROLE_HINT)).toBeInTheDocument();
+ });
+
+ it("reveals the Index hint on keyboard focus, so it is reachable without a pointer", async () => {
+ const user = userEvent.setup();
+ render( );
+
+ await tabTo(user, "Index help");
+
+ expect(await screen.findByText(INDEX_HINT)).toBeInTheDocument();
+ });
+
+ it("keeps both hints behind a hover or a focus rather than rendering them inline", () => {
+ render( );
+
+ expect(screen.queryByText(ROLE_HINT)).not.toBeInTheDocument();
+ expect(screen.queryByText(INDEX_HINT)).not.toBeInTheDocument();
+ });
+
+ it("gives every row its own pair of hints", () => {
+ render(
+ ,
+ );
+
+ expect(screen.getAllByRole("button", { name: "Role help" })).toHaveLength(2);
+ expect(screen.getAllByRole("button", { name: "Index help" })).toHaveLength(2);
+ });
+
+ it("still reports a typed index as a string through onChange", async () => {
+ const user = userEvent.setup();
+ const onChange = vi.fn();
+ render( );
+
+ await user.type(screen.getByPlaceholderText("Optional"), "3");
+
+ expect(onChange).toHaveBeenLastCalledWith([{ location: "message", index: "3" }]);
+ });
+});
diff --git a/ui/litellm-dashboard/src/components/add_model/cache_control_settings.tsx b/ui/litellm-dashboard/src/components/add_model/cache_control_settings.tsx
index 3691f888a91..5b672595dcc 100644
--- a/ui/litellm-dashboard/src/components/add_model/cache_control_settings.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/cache_control_settings.tsx
@@ -1,155 +1,168 @@
+import { CircleHelp, Minus, Plus } from "lucide-react";
import React from "react";
-import { Form, Switch, Select, Typography } from "antd";
-import { PlusOutlined, MinusCircleOutlined } from "@ant-design/icons";
+
+import { Button } from "@/components/ui/button";
+import { Label } from "@/components/ui/label";
+import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
+import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
+
import NumericalInput from "../shared/numerical_input";
-const { Text } = Typography;
+export const CACHE_CONTROL_LABEL = "Cache Control Injection Points";
-interface CacheControlInjectionPoint {
+export const CACHE_CONTROL_TOOLTIP =
+ "Tell litellm where to inject cache control checkpoints. You can specify either by role (to apply to all messages of that role) or by specific message index.";
+
+export const CACHE_CONTROL_DESCRIPTION =
+ "Providers like Anthropic, Bedrock API require users to specify where to inject cache control checkpoints, litellm can automatically add them for you as a cost saving feature.";
+
+export const CACHE_CONTROL_ROLE_HINT = "LiteLLM will mark all messages of this role as cacheable";
+
+export const CACHE_CONTROL_INDEX_HINT = "(Optional) If set litellm will mark the message at this index as cacheable";
+
+export type CacheControlRole = "user" | "system" | "assistant";
+
+export interface CacheControlInjectionPoint {
location: "message";
- role?: "user" | "system" | "assistant";
- index?: number;
+ role?: CacheControlRole;
+ index?: string | number;
}
-interface CacheControlSettingsProps {
- form: any; // Form instance from parent
- showCacheControl: boolean;
- onCacheControlChange: (checked: boolean) => void;
+export const NEW_CACHE_CONTROL_POINT: CacheControlInjectionPoint = { location: "message" };
+
+const LOCATION_ITEMS = [{ value: "message", label: "Message" }] as const;
+
+const ROLE_ITEMS = [
+ { value: "user", label: "User" },
+ { value: "system", label: "System" },
+ { value: "assistant", label: "Assistant" },
+] as const;
+
+const LabelWithHint: React.FC<{ label: string; hint: string }> = ({ label, hint }) => (
+
+ {label}
+
+
+
+ }
+ >
+
+
+ {hint}
+
+
+
+);
+
+interface CacheControlInjectionPointsProps {
+ value?: CacheControlInjectionPoint[];
+ onChange?: (points: CacheControlInjectionPoint[]) => void;
}
-const CacheControlSettings: React.FC = ({
- form,
- showCacheControl,
- onCacheControlChange,
-}) => {
- const updateCacheControlPoints = (injectionPoints: CacheControlInjectionPoint[]) => {
- const currentParams = form.getFieldValue("litellm_extra_params");
- try {
- let paramsObj = currentParams ? JSON.parse(currentParams) : {};
- if (injectionPoints.length > 0) {
- paramsObj.cache_control_injection_points = injectionPoints;
- } else {
- delete paramsObj.cache_control_injection_points;
- }
- if (Object.keys(paramsObj).length > 0) {
- form.setFieldValue("litellm_extra_params", JSON.stringify(paramsObj, null, 2));
- } else {
- form.setFieldValue("litellm_extra_params", "");
- }
- } catch (error) {
- console.error("Error updating cache control points:", error);
- }
- };
+/**
+ * Editor for `cache_control_injection_points`. It holds no form state of its own so that an antd
+ * `Form.Item` and a react-hook-form `FormField` can each host it while their pages migrate
+ * independently; both hand a child exactly `value` and `onChange`.
+ */
+const CacheControlInjectionPoints: React.FC = ({ value, onChange }) => {
+ const points = value ?? [];
+
+ const replaceAt = (index: number, point: CacheControlInjectionPoint) =>
+ onChange?.(points.map((existing, position) => (position === index ? point : existing)));
return (
- <>
-
-
-
+
+
{CACHE_CONTROL_DESCRIPTION}
- {showCacheControl && (
-
-
- Providers like Anthropic, Bedrock API require users to specify where to inject cache control checkpoints,
- litellm can automatically add them for you as a cost saving feature.
-
-
-
- {(fields, { add, remove }) => (
- <>
- {fields.map((field, index) => (
-
-
-
-
-
-
- {
- const values = form.getFieldValue("cache_control_points");
- updateCacheControlPoints(values);
- }}
- />
-
-
-
- {
- const values = form.getFieldValue("cache_control_points");
- updateCacheControlPoints(values);
- }}
- />
-
-
- {fields.length > 1 && (
- {
- remove(field.name);
- setTimeout(() => {
- const values = form.getFieldValue("cache_control_points");
- updateCacheControlPoints(values);
- }, 0);
- }}
- />
- )}
-
+ {points.map((point, index) => (
+
+
+ Type
+
+
+
+
+
+ {LOCATION_ITEMS.map((item) => (
+
+ {item.label}
+
))}
+
+
+
-
- add()}
- >
-
- Add Injection Point
-
-
- >
- )}
-
+
+
+
+ replaceAt(index, { ...point, role: (selected as CacheControlRole | null) ?? undefined })
+ }
+ >
+
+
+
+
+ None
+ {ROLE_ITEMS.map((item) => (
+
+ {item.label}
+
+ ))}
+
+
+
+
+
+
+ ) =>
+ replaceAt(index, {
+ ...point,
+ index: event.target.value === "" ? undefined : event.target.value,
+ })
+ }
+ />
+
+
+ {points.length > 1 && (
+
onChange?.(points.filter((_, position) => position !== index))}
+ >
+
+
+ )}
- )}
- >
+ ))}
+
+ onChange?.([...points, NEW_CACHE_CONTROL_POINT])}
+ >
+
+ Add Injection Point
+
+
);
};
-export default CacheControlSettings;
+export default CacheControlInjectionPoints;
diff --git a/ui/litellm-dashboard/src/components/add_model/conditional_public_model_name.test.tsx b/ui/litellm-dashboard/src/components/add_model/conditional_public_model_name.test.tsx
index 81633bd7a8a..967fc9c458a 100644
--- a/ui/litellm-dashboard/src/components/add_model/conditional_public_model_name.test.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/conditional_public_model_name.test.tsx
@@ -1,13 +1,13 @@
import { render, screen } from "@testing-library/react";
-import { Form } from "antd";
import { describe, expect, it } from "vitest";
+import { MountedFormHost } from "../../../tests/mounted-form-host";
import ConditionalPublicModelName from "./conditional_public_model_name";
describe("ConditionalPublicModelName", () => {
it("should render", () => {
render(
-
{
}}
>
- ,
+ ,
);
expect(screen.getByText("Model Mappings")).toBeInTheDocument();
diff --git a/ui/litellm-dashboard/src/components/add_model/conditional_public_model_name.tsx b/ui/litellm-dashboard/src/components/add_model/conditional_public_model_name.tsx
index 8f95973b558..ff165415105 100644
--- a/ui/litellm-dashboard/src/components/add_model/conditional_public_model_name.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/conditional_public_model_name.tsx
@@ -1,24 +1,46 @@
import React, { useEffect, useState } from "react";
-import { Form, Table } from "antd";
+import { Table } from "antd";
+import { useFormContext, useWatch } from "react-hook-form";
import { Input } from "@/components/ui/input";
-import { Tooltip } from "../atoms/index";
+import { SimpleTooltip } from "@/components/ui/tooltip";
+import { antdRules } from "../common_components/antdFormRules";
+import { MountedFormField, type MountedFormValues } from "../common_components/MountedFormField";
import { Providers } from "../provider_info_helpers";
+interface ModelMapping {
+ public_name: string;
+ litellm_model: string;
+}
+
+const modelMappingsRule = {
+ validator: async (_: unknown, value: unknown) => {
+ if (!value || (value as ModelMapping[]).length === 0) {
+ throw new Error("At least one model mapping is required");
+ }
+ const invalidMappings = (value as ModelMapping[]).filter(
+ (mapping) => !mapping.public_name || mapping.public_name.trim() === "",
+ );
+ if (invalidMappings.length > 0) {
+ throw new Error("All model mappings must have valid public names");
+ }
+ },
+};
+
const ConditionalPublicModelName: React.FC = () => {
- const form = Form.useFormInstance();
+ const form = useFormContext
();
const [tableKey, setTableKey] = useState(0); // Add a key to force table re-render
// Watch the 'model' field for changes and ensure it's always an array
- const modelValue = Form.useWatch("model", form) || [];
+ const modelValue = useWatch({ control: form.control, name: "model" }) || [];
const selectedModels = Array.isArray(modelValue) ? modelValue : [modelValue];
- const customModelName = Form.useWatch("custom_model_name", form);
+ const customModelName = useWatch({ control: form.control, name: "custom_model_name" }) as string | undefined;
const showPublicModelName = !selectedModels.includes("all-wildcard");
- const selectedProvider = Form.useWatch("custom_llm_provider", form);
+ const selectedProvider = useWatch({ control: form.control, name: "custom_llm_provider" });
// Force table to re-render when custom model name changes
useEffect(() => {
if (customModelName && selectedModels.includes("custom")) {
- const currentMappings = form.getFieldValue("model_mappings") || [];
- const updatedMappings = currentMappings.map((mapping: any) => {
+ const currentMappings = (form.getValues("model_mappings") as ModelMapping[]) || [];
+ const updatedMappings = currentMappings.map((mapping) => {
if (mapping.public_name === "custom" || mapping.litellm_model === "custom") {
if (selectedProvider === Providers.Azure) {
return {
@@ -33,7 +55,7 @@ const ConditionalPublicModelName: React.FC = () => {
}
return mapping;
});
- form.setFieldValue("model_mappings", updatedMappings);
+ form.setValue("model_mappings", updatedMappings);
setTableKey((prev) => prev + 1); // Force table re-render
}
}, [customModelName, selectedModels, selectedProvider, form]);
@@ -42,13 +64,13 @@ const ConditionalPublicModelName: React.FC = () => {
useEffect(() => {
if (selectedModels.length > 0 && !selectedModels.includes("all-wildcard")) {
// Check if we already have mappings that match the selected models
- const currentMappings = form.getFieldValue("model_mappings") || [];
+ const currentMappings = (form.getValues("model_mappings") as ModelMapping[]) || [];
// Only update if the mappings don't exist or don't match the selected models
const shouldUpdateMappings =
currentMappings.length !== selectedModels.length ||
!selectedModels.every((model) =>
- currentMappings.some((mapping: { public_name: string; litellm_model: string }) => {
+ currentMappings.some((mapping) => {
if (model === "custom") {
return mapping.litellm_model === "custom" || mapping.litellm_model === customModelName;
}
@@ -85,7 +107,7 @@ const ConditionalPublicModelName: React.FC = () => {
};
});
- form.setFieldValue("model_mappings", mappings);
+ form.setValue("model_mappings", mappings);
setTableKey((prev) => prev + 1); // Force table re-render
}
}
@@ -98,16 +120,16 @@ const ConditionalPublicModelName: React.FC = () => {
The name you specify in your API calls to LiteLLM Proxy
Example: If you name your public model{" "}
- example-name, and choose{" "}
- openai/qwen-plus-latest as the LiteLLM model
+ example-name, and choose{" "}
+ openai/qwen-plus-latest as the LiteLLM model
Usage: You make an API call to the LiteLLM proxy with{" "}
- model = "example-name"
+ model = "example-name"
Result: LiteLLM sends{" "}
- qwen-plus-latest to the provider
+ qwen-plus-latest to the provider
>
);
@@ -119,7 +141,7 @@ const ConditionalPublicModelName: React.FC = () => {
title: (
Public Model Name
-
+
),
dataIndex: "public_name",
@@ -130,12 +152,12 @@ const ConditionalPublicModelName: React.FC = () => {
value={text}
onChange={(e) => {
const newValue = e.target.value;
- const newMappings = [...form.getFieldValue("model_mappings")];
+ const newMappings = [...((form.getValues("model_mappings") as ModelMapping[]) ?? [])];
// Check conditions for Anthropic -1m suffix handling
const isAnthropic = selectedProvider === Providers.Anthropic;
const endsWith1m = newValue.endsWith("-1m");
- const litellmParams = form.getFieldValue("litellm_extra_params");
+ const litellmParams = form.getValues("litellm_extra_params") as string | undefined;
const isLitellmParamsEmpty = !litellmParams || litellmParams.trim() === "";
let finalPublicName = newValue;
@@ -147,14 +169,14 @@ const ConditionalPublicModelName: React.FC = () => {
null,
2,
);
- form.setFieldValue("litellm_extra_params", litellmParamsValue);
+ form.setValue("litellm_extra_params", litellmParamsValue);
// Remove -1m suffix from public_name
finalPublicName = newValue.slice(0, -3); // Remove "-1m" (3 characters)
}
newMappings[index].public_name = finalPublicName;
- form.setFieldValue("model_mappings", newMappings);
+ form.setValue("model_mappings", newMappings);
}}
/>
);
@@ -164,7 +186,7 @@ const ConditionalPublicModelName: React.FC = () => {
title: (
LiteLLM Model Name
-
+
),
dataIndex: "litellm_model",
@@ -173,41 +195,28 @@ const ConditionalPublicModelName: React.FC = () => {
];
return (
- <>
- {
- if (!value || value.length === 0) {
- throw new Error("At least one model mapping is required");
- }
- // Check if all mappings have valid public names
- const invalidMappings = value.filter(
- (mapping: any) => !mapping.public_name || mapping.public_name.trim() === "",
- );
- if (invalidMappings.length > 0) {
- throw new Error("All model mappings must have valid public names");
- }
- },
- },
- ]}
- >
+
+ Model Mappings
+
+
+ }
+ required
+ rules={{ validate: antdRules(modelMappingsRule) }}
+ className="mb-4"
+ >
+ {(control) => (
-
- >
+ )}
+
);
};
diff --git a/ui/litellm-dashboard/src/components/add_model/litellm_model_name.test.tsx b/ui/litellm-dashboard/src/components/add_model/litellm_model_name.test.tsx
index 37eecec1f3f..64074481603 100644
--- a/ui/litellm-dashboard/src/components/add_model/litellm_model_name.test.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/litellm_model_name.test.tsx
@@ -1,28 +1,28 @@
import { render } from "@testing-library/react";
-import { Form } from "antd";
import { describe, expect, it } from "vitest";
import { getPlaceholder, Providers } from "../provider_info_helpers";
+import { MountedFormHost } from "../../../tests/mounted-form-host";
import LiteLLMModelNameField from "./litellm_model_name";
describe("LitellmModelNameField", () => {
it("should render", () => {
const { getByText } = render(
-
+
- ,
+ ,
);
expect(getByText("LiteLLM Model Name(s)")).toBeInTheDocument();
});
it("should show Azure placeholder as 'my-deployment'", () => {
const { getByPlaceholderText, queryByPlaceholderText } = render(
-
+
- ,
+ ,
);
expect(getByPlaceholderText("my-deployment")).toBeInTheDocument();
expect(queryByPlaceholderText("gpt-3.5-turbo")).not.toBeInTheDocument();
diff --git a/ui/litellm-dashboard/src/components/add_model/litellm_model_name.tsx b/ui/litellm-dashboard/src/components/add_model/litellm_model_name.tsx
index 86167a8902e..0af21b2adf6 100644
--- a/ui/litellm-dashboard/src/components/add_model/litellm_model_name.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/litellm_model_name.tsx
@@ -1,7 +1,11 @@
import React from "react";
-import { Form, Select as AntSelect } from "antd";
+import { Select as AntSelect } from "antd";
+import { useFormContext, useWatch } from "react-hook-form";
import { Input } from "@/components/ui/input";
import { Row, Col } from "antd";
+import { antdRequired } from "../common_components/antdFormRules";
+import { labelWithHint } from "@/components/shared/form/LabelWithHint";
+import { MountedFormField, type MountedFormValues } from "../common_components/MountedFormField";
import { Providers } from "../provider_info_helpers";
interface LiteLLMModelNameFieldProps {
@@ -15,7 +19,9 @@ const LiteLLMModelNameField: React.FC = ({
providerModels,
getPlaceholder,
}) => {
- const form = Form.useFormInstance();
+ const form = useFormContext();
+ const modelValue = useWatch({ control: form.control, name: "model" });
+ const selectedModels = Array.isArray(modelValue) ? modelValue : [modelValue];
const handleModelChange = (value: string | string[]) => {
// Ensure value is always treated as an array
@@ -23,10 +29,11 @@ const LiteLLMModelNameField: React.FC = ({
// If "all-wildcard" is selected, clear the model_name field
if (values.includes("all-wildcard")) {
- form.setFieldsValue({ model_name: undefined, model_mappings: [] });
+ form.setValue("model_name", undefined);
+ form.setValue("model_mappings", []);
} else {
// Get current model value to check if we need to update
- const currentModel = form.getFieldValue("model");
+ const currentModel = form.getValues("model");
// Only update if the value has actually changed
if (JSON.stringify(currentModel) !== JSON.stringify(values)) {
@@ -45,10 +52,8 @@ const LiteLLMModelNameField: React.FC = ({
});
// Update both fields in one call to reduce re-renders
- form.setFieldsValue({
- model: values,
- model_mappings: mappings,
- });
+ form.setValue("model", values);
+ form.setValue("model_mappings", mappings);
}
}
};
@@ -67,10 +72,8 @@ const LiteLLMModelNameField: React.FC = ({
: [];
// Update both fields
- form.setFieldsValue({
- model: deploymentName,
- model_mappings: mappings,
- });
+ form.setValue("model", deploymentName);
+ form.setValue("model_mappings", mappings);
};
// Handle custom model name changes
@@ -78,7 +81,7 @@ const LiteLLMModelNameField: React.FC = ({
const customName = e.target.value;
// Immediately update the model mappings
- const currentMappings = form.getFieldValue("model_mappings") || [];
+ const currentMappings = (form.getValues("model_mappings") as any[]) || [];
const updatedMappings = currentMappings.map((mapping: any) => {
if (mapping.public_name === "custom" || mapping.litellm_model === "custom") {
if (selectedProvider === Providers.Azure) {
@@ -95,43 +98,54 @@ const LiteLLMModelNameField: React.FC = ({
return mapping;
});
- form.setFieldsValue({ model_mappings: updatedMappings });
+ form.setValue("model_mappings", updatedMappings);
};
return (
<>
-
-
- {selectedProvider === Providers.Azure ||
+ {(control) =>
+ selectedProvider === Providers.Azure ||
selectedProvider === Providers.OpenAI_Compatible ||
selectedProvider === Providers.Ollama ? (
- <>
-
- >
+ {
+ control.onChange(event);
+ if (selectedProvider === Providers.Azure) {
+ handleAzureDeploymentNameChange(event);
+ }
+ }}
+ />
) : providerModels.length > 0 ? (
{
+ control.onChange(value);
+ handleModelChange(value);
+ }}
optionFilterProp="children"
filterOption={(input, option) => (option?.label ?? "").toLowerCase().includes(input.toLowerCase())}
options={[
@@ -151,34 +165,40 @@ const LiteLLMModelNameField: React.FC = ({
style={{ width: "100%" }}
/>
) : (
-
- )}
-
+
+ )
+ }
+
- {/* Custom Model Name field */}
- prevValues.model !== currentValues.model}>
- {({ getFieldValue }) => {
- const selectedModels = getFieldValue("model") || [];
- const modelArray = Array.isArray(selectedModels) ? selectedModels : [selectedModels];
- return (
- modelArray.includes("custom") && (
-
-
-
- )
- );
- }}
-
-
+ {selectedModels.includes("custom") && (
+
+ {(control) => (
+ {
+ control.onChange(event);
+ handleCustomModelNameChange(event);
+ }}
+ />
+ )}
+
+ )}
diff --git a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.test.tsx b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.test.tsx
index 351b0db0b4b..c3e8accae0a 100644
--- a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.test.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.test.tsx
@@ -1,8 +1,8 @@
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
import { fireEvent, render, screen, waitFor } from "@testing-library/react";
-import { Form } from "antd";
import { beforeAll, describe, expect, it, vi } from "vitest";
import { Providers } from "../provider_info_helpers";
+import { MountedFormHost } from "../../../tests/mounted-form-host";
import ProviderSpecificFields from "./provider_specific_fields";
vi.mock("../networking", async () => {
@@ -129,9 +129,9 @@ describe("ProviderSpecificFields", () => {
const queryClient = createQueryClient();
render(
-
+
-
+
,
);
@@ -144,9 +144,9 @@ describe("ProviderSpecificFields", () => {
const queryClient = createQueryClient();
render(
-
+
-
+
,
);
@@ -165,9 +165,9 @@ describe("ProviderSpecificFields", () => {
const queryClient = createQueryClient();
render(
-
+
-
+
,
);
@@ -182,9 +182,9 @@ describe("ProviderSpecificFields", () => {
const queryClient = createQueryClient();
render(
-
+
-
+
,
);
@@ -200,9 +200,9 @@ describe("ProviderSpecificFields", () => {
const queryClient = createQueryClient();
render(
-
+
-
+
,
);
@@ -231,9 +231,9 @@ describe("ProviderSpecificFields", () => {
const queryClient = createQueryClient();
render(
-
+
-
+
,
);
@@ -256,9 +256,9 @@ describe("ProviderSpecificFields", () => {
const queryClient = createQueryClient();
render(
-
+
-
+
,
);
@@ -281,9 +281,9 @@ describe("ProviderSpecificFields", () => {
const queryClient = createQueryClient();
render(
-
+
-
+
,
);
@@ -316,9 +316,9 @@ describe("ProviderSpecificFields", () => {
const queryClient = createQueryClient();
render(
-
+
-
+
,
);
diff --git a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx
index 2bbf2b0cfa5..955167c4351 100644
--- a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx
@@ -1,10 +1,18 @@
import { useProviderFields } from "@/app/(dashboard)/hooks/providers/useProviderFields";
import { UploadOutlined } from "@ant-design/icons";
import { Input } from "@/components/ui/input";
-import { Button as Button2, Col, Form, Input as AntdInput, Row, Select, Typography, Upload, UploadProps } from "antd";
+import { Button as Button2, Col, Input as AntdInput, Row, Select, Typography, Upload, UploadProps } from "antd";
import React from "react";
+import { useFormContext } from "react-hook-form";
+import { antdRequired } from "../common_components/antdFormRules";
+import {
+ MountedFormField,
+ type MountedFieldControlProps,
+ type MountedFormValues,
+} from "../common_components/MountedFormField";
import { CredentialItem, ProviderCredentialFieldMetadata } from "../networking";
import { provider_map, Providers } from "../provider_info_helpers";
+import { labelWithHint } from "@/components/shared/form/LabelWithHint";
const { Link } = Typography;
interface ProviderSpecificFieldsProps {
@@ -100,7 +108,7 @@ export const createCredentialFromModel = (provider: string, modelData: any): Cre
const ProviderSpecificFields: React.FC = ({ selectedProvider, uploadProps }) => {
const selectedProviderEnum = Providers[selectedProvider as keyof typeof Providers] as Providers;
- const form = Form.useFormInstance(); // Get form instance from context
+ const form = useFormContext();
const { data: providerMetadata, isLoading, error: loadError } = useProviderFields();
@@ -185,12 +193,12 @@ const ProviderSpecificFields: React.FC = ({ selecte
const apiVersion = getApiVersionFromApiBase(event.target.value);
if (apiVersion) {
lastInferredApiVersionRef.current = apiVersion;
- form.setFieldsValue({ api_version: apiVersion });
+ form.setValue("api_version", apiVersion);
return;
}
- if (form.getFieldValue("api_version") === lastInferredApiVersionRef.current) {
- form.setFieldsValue({ api_version: "" });
+ if (form.getValues("api_version") === lastInferredApiVersionRef.current) {
+ form.setValue("api_version", "");
}
lastInferredApiVersionRef.current = null;
},
@@ -206,7 +214,7 @@ const ProviderSpecificFields: React.FC = ({ selecte
reader.onload = (e) => {
if (e.target) {
const jsonStr = e.target.result as string;
- form.setFieldsValue({ vertex_credentials: jsonStr });
+ form.setValue("vertex_credentials", jsonStr);
}
};
reader.readAsText(file);
@@ -216,10 +224,17 @@ const ProviderSpecificFields: React.FC = ({ selecte
},
};
- const renderFieldControl = (field: ProviderCredentialField) => {
+ const renderFieldControl = (field: ProviderCredentialField, control: MountedFieldControlProps) => {
if (field.type === "select") {
return (
-
+
{field.options?.map((option) => (
{option}
@@ -234,6 +249,7 @@ const ProviderSpecificFields: React.FC = ({ selecte
{
+ control.onChange(info);
if (uploadProps?.onChange) {
uploadProps.onChange(info);
}
@@ -247,6 +263,10 @@ const ProviderSpecificFields: React.FC = ({ selecte
if (field.type === "textarea") {
return (
= ({ selecte
}
if (field.type === "password") {
- return ;
+ return (
+
+ );
}
return (
{
+ control.onChange(event);
+ if (field.key === "api_base") {
+ handleApiBaseChange(event);
+ }
+ }}
/>
);
};
@@ -281,7 +318,7 @@ const ProviderSpecificFields: React.FC = ({ selecte
{loadError && allFields.length === 0 && (
-
+
{loadError instanceof Error ? loadError.message : "Failed to load provider credential fields"}
@@ -289,15 +326,15 @@ const ProviderSpecificFields: React.FC = ({ selecte
)}
{allFields.map((field) => (
-
- {renderFieldControl(field)}
-
+ {(control) => renderFieldControl(field, control)}
+
{/* Special case for Vertex Credentials help text */}
{field.key === "vertex_credentials" && (
diff --git a/ui/litellm-dashboard/src/components/atoms/Tooltip.tsx b/ui/litellm-dashboard/src/components/atoms/Tooltip.tsx
deleted file mode 100644
index 02b03b518bf..00000000000
--- a/ui/litellm-dashboard/src/components/atoms/Tooltip.tsx
+++ /dev/null
@@ -1,40 +0,0 @@
-import React from "react";
-import { CircleHelp } from "lucide-react";
-
-import { Tooltip as ShadcnTooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
-import { cn } from "@/lib/cva.config";
-
-interface TooltipProps {
- content: React.ReactNode;
- children?: React.ReactNode;
- width?: string;
- className?: string;
-}
-
-const widthClassNames: Record = {
- "360px": "max-w-[360px]",
- "500px": "max-w-[500px]",
- auto: "max-w-xs",
-};
-
-export const Tooltip: React.FC = ({ content, children, width = "auto", className }) => (
-
-
-
- }
- >
- {children ?? }
-
-
- {content}
-
-
-
-);
diff --git a/ui/litellm-dashboard/src/components/atoms/index.ts b/ui/litellm-dashboard/src/components/atoms/index.ts
deleted file mode 100644
index 4b0a2f8749b..00000000000
--- a/ui/litellm-dashboard/src/components/atoms/index.ts
+++ /dev/null
@@ -1 +0,0 @@
-export { Tooltip } from "./Tooltip";
diff --git a/ui/litellm-dashboard/src/components/common_components/KeyLifecycleSettings.test.tsx b/ui/litellm-dashboard/src/components/common_components/KeyLifecycleSettings.test.tsx
index cfcbd841f63..d23e9da703a 100644
--- a/ui/litellm-dashboard/src/components/common_components/KeyLifecycleSettings.test.tsx
+++ b/ui/litellm-dashboard/src/components/common_components/KeyLifecycleSettings.test.tsx
@@ -1,6 +1,5 @@
import React, { useState } from "react";
-// eslint-disable-next-line no-restricted-imports -- exercising KeyLifecycleSettings requires hosting it in a real antd Form (the component it's built on)
-import { Form } from "antd";
+import { Controller, useForm } from "react-hook-form";
import userEvent, { PointerEventsCheckLevel } from "@testing-library/user-event";
import { describe, expect, it, vi, beforeEach } from "vitest";
import { renderWithProviders, screen, waitFor } from "../../../tests/test-utils";
@@ -15,30 +14,37 @@ interface HarnessProps {
}
const Harness: React.FC = ({ isCreateMode = true, onFinish = () => {} }) => {
- const [form] = Form.useForm();
+ const form = useForm<{ duration: string }>({ defaultValues: { duration: "" } });
const [autoRotationEnabled, setAutoRotationEnabled] = useState(false);
const [rotationInterval, setRotationInterval] = useState("");
const [neverExpire, setNeverExpire] = useState(false);
return (
-
-
-
-
+
+ (
+
+ )}
+ />
submit
- form.resetFields()}>
+ form.reset()}>
reset
{rotationInterval}
-
+
);
};
@@ -169,7 +175,7 @@ describe("KeyLifecycleSettings", () => {
});
it("shows the custom interval input when Custom interval is selected, without propagating yet", async () => {
- const user = userEvent.setup();
+ const user = userEvent.setup({ pointerEventsCheck: PointerEventsCheckLevel.Never });
renderWithProviders( );
await user.click(screen.getByRole("switch"));
@@ -184,7 +190,7 @@ describe("KeyLifecycleSettings", () => {
});
it("propagates a typed custom interval to the parent", async () => {
- const user = userEvent.setup();
+ const user = userEvent.setup({ pointerEventsCheck: PointerEventsCheckLevel.Never });
renderWithProviders( );
await user.click(screen.getByRole("switch"));
diff --git a/ui/litellm-dashboard/src/components/common_components/MemberTable.tsx b/ui/litellm-dashboard/src/components/common_components/MemberTable.tsx
index 19c2377bafe..2622c037558 100644
--- a/ui/litellm-dashboard/src/components/common_components/MemberTable.tsx
+++ b/ui/litellm-dashboard/src/components/common_components/MemberTable.tsx
@@ -1,4 +1,4 @@
-import { Tooltip } from "@/components/atoms/Tooltip";
+import { SimpleTooltip } from "@/components/ui/tooltip";
import { Member } from "@/components/networking";
import { StatusBadge } from "@/components/shared/table_cells";
import { Button } from "@/components/ui/button";
@@ -60,9 +60,9 @@ export default function MemberTable({
{roleTooltip ? (
{roleColumnTitle}
-
+
-
+
) : (
roleColumnTitle
diff --git a/ui/litellm-dashboard/src/components/common_components/MetadataKeyValueFields.test.tsx b/ui/litellm-dashboard/src/components/common_components/MetadataKeyValueFields.test.tsx
index 365cc8bbff5..073b11947a5 100644
--- a/ui/litellm-dashboard/src/components/common_components/MetadataKeyValueFields.test.tsx
+++ b/ui/litellm-dashboard/src/components/common_components/MetadataKeyValueFields.test.tsx
@@ -1,12 +1,14 @@
import { render, screen, waitFor } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
-import { Form } from "antd";
import React from "react";
import { describe, expect, it, vi } from "vitest";
+import { z } from "zod/v4";
import { TeamMetadataField } from "@/app/(dashboard)/hooks/teams/useTeamMetadataSchema";
+import { useZodForm } from "@/lib/forms/useZodForm";
import MetadataKeyValueFields, {
MetadataPair,
metadataObjectToPairs,
+ metadataPairsSchema,
metadataPairsToObject,
} from "./MetadataKeyValueFields";
@@ -95,13 +97,21 @@ interface HarnessProps {
schemaLoading?: boolean;
}
+const harnessSchema = z.object({ metadata: metadataPairsSchema });
+
const Harness: React.FC = ({ onFinish, initialMetadata, schemaFields, schemaLoading }) => {
- const [form] = Form.useForm();
+ const form = useZodForm(harnessSchema, { defaultValues: { metadata: initialMetadata ?? [] } });
return (
-
-
+ onFinish(values))}>
+
Save
-
+
);
};
diff --git a/ui/litellm-dashboard/src/components/common_components/MetadataKeyValueFields.tsx b/ui/litellm-dashboard/src/components/common_components/MetadataKeyValueFields.tsx
index da085f95ad8..1c67d48128f 100644
--- a/ui/litellm-dashboard/src/components/common_components/MetadataKeyValueFields.tsx
+++ b/ui/litellm-dashboard/src/components/common_components/MetadataKeyValueFields.tsx
@@ -1,14 +1,36 @@
-import { MinusCircleOutlined, PlusOutlined } from "@ant-design/icons";
-import { Button, Form, FormInstance, Input, Skeleton, Space } from "antd";
+import { CircleMinus, Plus } from "lucide-react";
import React, { useEffect, useRef } from "react";
+import {
+ useFieldArray,
+ type Control,
+ type FieldArrayPath,
+ type FieldPath,
+ type FieldValues,
+ type UseFormGetValues,
+} from "react-hook-form";
+import { z } from "zod/v4";
import { TeamMetadataField } from "@/app/(dashboard)/hooks/teams/useTeamMetadataSchema";
+import { FormField } from "@/components/shared/form/FormField";
+import { Button } from "@/components/ui/button";
+import { Input } from "@/components/ui/input";
+import { Skeleton } from "@/components/ui/skeleton";
export interface MetadataPair {
key: string;
value: string;
}
+export const metadataPairsSchema = z
+ .array(z.object({ key: z.string().min(1, "Missing key"), value: z.string().optional() }))
+ .superRefine((pairs, ctx) => {
+ pairs.forEach((pair, index) => {
+ if (pair.key && pairs.filter((other) => other.key === pair.key).length > 1) {
+ ctx.addIssue({ code: "custom", message: "Duplicate key", path: [index, "key"] });
+ }
+ });
+ });
+
function formatMetadataValue(value: unknown): string {
if (typeof value !== "string") {
return JSON.stringify(value) ?? "";
@@ -48,87 +70,82 @@ export function metadataPairsToObject(
);
}
-interface MetadataKeyValueFieldsProps {
- form: FormInstance;
- name?: string;
+interface MetadataKeyValueFieldsProps {
+ control: Control;
+ getValues: UseFormGetValues;
+ name: FieldArrayPath;
schemaFields?: readonly TeamMetadataField[];
schemaLoading?: boolean;
}
-const MetadataKeyValueFields: React.FC = ({
- form,
- name = "metadata",
+const MetadataKeyValueFields = ({
+ control,
+ getValues,
+ name,
schemaFields = [],
schemaLoading = false,
-}) => {
+}: MetadataKeyValueFieldsProps) => {
+ const { fields, append, remove } = useFieldArray({ control, name });
const seededRef = useRef(false);
useEffect(() => {
if (seededRef.current || schemaLoading || schemaFields.length === 0) return;
seededRef.current = true;
- const pairs: (Partial | undefined)[] = form.getFieldValue(name) ?? [];
+ const pairs: (Partial | undefined)[] = getValues(name as unknown as FieldPath) ?? [];
if (!Array.isArray(pairs)) return;
const existingKeys = new Set(pairs.map((pair) => pair?.key).filter(Boolean));
const seeded = schemaFields
.filter((field) => !existingKeys.has(field.key))
.map((field) => ({ key: field.key, value: "" }));
if (seeded.length > 0) {
- form.setFieldValue(name, [...pairs, ...seeded]);
+ append(seeded as never, { shouldFocus: false });
}
- }, [form, name, schemaFields, schemaLoading]);
+ }, [append, getValues, name, schemaFields, schemaLoading]);
if (schemaLoading) {
return (
-
-
+
+
+
+
);
}
return (
-
- {(fields, { add, remove }) => (
- <>
- {fields.map(({ key, name: fieldName, ...restField }) => (
-
- {
- if (!value) return Promise.resolve();
- const all: (Partial | undefined)[] = form.getFieldValue(name) ?? [];
- const dupes = all.filter((entry) => entry?.key === value);
- if (dupes.length > 1) {
- return Promise.reject(new Error("Duplicate key"));
- }
- return Promise.resolve();
- },
- },
- ]}
- >
-
-
-
-
-
- remove(fieldName)}
- style={{ color: "#ef4444" }}
- />
-
- ))}
-
- add()} block icon={ }>
- Add Key-Value Pair
-
-
- >
- )}
-
+ <>
+ {fields.map((field, index) => (
+
+ }>
+ {({ ref, value, ...rest }) => (
+
+ )}
+
+ }>
+ {({ ref, value, ...rest }) => (
+
+ )}
+
+ remove(index)}
+ >
+
+
+
+ ))}
+
append({ key: "", value: "" } as never, { shouldFocus: false })}
+ >
+
+ Add Key-Value Pair
+
+ >
);
};
diff --git a/ui/litellm-dashboard/src/components/common_components/MountedFormField.test.tsx b/ui/litellm-dashboard/src/components/common_components/MountedFormField.test.tsx
new file mode 100644
index 00000000000..62cdd935131
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/common_components/MountedFormField.test.tsx
@@ -0,0 +1,187 @@
+import React from "react";
+import { render, renderHook, screen, waitFor } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
+import { describe, expect, it, vi } from "vitest";
+import { useForm } from "react-hook-form";
+import type { UseFormGetValues } from "react-hook-form";
+
+import {
+ MountedFormField,
+ MountedFormProvider,
+ projectMountedValues,
+ useMountRegistry,
+ type MountedFieldName,
+ type MountedFormValues,
+ type MountRegistry,
+} from "./MountedFormField";
+
+const registryOf = (names: readonly MountedFieldName[]): MountRegistry => ({
+ register: () => () => undefined,
+ mountedNames: () => names,
+});
+
+const getValuesOf = (store: Readonly
>): UseFormGetValues =>
+ ((names: readonly string[]) => names.map((name) => store[name])) as unknown as UseFormGetValues;
+
+const project = (store: Readonly>) =>
+ projectMountedValues(registryOf(Object.keys(store)), getValuesOf(store));
+
+const projectPaths = (entries: readonly (readonly [MountedFieldName, unknown])[]) => {
+ const store = Object.fromEntries(
+ entries.map(([name, value]) => [Array.isArray(name) ? name.join(".") : (name as string), value]),
+ );
+ return projectMountedValues(registryOf(entries.map(([name]) => name)), getValuesOf(store));
+};
+
+describe("projectMountedValues", () => {
+ it("keeps a flat name flat", () => {
+ expect(project({ server_name: "s1", transport: "http" })).toStrictEqual({ server_name: "s1", transport: "http" });
+ });
+
+ it("nests an ARRAY name into a credentials object", () => {
+ expect(
+ projectPaths([
+ [["credentials", "aws_region_name"], "us-east-1"],
+ [["credentials", "aws_access_key_id"], "AKIA"],
+ ]),
+ ).toStrictEqual({ credentials: { aws_region_name: "us-east-1", aws_access_key_id: "AKIA" } });
+ });
+
+ it("keeps a literal dotted STRING name flat, matching antd getNamePath toArray", () => {
+ expect(projectPaths([["a.b", 1]])).toStrictEqual({ "a.b": 1 });
+ expect(projectPaths([["schema.property.with.dots", "v"]])).toStrictEqual({ "schema.property.with.dots": "v" });
+ });
+
+ it("rebuilds Form.List rows as an array, not an object keyed by digits", () => {
+ const projected = projectPaths([
+ [["env_vars", "0", "name"], "API_KEY"],
+ [["env_vars", "0", "description"], "the key"],
+ [["env_vars", "1", "name"], "REGION"],
+ ]);
+ expect(projected).toStrictEqual({
+ env_vars: [{ name: "API_KEY", description: "the key" }, { name: "REGION" }],
+ });
+ expect(Array.isArray(projected.env_vars)).toBe(true);
+ });
+
+ it("rebuilds static_headers rows, the second Form.List site", () => {
+ expect(
+ projectPaths([
+ [["static_headers", "0", "key"], "X-Tenant"],
+ [["static_headers", "0", "value"], "acme"],
+ ]),
+ ).toStrictEqual({
+ static_headers: [{ key: "X-Tenant", value: "acme" }],
+ });
+ });
+
+ it("emits a mounted-but-unset field as a key holding undefined, matching antd onFinish", () => {
+ const projected = project({ alias: undefined });
+ expect(Object.keys(projected)).toStrictEqual(["alias"]);
+ expect(projected.alias).toBeUndefined();
+ });
+
+ it("leaves a sparse row index as a hole rather than shifting later rows down", () => {
+ const projected = projectPaths([[["env_vars", "2", "name"], "THIRD"]]) as { env_vars: readonly unknown[] };
+ expect(projected.env_vars).toHaveLength(3);
+ expect(projected.env_vars[2]).toStrictEqual({ name: "THIRD" });
+ });
+
+ it("mixes flat, nested and list names in one projection", () => {
+ expect(
+ projectPaths([
+ ["transport", "http"],
+ [["credentials", "client_id"], "cid"],
+ [["env_vars", "0", "name"], "K"],
+ ]),
+ ).toStrictEqual({
+ transport: "http",
+ credentials: { client_id: "cid" },
+ env_vars: [{ name: "K" }],
+ });
+ });
+});
+
+describe("useMountRegistry lifecycle", () => {
+ const GatedForm: React.FC<{
+ showOptional: boolean;
+ showRequired: boolean;
+ onFinish: (v: MountedFormValues) => void;
+ }> = ({ showOptional, showRequired, onFinish }) => {
+ const form = useForm({ mode: "onChange", defaultValues: { server_name: "keep" } });
+ const registry = useMountRegistry();
+ return (
+
+ {
+ event.preventDefault();
+ void form
+ .trigger(registry.mountedNames().map((n) => (Array.isArray(n) ? n.join(".") : (n as string))))
+ .then((valid) => {
+ if (valid) onFinish(projectMountedValues(registry, form.getValues));
+ });
+ }}
+ >
+
+ {(control) => (
+
+ )}
+
+ {showOptional && (
+
+ {(control) => (
+
+ )}
+
+ )}
+ {showRequired && (
+
+ {(control) => (
+
+ )}
+
+ )}
+ Submit
+
+
+ );
+ };
+
+ it("drops a field's key from the submitted payload once its gate unmounts it", async () => {
+ const onFinish = vi.fn();
+ const { rerender } = render( );
+
+ await userEvent.click(screen.getByRole("button", { name: "Submit" }));
+ await waitFor(() => expect(onFinish).toHaveBeenCalledTimes(1));
+ expect(Object.keys(onFinish.mock.calls[0][0] as object)).toContain("alias");
+
+ rerender( );
+ await userEvent.click(screen.getByRole("button", { name: "Submit" }));
+ await waitFor(() => expect(onFinish).toHaveBeenCalledTimes(2));
+ expect(Object.keys(onFinish.mock.calls[1][0] as object)).not.toContain("alias");
+ });
+
+ it("submits after a required field is unmounted, rather than validating a field the user can no longer see", async () => {
+ const onFinish = vi.fn();
+ const { rerender } = render( );
+
+ await userEvent.click(screen.getByRole("button", { name: "Submit" }));
+ expect(await screen.findByText("Token URL is required")).toBeInTheDocument();
+ expect(onFinish).not.toHaveBeenCalled();
+
+ rerender( );
+ await userEvent.click(screen.getByRole("button", { name: "Submit" }));
+ await waitFor(() => expect(onFinish).toHaveBeenCalledTimes(1));
+ expect(Object.keys(onFinish.mock.calls[0][0] as object)).not.toContain("token_url");
+ });
+
+ it("keeps a name mounted while a second field still holds a registration on it", () => {
+ const registry = renderHook(() => useMountRegistry()).result.current;
+ const releaseFirst = registry.register("credentials.scopes");
+ registry.register("credentials.scopes");
+
+ releaseFirst();
+
+ expect(registry.mountedNames()).toStrictEqual(["credentials.scopes"]);
+ });
+});
diff --git a/ui/litellm-dashboard/src/components/common_components/MountedFormField.tsx b/ui/litellm-dashboard/src/components/common_components/MountedFormField.tsx
new file mode 100644
index 00000000000..f0efa885d92
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/common_components/MountedFormField.tsx
@@ -0,0 +1,177 @@
+"use client";
+
+import * as React from "react";
+import {
+ Controller,
+ type Control,
+ type ControllerProps,
+ type RegisterOptions,
+ type UseFormGetValues,
+} from "react-hook-form";
+
+import { Field, FieldDescription, FieldError, FieldLabel } from "@/components/shared/form/field";
+
+export type MountedFormValues = Record;
+
+const fieldKey = (name: string | readonly string[]): string =>
+ Array.isArray(name) ? name.join(".") : (name as string);
+
+export type MountedFieldName = string | readonly string[];
+
+export interface MountRegistry {
+ readonly register: (name: MountedFieldName) => () => void;
+ readonly mountedNames: () => readonly MountedFieldName[];
+}
+
+export interface MountedFormContextValue {
+ readonly control: Control;
+ readonly registry: MountRegistry;
+}
+
+const missingProvider = (): never => {
+ throw new Error("MountedFormField requires a MountedFormProvider ancestor");
+};
+
+const MountedFormContext = React.createContext({
+ get control(): Control {
+ return missingProvider();
+ },
+ registry: {
+ register: missingProvider,
+ mountedNames: missingProvider,
+ },
+});
+
+export const MountedFormProvider = MountedFormContext.Provider;
+
+export const useMountRegistry = (): MountRegistry => {
+ const counts = React.useRef