diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 5607a170e33..a7ec31f2ffd 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -57,7 +57,7 @@ "limit": 5681 }, "reportMissingTypeArgument": { - "limit": 15608 + "limit": 15605 }, "reportMissingTypeStubs": { "limit": 40 @@ -108,7 +108,7 @@ "limit": 39154 }, "reportUnknownParameterType": { - "limit": 19947 + "limit": 19944 }, "reportUnknownVariableType": { "limit": 30772 @@ -132,7 +132,7 @@ "limit": 27 }, "reportUnusedClass": { - "limit": 23 + "limit": 21 }, "reportUnusedFunction": { "limit": 139 diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml index 7a8031216e0..bb580c82760 100644 --- a/enterprise/pyproject.toml +++ b/enterprise/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-enterprise" -version = "0.1.56" +version = "0.1.57" description = "Package for LiteLLM Enterprise features" readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.1.56" +version = "0.1.57" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-enterprise==", diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index e39f0dcf55a..e1d62b70c29 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.86" +version = "0.4.87" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.86" +version = "0.4.87" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-proxy-extras==", diff --git a/litellm/_logging.py b/litellm/_logging.py index 6add9d79a5b..7d3a30c6d1a 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -10,7 +10,7 @@ from typing import Any, Final import litellm from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads -from litellm.litellm_core_utils.secret_redaction import redact_string +from litellm.litellm_core_utils.secret_redaction import redact_string, redact_structured_value set_verbose = False @@ -59,6 +59,12 @@ def _redact_string(value: str) -> str: return redact_string(value) +def _redact_structured_value(key: str | None, value: str) -> str: + if not _ENABLE_SECRET_REDACTION: + return value + return redact_structured_value(key, value) + + def redact_secrets(value: str) -> str: """Public API: redact known secret/credential patterns from an arbitrary string. @@ -265,7 +271,7 @@ class JsonFormatter(Formatter): if record.exc_info: json_record["stacktrace"] = record.exc_text or self.formatException(record.exc_info) - return safe_dumps(json_record) + return safe_dumps(json_record, value_transform=_redact_structured_value) class CorrelationPlainFormatter(logging.Formatter): @@ -276,7 +282,7 @@ class CorrelationPlainFormatter(logging.Formatter): """ def format(self, record: logging.LogRecord) -> str: - formatted: Final = super().format(record) + formatted: Final = _redact_string(super().format(record)) trace_id: Final = getattr(record, "trace_id", None) session_id: Final = getattr(record, "session_id", None) if not trace_id and not session_id: diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 9363db12385..f2bd18cd046 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -20,6 +20,7 @@ from litellm.integrations.opentelemetry_utils.gen_ai_semconv import ( OTELSemconvCategory, parse_semconv_opt_in, ) +from litellm.integrations.otel.model.db_endpoint import db_span_attributes from litellm.integrations.otel.model.semconv import Metric from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.secret_redaction import redact_string @@ -718,6 +719,28 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): self._handle_failure(kwargs, response_obj, start_time, end_time) + def _start_service_span(self, payload: ServiceLoggerPayload, parent_otel_span: Span, start_time_ns: int) -> Span: + """Open a service span, named and classified by what the service is. + + A datastore call is an outbound CLIENT span carrying ``db.*`` semconv. + Without those a Postgres span says only ``service=postgres``, so the + backend falls back to the transport peer, which for Prisma is the local + query engine on loopback. Everything else stays an INTERNAL span. + """ + from opentelemetry import trace + from opentelemetry.trace import SpanKind + + attributes: Final = db_span_attributes(payload.service.value, payload.call_type) + span: Final = self.tracer.start_span( + name=payload.service, + context=trace.set_span_in_context(parent_otel_span), + start_time=start_time_ns, + kind=SpanKind.CLIENT if attributes else SpanKind.INTERNAL, + ) + for key, value in attributes.items(): + self.safe_set_attribute(span=span, key=key, value=value) + return span + async def async_service_success_hook( self, payload: ServiceLoggerPayload, @@ -726,7 +749,6 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): end_time: datetime | float | None = None, event_metadata: dict | None = None, ): - from opentelemetry import trace from opentelemetry.trace import Status, StatusCode _start_time_ns = 0 @@ -743,12 +765,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): _end_time_ns = self._to_ns(end_time) if parent_otel_span is not None: - _span_name: Final = payload.service - service_logging_span: Final = self.tracer.start_span( - name=_span_name, - context=trace.set_span_in_context(parent_otel_span), - start_time=_start_time_ns, - ) + service_logging_span: Final = self._start_service_span(payload, parent_otel_span, _start_time_ns) self.safe_set_attribute( span=service_logging_span, key="call_type", @@ -786,7 +803,6 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): end_time: float | datetime | None = None, event_metadata: dict | None = None, ): - from opentelemetry import trace from opentelemetry.trace import Status, StatusCode _start_time_ns = 0 @@ -803,12 +819,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): _end_time_ns = self._to_ns(end_time) if parent_otel_span is not None: - _span_name: Final = payload.service - service_logging_span: Final = self.tracer.start_span( - name=_span_name, - context=trace.set_span_in_context(parent_otel_span), - start_time=_start_time_ns, - ) + service_logging_span: Final = self._start_service_span(payload, parent_otel_span, _start_time_ns) self.safe_set_attribute( span=service_logging_span, key="call_type", diff --git a/litellm/integrations/otel/mappers/genai.py b/litellm/integrations/otel/mappers/genai.py index 032441535e0..79487e69ac4 100644 --- a/litellm/integrations/otel/mappers/genai.py +++ b/litellm/integrations/otel/mappers/genai.py @@ -18,6 +18,7 @@ from litellm.integrations.otel.mappers.utils import ( serialize_messages, tool_definition_attrs, ) +from litellm.integrations.otel.model.db_endpoint import db_span_attributes from litellm.integrations.otel.model.payloads import ( GuardrailSpanData, LLMCallSpanData, @@ -27,7 +28,6 @@ from litellm.integrations.otel.model.payloads import ( ToolDefinition, ) from litellm.integrations.otel.model.semconv import ( - DB, MCP, Error, GenAI, @@ -36,7 +36,6 @@ from litellm.integrations.otel.model.semconv import ( RpcSystem, Server, ) -from litellm.integrations.otel.model.spans import db_system class GenAIMapper: @@ -182,12 +181,8 @@ class GenAIMapper: def _service(cls, data: ServiceSpanData) -> AttributeMap: attrs: Final = collect(cls._SERVICE_ATTRS, data) # An outbound datastore call (DB_CALL / CLIENT span) also carries db.* - # semconv. Internal services (router, budget jobs, …) have no db.system, - # so they get only the litellm.service.* keys above. - system: Final = db_system(data.service_name) - if system is not None: - attrs[DB.SYSTEM_NAME] = system - if data.call_type: - attrs[DB.OPERATION_NAME] = data.call_type + # semconv naming the server it reached. Internal services (router, budget + # jobs, …) have no db.system, so they get only the litellm.service.* keys. + attrs.update(db_span_attributes(data.service_name, data.call_type)) attrs.update({f"{LiteLLM.METADATA_PREFIX}{key}": value for key, value in data.event_metadata.items()}) return attrs diff --git a/litellm/integrations/otel/model/db_endpoint.py b/litellm/integrations/otel/model/db_endpoint.py new file mode 100644 index 00000000000..562162a8f31 --- /dev/null +++ b/litellm/integrations/otel/model/db_endpoint.py @@ -0,0 +1,164 @@ +"""OTel ``db.*`` / ``server.*`` attributes naming the database litellm talks to. + +Prisma reaches PostgreSQL through a query engine listening on loopback, so +transport-level instrumentation attributes the work to ``localhost`` and an +operator cannot tell it is a PostgreSQL call or correlate it with the database's +own metrics. These attributes name the real server on litellm's DB spans. + +Only the host, port, database and schema of the DSN are read, so no credential +can reach an exporter. +""" + +from __future__ import annotations + +import os +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final +from urllib.parse import ParseResult, parse_qs, unquote, urlparse + +from litellm.integrations.otel.model.semconv import DB, Server +from litellm.integrations.otel.model.spans import POSTGRESQL, db_system + +_DATABASE_URL_ENV: Final = "DATABASE_URL" +_READ_REPLICA_ENV: Final = "DATABASE_URL_READ_REPLICA" +_DEFAULT_POSTGRES_PORT: Final = 5432 +_DEFAULT_POSTGRES_SCHEMA: Final = "public" +_POSTGRES_SCHEMES: Final = frozenset({"postgres", "postgresql"}) +_EMPTY_ATTRIBUTES: Final[Mapping[str, str | int]] = MappingProxyType({}) + + +@dataclass(frozen=True, slots=True) +class DatabaseEndpoint: + """The non-sensitive identity of a PostgreSQL server, parsed from a DSN.""" + + address: str | None + port: int | None + namespace: str | None + + +def parse_database_endpoint(url: str | None) -> DatabaseEndpoint | None: + """Parse a PostgreSQL DSN into its exportable endpoint identity. + + Returns ``None`` for an absent, malformed or non-PostgreSQL URL rather than + raising: an unparseable DSN must degrade to a span without endpoint + attributes, never break the request that emitted it. + """ + if not url: + return None + try: + parsed: Final = urlparse(url) + if parsed.scheme not in _POSTGRES_SCHEMES: + return None + query: Final = parse_qs(parsed.query) + raw_database: Final = (parsed.path or "").lstrip("/") + if _is_misparsed_authority(parsed, url, raw_database): + return None + # ``host=`` beats the netloc: it is how libpq names a Unix socket + # directory and how the Cloud SQL connector sits behind a localhost + # netloc, where the netloc is the very answer this module replaces. + address: Final = _first(query.get("host")) or parsed.hostname + # ``port=`` accompanies ``host=`` in a libpq URI, so honour it the same way. + port: Final = _port(_first(query.get("port")), parsed.port) if address else None + namespace: Final = _namespace(unquote(raw_database), _first(query.get("schema"))) + except ValueError: + return None + if address is None and namespace is None: + return None + return DatabaseEndpoint(address=address, port=port, namespace=namespace) + + +def _is_misparsed_authority(parsed: ParseResult, url: str, raw_database: str) -> bool: + """Whether the URL authority may have been truncated by an unencoded character. + + ``/``, ``#`` or ``?`` in a password ends the netloc early, so urlparse hands + back the username as the host, the leading digits of the password as the + port, and the rest of the credential as the path, query or fragment. The + stranded userinfo ``@`` is the only surviving evidence. + + A database name cannot hold an unencoded slash either, so a second path + segment is the same evidence. + + A DSN that carries the at-sign in a query parameter instead, such as + ``?application_name=svc@prod``, is indistinguishable from a mis-split by any + property of the parse: both leave no userinfo, a host, a port and a path. + Since guessing wrong publishes a credential fragment to a tracing backend, + that ambiguity resolves to refusing the endpoint. Such a DSN loses + ``server.address`` and ``db.namespace`` and keeps the rest of the span, + which is the cheaper error of the two. Percent-encode the at-sign to keep + them. + """ + if "/" in raw_database: + return True + return "@" in url and "@" not in parsed.netloc + + +def _first(values: Sequence[str] | None) -> str: + return values[0] if values else "" + + +def _port(from_query: str, from_netloc: int | None) -> int: + return int(from_query) if from_query.isdigit() else (from_netloc or _DEFAULT_POSTGRES_PORT) + + +def _namespace(database: str, schema: str) -> str | None: + """``{database}|{schema}`` per the PostgreSQL semconv, dropping absent halves. + + Only Prisma's literal default schema stays implicit. The match is + case-sensitive because Prisma quotes the name, so ``?schema=PUBLIC`` builds + a second schema alongside ``public`` and the two must not collapse to one + namespace. + """ + qualifier: Final = "" if schema == _DEFAULT_POSTGRES_SCHEMA else schema + return "|".join(part for part in (database, qualifier) if part) or None + + +def postgres_endpoint() -> DatabaseEndpoint | None: + """The PostgreSQL endpoint the process is currently connected to. + + Read from ``os.environ`` on every span, deliberately, on both counts. + + The environment is what Prisma itself connects with, so the span cannot + disagree with the connection; ``get_secret_str`` would consult a configured + secret manager first and could name a different server than the one serving + the query. And the value is not static: the RDS IAM refresh rebuilds the URL + from ``DATABASE_HOST``/``PORT``/``NAME``/``SCHEMA`` every rotation, the + reconnect path re-reads ``DATABASE_URL``, and the DB-backed + ``environment_variables`` config overlay can rewrite any of them after + startup, so a value cached for the process lifetime goes stale against a + connection that has genuinely moved. Nothing is memoized either: a cache + keyed on the URL would hold a rotated credential past its rotation, and the + parse is a single ``urlparse`` on a short string. + + A configured read replica yields ``None``: ``RoutingPrismaWrapper`` picks + reader or writer per Prisma call, underneath the span, so naming the writer + would attribute replica reads to the primary. + """ + if os.environ.get(_READ_REPLICA_ENV): + return None + return parse_database_endpoint(os.environ.get(_DATABASE_URL_ENV, "")) + + +def db_span_attributes(service_name: str, call_type: str | None = None) -> Mapping[str, str | int]: + """The ``db.*``/``server.*`` attributes for a datastore service call. + + Empty for services that are not outbound datastore calls. Endpoint + attributes are PostgreSQL-only: ``DATABASE_URL`` says nothing about where + the redis-backed services point. ``db.system`` rides alongside the current + ``db.system.name`` because Datadog's OTLP intake still types a database span + from the older key. + """ + system: Final = db_system(service_name) + if system is None: + return _EMPTY_ATTRIBUTES + endpoint: Final = postgres_endpoint() if system == POSTGRESQL else None + pairs: Final[tuple[tuple[str, str | int | None], ...]] = ( + (DB.SYSTEM_NAME, system), + (DB.SYSTEM_LEGACY, system), + (DB.OPERATION_NAME, call_type), + (Server.ADDRESS, endpoint.address if endpoint is not None else None), + (Server.PORT, endpoint.port if endpoint is not None else None), + (DB.NAMESPACE, endpoint.namespace if endpoint is not None else None), + ) + return MappingProxyType({key: value for key, value in pairs if value}) diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index 3d585c36b67..ada2822ba66 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -238,7 +238,11 @@ class DB: """ SYSTEM_NAME: Final = "db.system.name" + # Superseded by SYSTEM_NAME, dual-emitted because Datadog's OTLP intake + # still infers a span's database type from this key. + SYSTEM_LEGACY: Final = "db.system" OPERATION_NAME: Final = "db.operation.name" + NAMESPACE: Final = "db.namespace" class HTTP: diff --git a/litellm/integrations/otel/model/spans.py b/litellm/integrations/otel/model/spans.py index 0f67f0e7a7c..08318f78b7c 100644 --- a/litellm/integrations/otel/model/spans.py +++ b/litellm/integrations/otel/model/spans.py @@ -115,10 +115,12 @@ SPAN_REGISTRY: Final[dict[SpanRole, SpanSpec]] = { # redis-backed spend queues. Any service not mapped here is litellm-internal work # and stays an INTERNAL ``SERVICE`` span. This table is the single source of # datastore knowledge — both the role classifier and the mapper read it. +POSTGRESQL: Final = "postgresql" + _DB_SYSTEM_BY_SERVICE: Final[dict[str, str]] = { "redis": "redis", - "postgres": "postgresql", - "batch_write_to_db": "postgresql", + "postgres": POSTGRESQL, + "batch_write_to_db": POSTGRESQL, } diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 58300d87a5a..b20c75bbc2a 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1094,10 +1094,10 @@ class Logging(LiteLLMLoggingBaseClass): data=additional_args.get("complete_input_dict", {}), ) - _metadata["raw_request"] = str(curl_command) + _metadata["raw_request"] = _redact_string(str(curl_command)) # split up, so it's easier to parse in the UI self.model_call_details["raw_request_typed_dict"] = RawRequestTypedDict( - raw_request_api_base=str(additional_args.get("api_base") or ""), + raw_request_api_base=self._get_masked_api_base(str(additional_args.get("api_base") or "")), raw_request_body=self._get_raw_request_body(additional_args.get("complete_input_dict", {})), # NOTE: setting ignore_sensitive_headers to True will cause # the Authorization header to be leaked when calls to the health @@ -1111,8 +1111,10 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["raw_request_typed_dict"] = RawRequestTypedDict( error=str(e), ) - _metadata["raw_request"] = f"Unable to Log \ + _metadata["raw_request"] = _redact_string( + f"Unable to Log \ raw request: {e}" + ) if getattr(self, "logger_fn", None) and callable(self.logger_fn): try: self.logger_fn( @@ -1206,15 +1208,16 @@ class Logging(LiteLLMLoggingBaseClass): if _is_debugging_on() or self.litellm_request_debug: if json_logs: masked_headers: Final = self._get_masked_headers(headers) + masked_api_base: Final = self._get_masked_api_base(str(api_base or "")) if self.litellm_request_debug: verbose_logger.warning( # .warning ensures this shows up in all environments "POST Request Sent from LiteLLM", - extra={"api_base": {api_base}, **masked_headers}, + extra={"api_base": {masked_api_base}, **masked_headers}, ) else: verbose_logger.debug( "POST Request Sent from LiteLLM", - extra={"api_base": {api_base}, **masked_headers}, + extra={"api_base": {masked_api_base}, **masked_headers}, ) else: headers = additional_args.get("headers", {}) @@ -1254,8 +1257,6 @@ class Logging(LiteLLMLoggingBaseClass): curl_command = "\nRequest Sent from LiteLLM:\n" request_str: Final = additional_args.get("request_str", "") curl_command += request_str - elif api_base == "": - curl_command = str(self.model_call_details) return curl_command def _get_masked_headers(self, headers: dict, ignore_sensitive_headers: bool = False) -> dict: diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 07d5e6314dd..e4c1c9fc5cf 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1816,16 +1816,19 @@ def split_concatenated_json_objects(raw: str) -> list[dict[str, Any]]: This helper uses ``json.JSONDecoder.raw_decode()`` to walk the string and extract each JSON object individually. + The walk degrades gracefully: if the string is malformed or truncated + (e.g. a stream that ended mid-tool-call), whatever complete objects were + parsed before the bad tail are returned and the remainder is discarded + with a warning, rather than raising. The sole caller + (``_convert_to_bedrock_tool_call_invoke``) treats an empty result as + ``input={}`` so the conversation can continue instead of hard-failing. + Returns ------- list[dict] A list of parsed dicts – one per JSON object found. If *raw* is - empty or whitespace-only, an empty list is returned. - - Raises - ------ - json.JSONDecodeError - If the string contains text that cannot be parsed as JSON at all. + empty, whitespace-only, or wholly unparseable, an empty list is + returned. """ import json @@ -1845,7 +1848,17 @@ def split_concatenated_json_objects(raw: str) -> list[dict[str, Any]]: if idx >= length: break - obj, end_idx = decoder.raw_decode(raw, idx) + try: + obj, end_idx = decoder.raw_decode(raw, idx) + except json.JSONDecodeError as e: + verbose_logger.warning( + "split_concatenated_json_objects: discarding unparseable tool-call " + "arguments tail after %d complete object(s); decode_start=%d error=%s", + len(results), + idx, + e, + ) + break if isinstance(obj, dict): results.append(obj) else: diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 2ffe015c727..0ed15c43ccf 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -3712,7 +3712,13 @@ def _convert_to_bedrock_tool_call_invoke( _parts_list.append(cache_point_block) return _parts_list except Exception as e: - raise Exception(f"Unable to convert openai tool calls={tool_calls} to bedrock tool calls. Received error={e}") + tool_call_ids: Final = tuple(tool.get("id") for tool in tool_calls if isinstance(tool, dict)) + raise litellm.BadRequestError( + message=f"Unable to convert openai tool calls with ids={tool_call_ids} to bedrock tool calls. " + f"Received error={e}", + model=model or "", + llm_provider="bedrock", + ) from e def _append_bedrock_tool_result_media_block( diff --git a/litellm/litellm_core_utils/safe_json_dumps.py b/litellm/litellm_core_utils/safe_json_dumps.py index ebf45ed747c..a1b71593dda 100644 --- a/litellm/litellm_core_utils/safe_json_dumps.py +++ b/litellm/litellm_core_utils/safe_json_dumps.py @@ -1,4 +1,5 @@ import json +from collections.abc import Callable from typing import Any, Final from pydantic import BaseModel @@ -11,20 +12,32 @@ def strip_null_bytes(value: str) -> str: return value.replace("\x00", "") -def safe_dumps(data: Any, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH) -> str: +def safe_dumps( + data: Any, + max_depth: int = DEFAULT_MAX_RECURSE_DEPTH, + value_transform: Callable[[str | None, str], str] | None = None, +) -> str: """ Recursively serialize data while detecting circular references. If a circular reference is detected then a marker string is returned. NUL bytes are stripped from strings to prevent PostgreSQL 22P05 errors. + + value_transform, when given, is applied to every string leaf (and to the + str() fallback for non-serializable objects) with the mapping key the leaf + was reached under, so callers can rewrite values without touching structure. """ - def _serialize(obj: Any, seen: set, depth: int) -> Any: + def _transform(key: str | None, value: str) -> str: + return value if value_transform is None else value_transform(key, value) + + def _serialize(obj: Any, seen: set, depth: int, key: str | None = None) -> Any: # Check for maximum depth. if depth > max_depth: return "MaxDepthExceeded" # Base-case: if it is a primitive, simply return it. if isinstance(obj, str): - return obj.replace("\x00", "") if "\x00" in obj else obj + cleaned = obj.replace("\x00", "") if "\x00" in obj else obj + return _transform(key, cleaned) if isinstance(obj, (int, float, bool, type(None))): return obj # Check for circular reference. @@ -37,30 +50,30 @@ def safe_dumps(data: Any, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH) -> str: for k, v in obj.items(): if isinstance(k, (str)): clean_k = k.replace("\x00", "") if "\x00" in k else k - result[clean_k] = _serialize(v, seen, depth + 1) + result[clean_k] = _serialize(v, seen, depth + 1, clean_k) seen.remove(id(obj)) return result elif isinstance(obj, list): - result = [_serialize(item, seen, depth + 1) for item in obj] + result = [_serialize(item, seen, depth + 1, key) for item in obj] seen.remove(id(obj)) return result elif isinstance(obj, tuple): - result = tuple(_serialize(item, seen, depth + 1) for item in obj) + result = tuple(_serialize(item, seen, depth + 1, key) for item in obj) seen.remove(id(obj)) return result elif isinstance(obj, set): - result = sorted([_serialize(item, seen, depth + 1) for item in obj]) + result = sorted([_serialize(item, seen, depth + 1, key) for item in obj]) seen.remove(id(obj)) return result elif isinstance(obj, BaseModel): dumped: Final = obj.model_dump() - result = _serialize(dumped, seen, depth + 1) + result = _serialize(dumped, seen, depth + 1, key) seen.remove(id(obj)) return result else: # Fall back to string conversion for non-serializable objects. try: - return strip_null_bytes(str(obj)) + return _transform(key, strip_null_bytes(str(obj))) except Exception: return "Unserializable Object" diff --git a/litellm/litellm_core_utils/secret_redaction.py b/litellm/litellm_core_utils/secret_redaction.py index c991a953530..5d5bd547d22 100644 --- a/litellm/litellm_core_utils/secret_redaction.py +++ b/litellm/litellm_core_utils/secret_redaction.py @@ -24,9 +24,6 @@ def _build_secret_patterns() -> "re.Pattern[str]": r"(?:client_secret|azure_password|azure_username)\s+[^\s,'\"})\]{}>]+", # AWS access key IDs r"(?:AKIA|ASIA)[0-9A-Z]{16}", - # AWS secrets / session tokens / access key IDs (key=value) - r"(?:aws_secret_access_key|aws_session_token|aws_access_key_id)" - r"\s*[:=]\s*[A-Za-z0-9/+=]{20,}", # Bearer tokens (OAuth, JWT, etc.) r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*", # Basic auth headers @@ -61,6 +58,7 @@ def _build_secret_patterns() -> "re.Pattern[str]": # private_key with PEM-aware value capture r"""private_key['\"]?\s*[:=]\s*['\"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'\"})\]{}>]+)""", r"(?:master_key|xai_key|database_url|db_url|connection_string|" + r"aws_secret_access_key|aws_session_token|aws_access_key_id|" r"signing_key|encryption_key|" r"auth_token|access_token|refresh_token|" r"slack_webhook_url|webhook_url|" @@ -83,3 +81,19 @@ _SECRET_RE: Final = _build_secret_patterns() def redact_string(value: str) -> str: """Scrub known secret/credential patterns from *value* and return the result.""" return _SECRET_RE.sub(_REDACTED, value) + + +def redact_structured_value(key: str | None, value: str) -> str: + """Scrub *value* as it appeared under *key* inside a structured record. + + redact_string() replaces a whole ``key: value`` span with REDACTED, which is + fine inside free text but destroys the surrounding syntax when the span is a + JSON member rather than message content. This renders the pair the way a dict + repr would, so the key-name patterns still fire, but collapses only the value + so the caller's structure survives. + """ + scrubbed: Final = redact_string(value) + if scrubbed != value or key is None: + return scrubbed + rendered: Final = f"'{key}': '{value}'" + return _REDACTED if redact_string(rendered) != rendered else value diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index 0d6d942e686..d147063df73 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -5,7 +5,7 @@ Common base config for all LLM providers import types from abc import ABC, abstractmethod from collections.abc import AsyncIterator, Iterator -from typing import TYPE_CHECKING, Any, Final, Union, cast +from typing import TYPE_CHECKING, Any, Final, Union import httpx from pydantic import BaseModel @@ -90,9 +90,9 @@ class BaseConfig(ABC): return type_to_response_format_param(response_format=response_format) def is_thinking_enabled(self, non_default_params: dict) -> bool: - return (non_default_params.get("thinking") or {}).get("type") == "enabled" or non_default_params.get( - "reasoning_effort" - ) is not None + thinking: Final = non_default_params.get("thinking") + thinking_type: Final = thinking.get("type") if isinstance(thinking, dict) else None + return thinking is True or thinking_type == "enabled" or non_default_params.get("reasoning_effort") is not None def is_max_tokens_in_request(self, non_default_params: dict) -> bool: """ @@ -112,7 +112,10 @@ class BaseConfig(ABC): if is_thinking_enabled and ( "max_tokens" not in non_default_params and "max_completion_tokens" not in non_default_params ): - thinking_token_budget: Final = cast(dict, optional_params["thinking"]).get("budget_tokens", None) + thinking_value: Final = optional_params.get("thinking") + thinking_token_budget: Final = ( + thinking_value.get("budget_tokens") if isinstance(thinking_value, dict) else None + ) if thinking_token_budget is not None: optional_params["max_tokens"] = thinking_token_budget + DEFAULT_MAX_TOKENS diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index fd07999395b..cee89f42c2d 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1090,7 +1090,10 @@ class AmazonConverseConfig(BaseConfig): is_thinking_enabled: Final = self.is_thinking_enabled(optional_params) is_max_tokens_in_request: Final = self.is_max_tokens_in_request(non_default_params) if is_thinking_enabled and not is_max_tokens_in_request: - thinking_token_budget: Final = cast(dict, optional_params["thinking"]).get("budget_tokens", None) + thinking_value: Final = optional_params.get("thinking") + thinking_token_budget: Final = ( + thinking_value.get("budget_tokens") if isinstance(thinking_value, dict) else None + ) if thinking_token_budget is not None: optional_params["maxTokens"] = thinking_token_budget + DEFAULT_MAX_TOKENS diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index eccc783dd8b..3ee803646a9 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2,7 +2,7 @@ import asyncio import json import os import ssl -from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping +from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence from contextlib import asynccontextmanager from functools import lru_cache from types import ModuleType @@ -73,6 +73,7 @@ from litellm.llms.custom_httpx.http_handler import ( from litellm.responses.streaming_iterator import ( BaseResponsesAPIStreamingIterator, MockResponsesAPIStreamingIterator, + ProjectQuotaCallback, ResponsesAPIStreamingIterator, ResponsesWebSocketStreaming, SyncResponsesAPIStreamingIterator, @@ -256,6 +257,27 @@ def _has_pre_call_deployment_hook(logging_obj: LiteLLMLoggingObj) -> bool: return False +def _collect_ws_project_quota_callbacks() -> tuple[ProjectQuotaCallback, ...]: + """Duck-type discover proxy hooks exposing per-frame project ITPM/OTPM + enforcement, so the Responses WebSocket loop can charge every + ``response.create`` frame, not just the connection's first one. + + Uses duck-typing on ``litellm.callbacks`` (rather than importing the + proxy hook directly) to avoid a layering violation (SDK importing from + the proxy layer). + """ + import litellm as _litellm + + callbacks: Final = cast( # cast-ok: callback registry is inspected before protocol use + Sequence[object], _litellm.callbacks + ) + return tuple( + cast(ProjectQuotaCallback, callback) # cast-ok: required callback method is callable + for callback in callbacks + if callable(getattr(callback, "enforce_project_io_token_quota_for_frame", None)) + ) + + class BaseLLMHTTPHandler: async def _make_common_async_call( self, @@ -6179,6 +6201,8 @@ class BaseLLMHTTPHandler: - Uses ManagedResponsesWebSocketHandler which makes HTTP streaming calls - Forwards events over the websocket connection """ + _ws_quota_callbacks: Final = _collect_ws_project_quota_callbacks() + if responses_api_provider_config is None or not responses_api_provider_config.supports_native_websocket(): from litellm.responses.streaming_iterator import ( ManagedResponsesWebSocketHandler, @@ -6195,6 +6219,7 @@ class BaseLLMHTTPHandler: timeout=timeout, custom_llm_provider=custom_llm_provider, first_message=first_message, + quota_callbacks=_ws_quota_callbacks, **kwargs, ) await handler.run() @@ -6315,6 +6340,7 @@ class BaseLLMHTTPHandler: first_message=first_message, guardrail_callbacks=_ws_guardrail_callbacks, output_guardrail_callbacks=_ws_output_guardrail_callbacks, + quota_callbacks=_ws_quota_callbacks, authorized_model=model, ) await streaming.bidirectional_forward() @@ -9407,6 +9433,27 @@ class BaseLLMHTTPHandler: ) ###### VECTOR STORE HANDLER ###### + @staticmethod + def _pre_call_direct_vector_store_search( + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: str, + vector_store_id: str, + query: str | Sequence[str], + ) -> None: + """Direct providers have no HTTP request to echo, and an empty api_base makes the debug + logger fall back to dumping model_call_details, which holds stored provider credentials.""" + endpoint: Final = f"{custom_llm_provider}://{vector_store_id}" + logging_obj.pre_call( + input="", + api_key="", + additional_args={ # mutable-ok: pre_call's additional_args contract is a dict + "query": query, + "vector_store_id": vector_store_id, + "api_base": endpoint, + "request_str": f"direct vector store search: {endpoint}", + }, + ) + async def async_vector_store_search_handler( self, vector_store_id: str, @@ -9423,13 +9470,11 @@ class BaseLLMHTTPHandler: _is_async: bool = False, ) -> VectorStoreSearchResponse: if isinstance(vector_store_provider_config, BaseDirectVectorStoreConfig): - logging_obj.pre_call( - input="", - api_key="", - additional_args={ # mutable-ok: pre_call's additional_args contract is a dict - "query": query, - "vector_store_id": vector_store_id, - }, + self._pre_call_direct_vector_store_search( + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + vector_store_id=vector_store_id, + query=query, ) return await vector_store_provider_config.aexecute_search_vector_store_request( vector_store_id=vector_store_id, @@ -9554,13 +9599,11 @@ class BaseLLMHTTPHandler: ) if isinstance(vector_store_provider_config, BaseDirectVectorStoreConfig): - logging_obj.pre_call( - input="", - api_key="", - additional_args={ # mutable-ok: pre_call's additional_args contract is a dict - "query": query, - "vector_store_id": vector_store_id, - }, + self._pre_call_direct_vector_store_search( + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + vector_store_id=vector_store_id, + query=query, ) return vector_store_provider_config.execute_search_vector_store_request( vector_store_id=vector_store_id, diff --git a/litellm/llms/deepseek/chat/transformation.py b/litellm/llms/deepseek/chat/transformation.py index 24da5b79261..566c960333a 100644 --- a/litellm/llms/deepseek/chat/transformation.py +++ b/litellm/llms/deepseek/chat/transformation.py @@ -131,9 +131,11 @@ class DeepSeekChatConfig(OpenAIGPTConfig): - model supports reasoning (capability check) - user explicitly passed thinking={"type": "enabled"} (opt-in check) """ + thinking: Final = optional_params.get("thinking") return ( supports_reasoning(model=model, custom_llm_provider="deepseek") - and (optional_params.get("thinking") or {}).get("type") == "enabled" + and isinstance(thinking, dict) + and thinking.get("type") == "enabled" ) @staticmethod diff --git a/litellm/llms/tinyfish/search/transformation.py b/litellm/llms/tinyfish/search/transformation.py index ba9ca2e1bde..b688dc2cd01 100644 --- a/litellm/llms/tinyfish/search/transformation.py +++ b/litellm/llms/tinyfish/search/transformation.py @@ -14,6 +14,7 @@ import httpx from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.search.transformation import ( @@ -22,7 +23,7 @@ from litellm.llms.base_llm.search.transformation import ( ) from litellm.secret_managers.main import get_secret_str -_UrlEncodableParams: Final = TypeAdapter(dict[str, str | int | bool]) +_UrlEncodableParams: Final = TypeAdapter(dict[str, str | int | float | bool]) _StrList: Final = TypeAdapter(list[str]) _StrFrozenSet: Final = TypeAdapter(frozenset[str]) @@ -94,16 +95,16 @@ class TinyfishSearchConfig(BaseSearchConfig): TinyFish equivalents: - ``query`` (str or list[str]) → ``query`` (list joined by spaces) - ``country`` → ``location`` - - ``search_domain_filter`` (list[str]) → folded into the query as - ``() (site:a OR site:b ...)`` (TinyFish has no first-class - field today; see ML-2084 for the planned ``include_domains``) + - ``search_domain_filter`` (list[str]) → folded into the query using + search operators - ``max_results`` → not sent on the wire; stashed on ``self._caller_max_results`` for client-side response truncation (TinyFish doesn't honor it server-side) - ``max_tokens_per_page`` → silently dropped (no TinyFish equivalent) Any other ``optional_params`` keys are forwarded to TinyFish as-is. - dict/list values are JSON-encoded so they survive ``urlencode``. + dict and list values are JSON-encoded so structured payloads survive + ``urlencode``. Returns: ``{_TINYFISH_PARAMS_KEY: }``. @@ -144,14 +145,12 @@ class TinyfishSearchConfig(BaseSearchConfig): supported_perplexity: Final = _StrFrozenSet.validate_python(raw_supported) for param, value in optional_params.items(): if param not in supported_perplexity and param not in request_data: - # `fetch` expects a JSON-encoded object on the wire; accept the - # natural Python dict form and serialize here so callers don't - # have to pre-stringify. - if isinstance(value, dict): + # Serialize dicts/lists as JSON so structured params survive urlencode. + if isinstance(value, (dict, list)): value = json.dumps(value, separators=(",", ":")) # `urlencode` would render Python bool as "True"/"False" - # (capitalized). ux-labs validators require lowercase - # "true"/"false" (e.g. `include_thumbnail`); normalize here. + # (capitalized). TinyFish Search's bool params require lowercase + # "true"/"false" strings on the wire; normalize here. elif isinstance(value, bool): value = "true" if value else "false" request_data[param] = value @@ -167,17 +166,35 @@ class TinyfishSearchConfig(BaseSearchConfig): """ Transform a TinyFish response to LiteLLM's unified ``SearchResponse``. - Mappings (per-result): - - ``title`` → ``SearchResult.title`` (defaults to ``""`` if missing/null) - - ``url`` → ``SearchResult.url`` (defaults to ``""``) - - ``snippet`` → ``SearchResult.snippet`` (defaults to ``""``) - - all other per-result fields (``position``, ``site_name``, - ``thumbnail_url``, ``fetch``, ``fetch_error``, ...) ride through as - extras on ``SearchResult`` via its ``extra="allow"`` config. + Per-result field handling: + - ``title``, ``url``, ``snippet`` are declared on ``SearchResult`` and + populated by ``SearchResponse.model_validate`` when present. Missing + or ``None`` values are defaulted to ``""`` beforehand by + ``_default_missing_result_fields`` so a degraded result flows through + instead of failing the whole call. + - All undeclared per-result fields (``position``, ``site_name``, and + any others TinyFish returns) ride through as extras via + ``SearchResult``'s ``extra="allow"`` config — accessible as + attributes on the result object or enumerable via + ``result.model_extra``. - Top-level ``parameter_warnings`` (see ML-2085) is read when present and - each entry is re-fired via ``verbose_logger.warning``. Absent or - malformed entries are silently skipped — never throws. + Top-level ``parameter_warnings`` is read when present and each entry + is re-fired via ``verbose_logger.warning``. Absent or malformed + entries are silently skipped — never throws. + + Top-level extras (``query``, ``total_results``, ``page``, and any + future TinyFish additions) ride through via + ``SearchResponse.extra="allow"``. The validated response is returned + in place after truncating ``results`` to the caller's ``max_results``, + so every field pydantic populated survives regardless of which + storage bucket (declared attribute or ``__pydantic_extra__``) holds it. + + TinyFish response headers (e.g. ``x-request-id``, ``retry-after``, + ``x-ratelimit-limit`` — httpx normalizes header names to lowercase) + are stashed on ``response._hidden_params["headers"]`` (raw) and + ``response._hidden_params["additional_headers"]`` (sanitized via + ``process_response_headers``) so callers can correlate a search with + server-side logs. Error paths routed through ``self._wrap_error`` for uniform ``"TinyFish Search: . See for details."`` wrapping: @@ -223,7 +240,12 @@ class TinyfishSearchConfig(BaseSearchConfig): _emit_parameter_warnings(parsed) max_results: Final = self._caller_max_results or _TINYFISH_RESULT_CAP - return SearchResponse(results=list(parsed.results[:max_results])) + parsed.results = parsed.results[:max_results] + raw_headers: Final = dict(raw_response.headers) + hidden: Final = parsed._hidden_params # pyright: ignore[reportPrivateUsage] # sole hidden-params channel + hidden["headers"] = raw_headers + hidden["additional_headers"] = process_response_headers(raw_headers) + return parsed def _wrap_error( self, @@ -243,9 +265,9 @@ class TinyfishSearchConfig(BaseSearchConfig): carry the ``TinyFish Search:`` prefix — the bare error already names the host in the URL, so attribution is implicit there. """ - # ux-labs frontend wraps every error body as {"error": {"code", "message", "details"?}}. + # TinyFish Search wraps every error body as {"error": {"code", "message", "details"?}}. # Best-effort unwrap to surface the inner message; fall back to the raw body - # for non-ux-labs responses (CDN HTML pages, other JSON envelopes, plain text). + # for other envelope shapes (CDN HTML pages, other JSON envelopes, plain text). inner_message = error_message try: body: Final[object] = json.loads(error_message) # any-ok: json.loads -> Any @@ -290,7 +312,7 @@ def _default_missing_result_fields(raw_json: object) -> None: def _emit_parameter_warnings(parsed: SearchResponse) -> None: - """Re-fire TinyFish-side ``parameter_warnings`` (see ML-2085) as warnings. + """Re-fire TinyFish-side ``parameter_warnings`` as warnings. Defensive: skip silently on any shape we don't recognize so a malformed entry (or an early/partial rollout of the field) never throws. diff --git a/litellm/main.py b/litellm/main.py index cc27da830d8..f0b20eba9b6 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5007,7 +5007,6 @@ def completion( tool_choice = validate_chat_completion_tool_choice(tool_choice=tool_choice) # validate optional params stop = validate_openai_optional_params(stop=stop) - # normalize camelCase thinking keys (e.g. budgetTokens -> budget_tokens) thinking = validate_and_fix_thinking_param(thinking=thinking) ######### unpacking kwargs ##################### diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 409022016b0..07f9027313b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -14456,6 +14456,25 @@ "supports_tool_choice": true, "supports_output_config": true }, + "databricks/databricks-claude-opus-4-6": { + "input_cost_per_token": 5.00003e-06, + "input_dbu_cost_per_token": 7.1429e-05, + "litellm_provider": "databricks", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 2.5000010000000002e-05, + "output_dbu_cost_per_token": 0.000357143, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "databricks/databricks-claude-sonnet-4": { "input_cost_per_token": 2.9999900000000002e-06, "input_dbu_cost_per_token": 4.2857e-05, @@ -14513,6 +14532,25 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "databricks/databricks-claude-sonnet-4-6": { + "input_cost_per_token": 2.9999900000000002e-06, + "input_dbu_cost_per_token": 4.2857e-05, + "litellm_provider": "databricks", + "max_input_tokens": 1000000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 1.5000020000000002e-05, + "output_dbu_cost_per_token": 0.000214286, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "databricks/databricks-gemini-2-5-flash": { "input_cost_per_token": 3.0001999999999996e-07, "input_dbu_cost_per_token": 4.285999999999999e-06, @@ -14547,6 +14585,74 @@ "supports_function_calling": true, "supports_tool_choice": true }, + "databricks/databricks-gemini-3-1-flash-lite": { + "input_cost_per_token": 3.1248e-07, + "input_dbu_cost_per_token": 4.464e-06, + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 1.87502e-06, + "output_dbu_cost_per_token": 2.6786e-05, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supports_function_calling": true, + "supports_tool_choice": true + }, + "databricks/databricks-gemini-3-1-pro": { + "input_cost_per_token": 2.49998e-06, + "input_dbu_cost_per_token": 3.5714e-05, + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 1.5000020000000002e-05, + "output_dbu_cost_per_token": 0.000214286, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supports_function_calling": true, + "supports_tool_choice": true + }, + "databricks/databricks-gemini-3-flash": { + "input_cost_per_token": 6.2503e-07, + "input_dbu_cost_per_token": 8.929e-06, + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 3.74997e-06, + "output_dbu_cost_per_token": 5.3571e-05, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supports_function_calling": true, + "supports_tool_choice": true + }, + "databricks/databricks-gemini-3-pro": { + "input_cost_per_token": 2.49998e-06, + "input_dbu_cost_per_token": 3.5714e-05, + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 1.5000020000000002e-05, + "output_dbu_cost_per_token": 0.000214286, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supports_function_calling": true, + "supports_tool_choice": true + }, "databricks/databricks-gemma-3-12b": { "input_cost_per_token": 1.5000999999999998e-07, "input_dbu_cost_per_token": 2.1429999999999996e-06, @@ -14592,6 +14698,126 @@ "output_dbu_cost_per_token": 0.000142857, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving" }, + "databricks/databricks-gpt-5-1-codex-max": { + "input_cost_per_token": 1.24999e-06, + "input_dbu_cost_per_token": 1.7857e-05, + "litellm_provider": "databricks", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 9.999990000000002e-06, + "output_dbu_cost_per_token": 0.000142857, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving" + }, + "databricks/databricks-gpt-5-1-codex-mini": { + "input_cost_per_token": 2.4997e-07, + "input_dbu_cost_per_token": 3.571e-06, + "litellm_provider": "databricks", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 1.99997e-06, + "output_dbu_cost_per_token": 2.8571e-05, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving" + }, + "databricks/databricks-gpt-5-2": { + "input_cost_per_token": 1.75e-06, + "input_dbu_cost_per_token": 2.5e-05, + "litellm_provider": "databricks", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 1.4e-05, + "output_dbu_cost_per_token": 0.0002, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving" + }, + "databricks/databricks-gpt-5-2-codex": { + "input_cost_per_token": 1.75e-06, + "input_dbu_cost_per_token": 2.5e-05, + "litellm_provider": "databricks", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 1.4e-05, + "output_dbu_cost_per_token": 0.0002, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving" + }, + "databricks/databricks-gpt-5-3-codex": { + "input_cost_per_token": 1.75e-06, + "input_dbu_cost_per_token": 2.5e-05, + "litellm_provider": "databricks", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 1.4e-05, + "output_dbu_cost_per_token": 0.0002, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving" + }, + "databricks/databricks-gpt-5-4": { + "input_cost_per_token": 2.49998e-06, + "input_dbu_cost_per_token": 3.5714e-05, + "litellm_provider": "databricks", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 1.5000020000000002e-05, + "output_dbu_cost_per_token": 0.000214286, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving" + }, + "databricks/databricks-gpt-5-4-mini": { + "input_cost_per_token": 7.4998e-07, + "input_dbu_cost_per_token": 1.0714e-05, + "litellm_provider": "databricks", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 4.50002e-06, + "output_dbu_cost_per_token": 6.4286e-05, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving" + }, + "databricks/databricks-gpt-5-4-nano": { + "input_cost_per_token": 1.9999e-07, + "input_dbu_cost_per_token": 2.857e-06, + "litellm_provider": "databricks", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 1.24999e-06, + "output_dbu_cost_per_token": 1.7857e-05, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving" + }, "databricks/databricks-gpt-5-mini": { "input_cost_per_token": 2.4997000000000006e-07, "input_dbu_cost_per_token": 3.571e-06, diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index e46e6299277..1ca4c657706 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -1341,7 +1341,7 @@ async def _persist_dcr_client_registration( ``update_mcp_server`` merges credential blobs: a re-registered public client must not inherit the previous client's secret or auth method. """ - if mcp_server.is_true_passthrough or mcp_server.is_oauth_delegate: + if mcp_server.is_client_forwarded_token: return "skipped" try: @@ -2187,7 +2187,7 @@ async def _build_oauth_protected_resource_response( ) if upstream_metadata is not None: - if mcp_server.is_true_passthrough or mcp_server.is_oauth_delegate: + if mcp_server.is_client_forwarded_token: return upstream_metadata return {**upstream_metadata, "resource": resource_url} diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 7fff6c12fe0..65855df89f6 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -123,6 +123,7 @@ from litellm.proxy._experimental.mcp_server.utils import ( iter_known_server_prefixes, iter_known_tool_name_spellings, logging_safe_mcp_headers, + lookup_mcp_server_auth_in_headers, match_known_server_prefix, match_known_tool_name, merge_mcp_headers, @@ -873,6 +874,53 @@ def _openapi_forwarded_extra_headers( return forwarded or None +def _resolve_openapi_tool_auth( + mcp_server: MCPServer, + mcp_auth_header: str | None, + mcp_server_auth_headers: Mapping[str, str | dict[str, str]] | None, # mutable-ok: sink shape + raw_headers: dict[str, str] | None, # mutable-ok: sink takes a concrete dict + user_api_key_auth: UserAPIKeyAuth | None, +) -> tuple[str | None, dict[str, str] | None, str | dict[str, str] | None]: # mutable-ok: sink shapes + """The caller's upstream credential for one ``spec_path`` server, for both OpenAPI dispatch arms. + + A per-server ``x-mcp-{alias}-authorization`` wins over the deprecated global / BYOK + ``mcp_auth_header``, the same precedence ``_call_regular_mcp_tool`` applies, so the OpenAPI and + managed paths cannot disagree about which credential is authoritative. The two kinds are not + interchangeable: a per-server value is already a complete header value and is forwarded verbatim, + while a BYOK credential is a raw secret that takes the server's auth-type prefix. Formatting the + former would ship ``Bearer Bearer ``. + + Returns the ``Authorization`` value to inject, the extra headers to forward, and the credential to + hand ``resolve_openapi_upstream_auth``, whose passthrough arm reads it via + ``_passthrough_token_from_mcp_auth_header``. The per-server Authorization travels only in the + credential, never also in the forwarded headers, because the resolver pops Authorization out of + those and would otherwise have two sources to reconcile. + """ + forwarded: Final = _openapi_forwarded_extra_headers(mcp_server, raw_headers, user_api_key_auth) + per_server: Final = ( + lookup_mcp_server_auth_in_headers( + mcp_server_auth_headers, + alias=mcp_server.alias, + server_name=mcp_server.server_name, + ) + if mcp_server_auth_headers + else None + ) + + if isinstance(per_server, dict): + authorization: Final = next((v for k, v in per_server.items() if k.lower() == "authorization"), None) + merged: Final = merge_mcp_headers(extra_headers=forwarded, static_headers=_without_authorization(per_server)) + if authorization is None: + byok: Final = _format_byok_openapi_auth_header(mcp_server, mcp_auth_header) if mcp_auth_header else None + return byok, merged, mcp_auth_header + return authorization, merged, per_server + if isinstance(per_server, str) and per_server: + return per_server, forwarded, per_server + if mcp_auth_header: + return _format_byok_openapi_auth_header(mcp_server, mcp_auth_header), forwarded, mcp_auth_header + return None, forwarded, None + + async def _resolve_byok_mcp_auth_header( mcp_server: MCPServer, user_api_key_auth: UserAPIKeyAuth | None, @@ -1740,12 +1788,14 @@ class MCPServerManager: server: The MCP server whose OAuth metadata must be resolved. Returns: - The resolved server, or the registered server when no discovery is - pending. + The resolved server; the registered server when no discovery is + pending, or when discovery failed for a client-forwarded-token + server, whose session consumes no discovered endpoint. Raises: HTTPException: Status 503 when discovery times out or returns - incomplete metadata. + incomplete metadata for a server whose OAuth flow the gateway + runs itself. """ acquisition: Final = self._get_or_start_oauth_discovery_task(server) if acquisition is None: @@ -1764,6 +1814,8 @@ class MCPServerManager: return await self.ensure_oauth_metadata_discovered(server) case _OAuthDiscoveryFailed(timed_out=timed_out): current: Final = self._registered_server(server) + if current.is_client_forwarded_token: + return current server_ref: Final = current.alias or current.server_name or current.name or current.server_id reason: Final = "timed out" if timed_out else "returned incomplete metadata" raise HTTPException( @@ -2321,23 +2373,30 @@ class MCPServerManager: openapi_key_prefix: Final = prefix_root + MCP_TOOL_PREFIX_SEPARATOR global_mcp_tool_registry.unregister_tools_with_prefix(openapi_key_prefix) - owned_raw: Final[set[str]] = set() - for p in iter_known_server_prefixes(server): - if p: - owned_raw.add(p) - if server.name: - owned_raw.add(server.name) + owned_normalized: Final = self._owned_mapping_values(server) - owned_normalized: Final = {normalize_server_name(x) for x in owned_raw} - - stale_mapping_keys: Final[list[str]] = [] - for tool_name, mapped_server in list(self.tool_name_to_mcp_server_name_mapping.items()): - if mapped_server in owned_raw or normalize_server_name(str(mapped_server)) in owned_normalized: - stale_mapping_keys.append(tool_name) + stale_mapping_keys: Final = tuple( + tool_name + for tool_name, mapped_server in self.tool_name_to_mcp_server_name_mapping.items() + if normalize_server_name(str(mapped_server)) in owned_normalized + ) for key in stale_mapping_keys: del self.tool_name_to_mcp_server_name_mapping[key] + def _owned_mapping_values(self, server: MCPServer) -> frozenset[str]: + return frozenset( + normalize_server_name(value) for value in (*iter_known_server_prefixes(server), server.name) if value + ) + + def _server_exposes_tool(self, server: MCPServer, tool_name: str) -> bool: + owned: Final = self._owned_mapping_values(server) + mapped_owners: Final = ( + self.tool_name_to_mcp_server_name_mapping.get(spelling) + for spelling in iter_known_tool_name_spellings(tool_name, server) + ) + return any(owner is not None and normalize_server_name(owner) in owned for owner in mapped_owners) + def remove_server(self, mcp_server: LiteLLM_MCPServerTable): """ Remove a server from the registry @@ -3182,10 +3241,6 @@ class MCPServerManager: # Get server-specific auth header if available server_auth_header: str | dict[str, str] | None = None if mcp_server_auth_headers: - from litellm.proxy._experimental.mcp_server.utils import ( - lookup_mcp_server_auth_in_headers, - ) - server_auth_header = lookup_mcp_server_auth_in_headers( mcp_server_auth_headers, alias=server.alias, @@ -5210,11 +5265,6 @@ class MCPServerManager: # the exact case of server alias/name (e.g., '1litellmagcgateway' vs '1LiteLLMAGCGateway') server_auth_header: dict[str, str] | str | None = None if mcp_server_auth_headers: - # Normalize keys for case-insensitive lookup - from litellm.proxy._experimental.mcp_server.utils import ( - lookup_mcp_server_auth_in_headers, - ) - server_auth_header = lookup_mcp_server_auth_in_headers( mcp_server_auth_headers, alias=mcp_server.alias, @@ -5249,7 +5299,7 @@ class MCPServerManager: user_api_key_auth=user_api_key_auth, ): extra_headers = _without_authorization(extra_headers) - elif mcp_server.is_true_passthrough or mcp_server.is_oauth_delegate: + elif mcp_server.is_client_forwarded_token: extra_headers = _client_forwarded_authorization_headers( mcp_server=mcp_server, oauth2_headers=oauth2_headers, @@ -5356,7 +5406,7 @@ class MCPServerManager: # Scoped to the two client-forwarded token modes this stack introduced; legacy # oauth2 + delegate_auth_to_upstream (is_oauth_passthrough) is being removed, so it is not # added here even though the list path still relays for it. - relays_upstream_auth: Final = mcp_server.is_true_passthrough or mcp_server.is_oauth_delegate + relays_upstream_auth: Final = mcp_server.is_client_forwarded_token server_label: Final = mcp_server.name or mcp_server.server_name or mcp_server.alias or "" async def _call_tool_via_client(client, params): @@ -5463,13 +5513,8 @@ class MCPServerManager: if mcp_server is None: raise ValueError(f"Tool {name} not found") - if resolved_by_server_name_only: - tool_known: Final = ( - name in self.tool_name_to_mcp_server_name_mapping - or prefixed_tool_name in self.tool_name_to_mcp_server_name_mapping - ) - if not tool_known: - raise ValueError(f"Tool {name} not found") + if resolved_by_server_name_only and not self._server_exposes_tool(mcp_server, name): + raise ValueError(f"Tool {name} not found") return mcp_server @@ -5712,16 +5757,20 @@ class MCPServerManager: server_name, ) - auth_header_value: Final = ( - _format_byok_openapi_auth_header(mcp_server, mcp_auth_header) if mcp_auth_header else None + auth_header_value, openapi_forwarded_headers, upstream_credential = _resolve_openapi_tool_auth( + mcp_server=mcp_server, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + raw_headers=raw_headers, + user_api_key_auth=user_api_key_auth, ) resolved_auth_headers, forwarded_headers = await self.resolve_openapi_upstream_auth( mcp_server=mcp_server, oauth2_headers=caller_oauth2_headers, raw_headers=raw_headers, - mcp_auth_header=mcp_auth_header, + mcp_auth_header=upstream_credential, user_api_key_auth=user_api_key_auth, - forwarded_headers=_openapi_forwarded_extra_headers(mcp_server, raw_headers, user_api_key_auth), + forwarded_headers=openapi_forwarded_headers, ) async def _call_openapi_via_handler(): @@ -5847,10 +5896,7 @@ class MCPServerManager: if matched is not None: matched_prefix, original_tool_name = matched matched_server: Final = prefix_to_server.get(matched_prefix) - if matched_server is not None and ( - original_tool_name in self.tool_name_to_mcp_server_name_mapping - or tool_name in self.tool_name_to_mcp_server_name_mapping - ): + if matched_server is not None and self._server_exposes_tool(matched_server, original_tool_name): return matched_server return None diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 4184fad009c..8d69d84e492 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -430,6 +430,7 @@ if MCP_AVAILABLE: MCPServerManager, _caller_authorization_fans_out, _client_forwarded_authorization_headers, + _resolve_openapi_tool_auth, _should_strip_caller_authorization, _without_authorization, global_mcp_server_manager, @@ -1704,7 +1705,7 @@ if MCP_AVAILABLE: ) extra_headers: dict[str, str] | None = None - is_client_forwarded_mode: Final = server.is_true_passthrough or server.is_oauth_delegate + is_client_forwarded_mode: Final = server.is_client_forwarded_token # In a multi-server listing scope the request-wide Authorization can only carry one token, # so it is withheld from a client-forwarded server when another server in scope also consumes # it (RFC 9700 cross-resource replay); such scopes must bind per-server via @@ -2013,6 +2014,9 @@ if MCP_AVAILABLE: prefetched_creds=_prefetched_oauth_creds, ) + if server.is_byok and server.auth_type != MCPAuth.oauth2 and server_auth_header is None: + server_auth_header = await _get_byok_credential(server, user_api_key_auth) + try: tools: Final = await global_mcp_server_manager._get_tools_from_server( server=server, @@ -2832,58 +2836,26 @@ if MCP_AVAILABLE: arguments = hook_result["arguments"] verbose_logger.debug("Executing local registry tool: %s", name) - # For BYOK servers the credential must be injected via a ContextVar - # because the tool function has headers baked into its closure. - # Pre-format the full Authorization header value using the server's - # configured auth_type so the generator doesn't need to know the prefix. - auth_header_value: str | None = None - if mcp_auth_header: - server_auth_type: Final = getattr(mcp_server, "auth_type", None) if mcp_server else None - if server_auth_type == MCPAuth.api_key: - auth_header_value = f"ApiKey {mcp_auth_header}" - elif server_auth_type == MCPAuth.basic: - auth_header_value = f"Basic {mcp_auth_header}" - else: - auth_header_value = f"Bearer {mcp_auth_header}" - - # Forward named client headers to OpenAPI tool upstream requests. - # MCPServer.extra_headers lists header names to copy from raw_headers. - # The strip decision is centralized in _should_strip_caller_authorization so this - # OpenAPI/local path agrees with the managed paths: M2M and the resolver-owned modes - # (token_exchange's raw subject token, authorization_code's stored token) must never - # have the caller's Authorization forwarded verbatim upstream. - forwarded_headers: dict[str, str] | None = None - if mcp_server and mcp_server.extra_headers and raw_headers: - normalized_raw: Final = {str(k).lower(): v for k, v in raw_headers.items() if isinstance(k, str)} - skip_caller_authorization: Final = _should_strip_caller_authorization( - mcp_server=mcp_server, - raw_headers=raw_headers, - user_api_key_auth=user_api_key_auth, - ) - for header_name in mcp_server.extra_headers: - if not isinstance(header_name, str): - continue - if skip_caller_authorization and header_name.lower() == "authorization": - continue - value = normalized_raw.get(header_name.lower()) - if value is not None: - if forwarded_headers is None: - forwarded_headers = {} - forwarded_headers[header_name] = value - - resolved_auth_headers: dict[str, str] | None = None - if mcp_server: - ( - resolved_auth_headers, - forwarded_headers, - ) = await global_mcp_server_manager.resolve_openapi_upstream_auth( - mcp_server=mcp_server, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - mcp_auth_header=mcp_auth_header, - user_api_key_auth=user_api_key_auth, - forwarded_headers=forwarded_headers, - ) + # The credential rides ContextVars because the tool function has its + # headers baked into the closure at registration time. + auth_header_value, openapi_forwarded_headers, upstream_credential = _resolve_openapi_tool_auth( + mcp_server=mcp_server, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + raw_headers=raw_headers, + user_api_key_auth=user_api_key_auth, + ) + ( + resolved_auth_headers, + forwarded_headers, + ) = await global_mcp_server_manager.resolve_openapi_upstream_auth( + mcp_server=mcp_server, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + mcp_auth_header=upstream_credential, + user_api_key_auth=user_api_key_auth, + forwarded_headers=openapi_forwarded_headers, + ) _auth_token: Final = _request_auth_header.set(auth_header_value) _extra_token: Final = _request_extra_headers.set(forwarded_headers) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index f128940c315..b0891b73fb8 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3059,6 +3059,8 @@ class NewProjectRequest(LiteLLM_BudgetTable): models: list[str] = [] model_rpm_limit: dict | None = None model_tpm_limit: dict | None = None + model_itpm_limit: Mapping[str, int] | None = None + model_otpm_limit: Mapping[str, int] | None = None blocked: bool = False object_permission: LiteLLM_ObjectPermissionBase | None = None @@ -3091,6 +3093,8 @@ class UpdateProjectRequest(LiteLLM_BudgetTable): models: list[str] | None = None model_rpm_limit: dict | None = None model_tpm_limit: dict | None = None + model_itpm_limit: Mapping[str, int] | None = None + model_otpm_limit: Mapping[str, int] | None = None blocked: bool | None = None budget_id: str | None = None object_permission: LiteLLM_ObjectPermissionBase | None = None @@ -4244,6 +4248,8 @@ class PassThroughEndpointLoggingTypedDict(TypedDict): LiteLLM_ManagementEndpoint_MetadataFields: Final = [ "model_rpm_limit", "model_tpm_limit", + "model_itpm_limit", + "model_otpm_limit", "default_estimated_output_tokens", "default_estimated_output_tokens_per_model", "mcp_rpm_limit", diff --git a/litellm/proxy/agent_endpoints/a2a_endpoints.py b/litellm/proxy/agent_endpoints/a2a_endpoints.py index 1cae87aed31..cea30ffad52 100644 --- a/litellm/proxy/agent_endpoints/a2a_endpoints.py +++ b/litellm/proxy/agent_endpoints/a2a_endpoints.py @@ -168,14 +168,13 @@ def _jsonrpc_error( ) -def _get_agent(agent_id: str): +async def _get_agent(agent_id: str) -> "AgentResponse | None": """Look up an agent by ID or name. Returns None if not found.""" - from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry + from litellm.proxy.common_utils.registry_read_through import ( + get_agent_with_read_through, + ) - agent = global_agent_registry.get_agent_by_id(agent_id=agent_id) - if agent is None: - agent = global_agent_registry.get_agent_by_name(agent_name=agent_id) - return agent + return await get_agent_with_read_through(agent_id) def _enforce_inbound_trace_id(agent: "AgentResponse", request: Request) -> None: @@ -559,7 +558,7 @@ async def get_agent_card( ) try: - agent: Final = _get_agent(agent_id) + agent: Final = await _get_agent(agent_id) if agent is None: raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found") @@ -673,7 +672,7 @@ async def invoke_agent_a2a( params.pop(key) # Find the agent - agent: Final = _get_agent(agent_id) + agent: Final = await _get_agent(agent_id) if agent is None: return _jsonrpc_error(request_id, -32000, f"Agent '{agent_id}' not found", 404) diff --git a/litellm/proxy/agent_endpoints/a2a_routing.py b/litellm/proxy/agent_endpoints/a2a_routing.py index 038b6b4a840..8a795214750 100644 --- a/litellm/proxy/agent_endpoints/a2a_routing.py +++ b/litellm/proxy/agent_endpoints/a2a_routing.py @@ -25,10 +25,12 @@ async def route_a2a_agent_request( Returns None if not an A2A request (allows normal routing to continue). """ # Import here to avoid circular imports - from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry from litellm.proxy.agent_endpoints.auth.agent_permission_handler import ( AgentRequestHandler, ) + from litellm.proxy.common_utils.registry_read_through import ( + get_agent_with_read_through, + ) from litellm.proxy.route_llm_request import ( ROUTE_ENDPOINT_MAPPING, ProxyModelNotFoundError, @@ -44,11 +46,11 @@ async def route_a2a_agent_request( agent_name: Final = model_name[4:] # Look up agent in registry - agent: Final = global_agent_registry.get_agent_by_name(agent_name) + agent: Final = await get_agent_with_read_through(agent_name) if agent is None: verbose_proxy_logger.error("[A2A] Agent '%s' not found in registry", agent_name) route_name = ROUTE_ENDPOINT_MAPPING.get(route_type, route_type) - raise ProxyModelNotFoundError(route=route_name, model_name=model_name) + raise ProxyModelNotFoundError(route=route_name, model_name=model_name, retryable_with_model_read_through=False) # Verify the caller is permitted to use this agent (admins bypass the check) is_admin: Final = user_api_key_dict is not None and ( @@ -70,7 +72,7 @@ async def route_a2a_agent_request( if not agent.agent_card_params or "url" not in agent.agent_card_params: verbose_proxy_logger.error("[A2A] Agent '%s' has no URL configured", agent_name) route_name = ROUTE_ENDPOINT_MAPPING.get(route_type, route_type) - raise ProxyModelNotFoundError(route=route_name, model_name=model_name) + raise ProxyModelNotFoundError(route=route_name, model_name=model_name, retryable_with_model_read_through=False) # Inject API base and route to litellm data["api_base"] = agent.agent_card_params["url"] diff --git a/litellm/proxy/agent_endpoints/agent_registry.py b/litellm/proxy/agent_endpoints/agent_registry.py index 742fdf35b1e..64de6827679 100644 --- a/litellm/proxy/agent_endpoints/agent_registry.py +++ b/litellm/proxy/agent_endpoints/agent_registry.py @@ -600,3 +600,4 @@ class AgentRegistry: global_agent_registry: Final = AgentRegistry() +AGENT_RECONCILE_LOCK: Final = asyncio.Lock() diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index a48ef0f08bb..f742965ade2 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -179,7 +179,7 @@ async def anthropic_response( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, - request_data=data, + request_data=base_llm_response_processor.data, ) body: Final = AnthropicExceptionMapping.transform_to_anthropic_error( status_code=e.status_code, @@ -189,7 +189,7 @@ async def anthropic_response( return JSONResponse(status_code=e.status_code, content=body) except Exception as e: await proxy_logging_obj.post_call_failure_hook( - user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data + user_api_key_dict=user_api_key_dict, original_exception=e, request_data=base_llm_response_processor.data ) verbose_proxy_logger.exception("litellm.proxy.proxy_server.anthropic_response(): Exception occured - %s", e) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index c9f9c00f120..a105bf19458 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -1172,7 +1172,7 @@ def enforce_output_token_estimates_are_admin_only( def get_model_rate_limit_from_metadata( user_api_key_dict: UserAPIKeyAuth, metadata_accessor_key: Literal["team_metadata", "organization_metadata", "project_metadata"], - rate_limit_key: Literal["model_rpm_limit", "model_tpm_limit"], + rate_limit_key: Literal["model_rpm_limit", "model_tpm_limit", "model_itpm_limit", "model_otpm_limit"], ) -> dict[str, int] | None: if getattr(user_api_key_dict, metadata_accessor_key): return getattr(user_api_key_dict, metadata_accessor_key).get(rate_limit_key) diff --git a/litellm/proxy/common_utils/registry_read_through.py b/litellm/proxy/common_utils/registry_read_through.py new file mode 100644 index 00000000000..460b348e188 --- /dev/null +++ b/litellm/proxy/common_utils/registry_read_through.py @@ -0,0 +1,224 @@ +"""Read-through recovery for in-memory registries in multi-replica deployments. + +A management write (POST /model/new, /guardrails, /v1/agents) lands on one +replica and reaches Postgres, but sibling replicas only refresh their in-memory +registries on the periodic config reload, so a request using the new object +immediately can land on a sibling that has never heard of it and fail 400/404. +On a registry miss, callers here fetch the missing row from the DB and load it +into the local registry before giving up. A short negative-result TTL per key +plus a global resync budget per window bound the DB load from lookups of +genuinely unknown names. +""" + +import asyncio +import time +from collections.abc import Awaitable, Callable +from typing import TYPE_CHECKING, Final + +from litellm._logging import verbose_proxy_logger +from litellm.caching.in_memory_cache import InMemoryCache + +if TYPE_CHECKING: + from prisma.types import ( + LiteLLM_AgentsTableInclude, + LiteLLM_AgentsTableWhereUniqueInput, + LiteLLM_GuardrailsTableWhereInput, + LiteLLM_ProxyModelTableWhereInput, + ) + + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.types.agents import AgentResponse + +READ_THROUGH_MISS_TTL_SECONDS: Final = 2.0 +READ_THROUGH_RESYNC_WINDOW_SECONDS: Final = 5.0 +READ_THROUGH_MAX_RESYNCS_PER_WINDOW: Final = 20 + + +class RegistryReadThrough: + __slots__ = ( + "_lock", + "_max_resyncs_per_window", + "_miss_ttl_seconds", + "_recent_misses", + "_resync", + "_resync_window_seconds", + "_window_resyncs", + "_window_started_at", + ) + + def __init__( + self, + resync: Callable[[str], Awaitable[bool]], + miss_ttl_seconds: float = READ_THROUGH_MISS_TTL_SECONDS, + max_resyncs_per_window: int = READ_THROUGH_MAX_RESYNCS_PER_WINDOW, + resync_window_seconds: float = READ_THROUGH_RESYNC_WINDOW_SECONDS, + ) -> None: + self._resync = resync + self._miss_ttl_seconds = miss_ttl_seconds + self._max_resyncs_per_window = max_resyncs_per_window + self._resync_window_seconds = resync_window_seconds + self._lock = asyncio.Lock() + self._recent_misses = InMemoryCache(max_size_in_memory=1000) + self._window_started_at = float("-inf") + self._window_resyncs = 0 + + def _consume_resync_budget(self) -> bool: + now: Final = time.monotonic() + if now - self._window_started_at >= self._resync_window_seconds: + self._window_started_at = now + self._window_resyncs = 0 + if self._window_resyncs >= self._max_resyncs_per_window: + return False + self._window_resyncs += 1 + return True + + async def attempt(self, key: str) -> bool: + if self._recent_misses.get_cache(key) is not None: + return False + async with self._lock: + if self._recent_misses.get_cache(key) is not None: + return False + if not self._consume_resync_budget(): + verbose_proxy_logger.warning( + "registry read-through for %r skipped: resync budget of %s per %ss exhausted", + key, + self._max_resyncs_per_window, + self._resync_window_seconds, + ) + return False + try: + found: Final = await self._resync(key) + except Exception as e: # noqa: BLE001 # a failed read-through must surface the original miss error, not a 500 + verbose_proxy_logger.warning("registry read-through for %r failed: %s", key, e) + return False + if not found: + self._recent_misses.set_cache(key, True, ttl=self._miss_ttl_seconds) + return found + + +def _db_backed_registries_enabled(object_type: str) -> bool: + from litellm.proxy import proxy_server + + if proxy_server.prisma_client is None or proxy_server.store_model_in_db is not True: + return False + return proxy_server.should_load_db_object(object_type=object_type) + + +async def _resync_model_deployments(model_name: str) -> bool: + from litellm.proxy import proxy_server + from litellm.repositories.model_repository import ModelRepository + + if not _db_backed_registries_enabled("models"): + return False + prisma_client: Final = proxy_server.prisma_client + assert prisma_client is not None + table: Final = ModelRepository(prisma_client).table + name_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_name": model_name} + id_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_id": model_name} + rows: Final = await table.find_many(where=name_filter) or await table.find_many(where=id_filter) + if not rows: + return False + router: Final = proxy_server.llm_router + if router is None: + await proxy_server.proxy_config.add_deployment( + prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj + ) + return proxy_server.llm_router is not None + async with proxy_server.MODEL_RECONCILE_LOCK: + proxy_server.proxy_config._add_deployment(db_models=rows) + proxy_server.llm_model_list = router.get_model_list() + return True + + +async def _resync_guardrails(guardrail_name: str) -> bool: + from litellm.proxy import proxy_server + from litellm.proxy.guardrails.guardrail_registry import ( + GUARDRAIL_RECONCILE_LOCK, + IN_MEMORY_GUARDRAIL_HANDLER, + ) + from litellm.repositories.table_repositories import GuardrailsRepository + from litellm.types.guardrails import Guardrail + + if not _db_backed_registries_enabled("guardrails"): + return False + prisma_client: Final = proxy_server.prisma_client + assert prisma_client is not None + active_row_filter: Final[LiteLLM_GuardrailsTableWhereInput] = { + "guardrail_name": guardrail_name, + "status": "active", + } + row: Final = await GuardrailsRepository(prisma_client).table.find_first(where=active_row_filter) + if row is None: + return False + async with GUARDRAIL_RECONCILE_LOCK: + IN_MEMORY_GUARDRAIL_HANDLER.sync_guardrail_from_db(guardrail=Guardrail(**dict(row))) + return _initialized_guardrail(guardrail_name) is not None + + +async def _resync_agents(agent_id_or_name: str) -> bool: + from litellm.proxy import proxy_server + from litellm.proxy.agent_endpoints.agent_registry import ( + AGENT_RECONCILE_LOCK, + agents_table, + global_agent_registry, + ) + from litellm.types.agents import AgentResponse + + if not _db_backed_registries_enabled("agents"): + return False + if _agent_from_registry(agent_id_or_name) is not None: + return True + prisma_client: Final = proxy_server.prisma_client + assert prisma_client is not None + table: Final = agents_table(prisma_client) + id_filter: Final[LiteLLM_AgentsTableWhereUniqueInput] = {"agent_id": agent_id_or_name} + name_filter: Final[LiteLLM_AgentsTableWhereUniqueInput] = {"agent_name": agent_id_or_name} + include_permission: Final[LiteLLM_AgentsTableInclude] = {"object_permission": True} + async with AGENT_RECONCILE_LOCK: + if _agent_from_registry(agent_id_or_name) is not None: + return True + row: Final = await table.find_unique(where=id_filter, include=include_permission) or await table.find_unique( + where=name_filter, include=include_permission + ) + if row is None: + return False + global_agent_registry.register_agent(agent_config=AgentResponse.model_validate(row.model_dump())) + return True + + +model_registry_read_through: Final = RegistryReadThrough(resync=_resync_model_deployments) +guardrail_registry_read_through: Final = RegistryReadThrough(resync=_resync_guardrails) +agent_registry_read_through: Final = RegistryReadThrough(resync=_resync_agents) + + +def _agent_from_registry(agent_id_or_name: str) -> "AgentResponse | None": + from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry + + by_id: Final = global_agent_registry.get_agent_by_id(agent_id=agent_id_or_name) + if by_id is not None: + return by_id + return global_agent_registry.get_agent_by_name(agent_name=agent_id_or_name) + + +async def get_agent_with_read_through(agent_id_or_name: str) -> "AgentResponse | None": + agent: Final = _agent_from_registry(agent_id_or_name) + if agent is not None: + return agent + if not await agent_registry_read_through.attempt(agent_id_or_name): + return None + return _agent_from_registry(agent_id_or_name) + + +def _initialized_guardrail(guardrail_name: str) -> "CustomGuardrail | None": + from litellm.proxy.guardrails import guardrail_endpoints + + return guardrail_endpoints.GUARDRAIL_REGISTRY.get_initialized_guardrail_callback(guardrail_name=guardrail_name) + + +async def get_initialized_guardrail_with_read_through(guardrail_name: str) -> "CustomGuardrail | None": + active: Final = _initialized_guardrail(guardrail_name) + if active is not None: + return active + if not await guardrail_registry_read_through.attempt(guardrail_name): + return None + return _initialized_guardrail(guardrail_name) diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index b68d4a68b79..e50a3a5a1e7 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -2305,8 +2305,12 @@ async def apply_guardrail( litellm_logging_obj = None start_time: Final = datetime.now(timezone.utc) + from litellm.proxy.common_utils.registry_read_through import ( + get_initialized_guardrail_with_read_through, + ) + try: - active_guardrail: Final[CustomGuardrail | None] = GUARDRAIL_REGISTRY.get_initialized_guardrail_callback( + active_guardrail: Final[CustomGuardrail | None] = await get_initialized_guardrail_with_read_through( guardrail_name=request.guardrail_name ) if active_guardrail is None: diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index 5f7374581a2..f6d348c1045 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -1,5 +1,6 @@ # litellm/proxy/guardrails/guardrail_registry.py +import asyncio import importlib import os from collections.abc import Callable, Iterator, Mapping @@ -813,4 +814,6 @@ class InMemoryGuardrailHandler: # In Memory Guardrail Handler for LiteLLM Proxy ######################################################## IN_MEMORY_GUARDRAIL_HANDLER: Final = InMemoryGuardrailHandler() + +GUARDRAIL_RECONCILE_LOCK: Final = asyncio.Lock() ######################################################## diff --git a/litellm/proxy/hooks/batch_rate_limiter.py b/litellm/proxy/hooks/batch_rate_limiter.py index 7e33583fc9d..d6229fb80a6 100644 --- a/litellm/proxy/hooks/batch_rate_limiter.py +++ b/litellm/proxy/hooks/batch_rate_limiter.py @@ -18,11 +18,12 @@ Quick summary: """ import json -from collections.abc import Iterable -from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn +from collections.abc import Iterable, Mapping, Sequence +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, TypeAlias from fastapi import HTTPException -from pydantic import BaseModel +from pydantic import BaseModel, Field, TypeAdapter import litellm from litellm._logging import verbose_proxy_logger @@ -40,10 +41,15 @@ from litellm.proxy._types import ( SpecialModelNames, UserAPIKeyAuth, ) +from litellm.proxy.auth.auth_utils import get_model_rate_limit_from_metadata from litellm.proxy.common_utils.proxy_rate_limit_error import ( ProxyRateLimitError, map_v3_rate_limit_type, ) +from litellm.proxy.hooks.parallel_request_limiter_v3 import ( + PROJECT_ITPM_DESCRIPTOR_KEY, + PROJECT_OTPM_DESCRIPTOR_KEY, +) from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit if TYPE_CHECKING: @@ -76,6 +82,11 @@ else: RateLimitDescriptor = dict[str, Any] +_BATCH_BODY_ADAPTER: Final = TypeAdapter(dict[str, object]) + +IncrementAmounts: TypeAlias = dict[Literal["requests", "tokens"], int] + + class BatchFileUsage(BaseModel): """ Internal model for batch file usage tracking, used for batch rate limiting @@ -83,6 +94,16 @@ class BatchFileUsage(BaseModel): total_tokens: int request_count: int + output_tokens: int = 0 + # Keyed by each row's own `body.model`, distinct from `total_tokens`/ + # `output_tokens` (the whole-file totals charged to the file-bound/ + # top-level routing model's key/team/model limits). A batch's rows can + # each target a different model, so the project's per-model ITPM/OTPM + # quota for a row's actual model must be charged with that row's own + # tokens -- see `_create_project_io_descriptors_for_models`. + per_model_usage: dict[str, dict[str, int]] = Field( + default_factory=dict + ) # mutable-ok: accumulated incrementally per row while parsing the batch file class _PROXY_BatchRateLimiter(CustomLogger): @@ -198,6 +219,15 @@ class _PROXY_BatchRateLimiter(CustomLogger): user_api_key_dict: UserAPIKeyAuth, data: dict, ) -> list["RateLimitDescriptor"]: + """Build the standard key/user/team/model descriptor list a batch is charged against. + + Deliberately excludes the project-scoped ITPM/OTPM descriptors: those + are charged per the JSONL row's own `body.model` once the file is + parsed (`_create_project_io_descriptors_for_models`), not the + file-bound/top-level routing model this function resolves. Charging + project quotas here would let a caller bind the file to a model + without a quota while rows execute against a quota-limited model. + """ return self.parallel_request_limiter._create_rate_limit_descriptors( user_api_key_dict=user_api_key_dict, data=data, @@ -206,6 +236,57 @@ class _PROXY_BatchRateLimiter(CustomLogger): model_has_failures=False, ) + @staticmethod + def _project_has_any_io_token_limits(user_api_key_dict: UserAPIKeyAuth) -> bool: + """True when the project has any per-model ITPM/OTPM quota configured. + + Used to stop the "skip batch input file processing" fast path from + bypassing a project quota configured for a model other than the + batch's file-bound/top-level routing model: the row models that + actually drive execution and billing aren't known until the JSONL + is parsed, so the file must be read whenever *any* model could be + quota-limited, not only when the routing model itself is. + """ + if user_api_key_dict.project_id is None: + return False + return bool( + get_model_rate_limit_from_metadata(user_api_key_dict, "project_metadata", "model_itpm_limit") + ) or bool(get_model_rate_limit_from_metadata(user_api_key_dict, "project_metadata", "model_otpm_limit")) + + def _create_project_io_descriptors_for_models( + self, + user_api_key_dict: UserAPIKeyAuth, + per_model_usage: Mapping[str, Mapping[str, int]], + ) -> tuple[list["RateLimitDescriptor"], list[IncrementAmounts]]: # mutable-ok: see below + """Build project ITPM/OTPM descriptors charged against each row's own model. + + One descriptor pair per distinct `body.model` found in the JSONL, + each incremented only by that model's own counted usage -- never the + whole-batch total -- so a quota-limited model can't hide behind an + unlimited routing model, and an unrelated model's rows can't inflate + a different model's counter. + """ + extra_descriptors: Final[list[RateLimitDescriptor]] = [] # mutable-ok: see above + extra_increments: Final[list[IncrementAmounts]] = [] # mutable-ok: see above + for model, usage in per_model_usage.items(): + model_descriptors: list[RateLimitDescriptor] = [] # mutable-ok: reset per loop iteration, not module state + self.parallel_request_limiter.add_project_io_token_rate_limit_descriptors_from_metadata( + user_api_key_dict=user_api_key_dict, + requested_model=model, + descriptors=model_descriptors, + ) + for descriptor in model_descriptors: + extra_descriptors.append(descriptor) + extra_increments.append( + { # mutable-ok: atomic limiter API requires mutable increment records + "requests": 0, + "tokens": usage.get("output_tokens", 0) + if descriptor["key"] == PROJECT_OTPM_DESCRIPTOR_KEY + else usage.get("total_tokens", 0), + } + ) + return extra_descriptors, extra_increments + def _should_skip_batch_input_file_processing( self, data: dict, @@ -232,6 +313,11 @@ class _PROXY_BatchRateLimiter(CustomLogger): routing deployment's trusted credentials and the batch is constrained to run on that provider. + The no-limits check also treats any project-configured ITPM/OTPM + quota as an applicable limit, even when it isn't scoped to the + routing model: a row can target a different, quota-limited model, + and that isn't knowable without parsing the JSONL. + Returns ``(should_skip, descriptors)`` where ``descriptors`` is the rate-limit descriptor list computed for the no-limits check, so the caller can reuse it for counter enforcement without recomputing. @@ -257,7 +343,9 @@ class _PROXY_BatchRateLimiter(CustomLogger): user_api_key_dict=user_api_key_dict, data=data, ) - if not self._has_applicable_batch_rate_limits(descriptors): + if not self._has_applicable_batch_rate_limits(descriptors) and not self._project_has_any_io_token_limits( + user_api_key_dict + ): verbose_proxy_logger.debug("Skipping batch input file processing: no rate limits configured") return True, None @@ -297,6 +385,58 @@ class _PROXY_BatchRateLimiter(CustomLogger): return False return True + def _estimate_entry_output_tokens( + self, + entry: Mapping[str, object], + min_configured_otpm_limit: int | None, + ) -> int: + """Conservative per-row output-token estimate for the project OTPM reservation. + + Batch completion never reconciles actual usage back into the rate + limiter, so this pre-call estimate is the only OTPM enforcement a + batch gets. Mirrors the real-time no-``max_tokens`` floor so a row + that omits an output cap can't be used to bypass OTPM the way an + unbounded streaming request could. + + Embeddings rows are identified by the row's own ``url`` (the OpenAI + batch schema puts the target route there, e.g. ``/v1/embeddings``), + never by body shape: a `/v1/responses` row also carries `body.input` + with no `messages`/`prompt`, so guessing from body shape alone would + misclassify a token-generating Responses row as a zero-output + embeddings row and let it skip the OTPM reservation entirely. + """ + url: Final = entry.get("url") + if isinstance(url, str) and "embeddings" in url: + return 0 # embeddings: no output tokens + raw_body: Final = entry.get("body") + body: Final[Mapping[str, object]] = ( + MappingProxyType(_BATCH_BODY_ADAPTER.validate_python(raw_body)) + if isinstance(raw_body, Mapping) + else MappingProxyType({}) # mutable-ok: immediately frozen empty fallback + ) + # `max_tokens`/`max_completion_tokens` cap chat completions; `/v1/responses` + # rows cap output with `max_output_tokens` instead -- omitting it here + # would fall through to the floor estimate for every capped Responses row. + explicit_cap: Final = next( + ( + v + for v in ( + body.get("max_tokens"), + body.get("max_completion_tokens"), + body.get("max_output_tokens"), + ) + if v is not None + ), + None, + ) + candidate_count: Final = self.parallel_request_limiter.get_output_candidate_count(body) + if explicit_cap is not None: + try: + return max(0, int(explicit_cap)) * candidate_count + except (TypeError, ValueError, OverflowError): + pass + return self.parallel_request_limiter.no_max_tokens_output_floor(min_configured_otpm_limit) * candidate_count + @staticmethod def _has_applicable_batch_rate_limits( descriptors: list["RateLimitDescriptor"], @@ -382,9 +522,22 @@ class _PROXY_BatchRateLimiter(CustomLogger): """Raise :class:`ProxyRateLimitError` (a 429) for batch rate limit exceeded.""" from datetime import datetime - # Find the descriptor for this status + # Find the descriptor for this status. Matching on (key, value) is + # required, not key alone: a batch can carry several project ITPM/OTPM + # descriptors sharing one key (e.g. `model_per_project_otpm`) but + # scoped to different models via `value` + # ("{project_id}:{model}") -- key-only matching would always resolve + # to the first same-keyed descriptor regardless of which one was + # actually over its limit. Falls back to key-only matching for + # statuses that predate `descriptor_value` (e.g. from should_rate_limit). + status_descriptor_value: Final = status.get("descriptor_value") descriptor_index: Final = next( - (i for i, d in enumerate(descriptors) if d.get("key") == status.get("descriptor_key")), + ( + i + for i, d in enumerate(descriptors) + if d.get("key") == status.get("descriptor_key") + and (status_descriptor_value is None or d.get("value") == status_descriptor_value) + ), 0, ) descriptor: Final[RateLimitDescriptor] = ( @@ -407,9 +560,27 @@ class _PROXY_BatchRateLimiter(CustomLogger): f"Limit resets at: {reset_time_formatted}" ) else: # tokens + # Project ITPM/OTPM descriptors are keyed "{project_id}:{model}" and + # charged with that model's own rows (see + # `_create_project_io_descriptors_for_models`), not the whole + # batch's totals -- report the matching per-model figure when one + # is available so the error reflects what was actually charged. + descriptor_model: Final = ( + descriptor.get("value", "").split(":", 1)[-1] + if descriptor.get("key") in (PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY) + else None + ) + model_usage: Final = batch_usage.per_model_usage.get(descriptor_model) if descriptor_model else None + batch_token_count: Final = ( + (model_usage or {}).get("output_tokens", batch_usage.output_tokens) + if descriptor.get("key") == PROJECT_OTPM_DESCRIPTOR_KEY + else (model_usage or {}).get("total_tokens", batch_usage.total_tokens) + if descriptor.get("key") == PROJECT_ITPM_DESCRIPTOR_KEY + else batch_usage.total_tokens + ) detail = ( f"Batch rate limit exceeded for {descriptor.get('key', 'unknown')}: {descriptor.get('value', 'unknown')}. " - f"Batch contains {batch_usage.total_tokens} tokens but only {remaining_display} tokens remaining " + f"Batch contains {batch_token_count} tokens but only {remaining_display} tokens remaining " f"out of {current_limit} TPM limit. " f"Limit resets at: {reset_time_formatted}" ) @@ -444,7 +615,10 @@ class _PROXY_BatchRateLimiter(CustomLogger): falls back to a per-process asyncio.Lock + in-memory operation. ``descriptors`` may be passed in by the pre-call hook to reuse the list - already computed when deciding whether to skip file processing. + already computed when deciding whether to skip file processing. It + never contains project ITPM/OTPM descriptors (those are model-specific + and only knowable once ``batch_usage.per_model_usage`` is populated by + parsing the JSONL), so this always builds and appends them here. """ if descriptors is None: descriptors = self._create_batch_rate_limit_descriptors( @@ -452,11 +626,20 @@ class _PROXY_BatchRateLimiter(CustomLogger): data=data, ) - increment: Final[dict[Literal["requests", "tokens"], int]] = { - "requests": batch_usage.request_count, - "tokens": batch_usage.total_tokens, - } - increments: Final[list[dict[Literal["requests", "tokens"], int]]] = [increment for _ in descriptors] + increments: list[IncrementAmounts] = [ # mutable-ok: reassigned below to append project IO increments + { # mutable-ok: atomic limiter API requires mutable increment records + "requests": batch_usage.request_count, + "tokens": batch_usage.total_tokens, + } + for _d in descriptors + ] + + project_io_descriptors, project_io_increments = self._create_project_io_descriptors_for_models( + user_api_key_dict=user_api_key_dict, + per_model_usage=batch_usage.per_model_usage, + ) + descriptors = [*descriptors, *project_io_descriptors] + increments = [*increments, *project_io_increments] rate_limit_response: Final = await self.parallel_request_limiter.atomic_check_and_increment_by_n( descriptors=descriptors, @@ -482,6 +665,7 @@ class _PROXY_BatchRateLimiter(CustomLogger): custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai", user_api_key_dict: UserAPIKeyAuth | None = None, data: dict | None = None, + descriptors: Sequence["RateLimitDescriptor"] | None = None, ) -> BatchFileUsage: """ Count number of requests and tokens in a batch input file. @@ -490,10 +674,37 @@ class _PROXY_BatchRateLimiter(CustomLogger): file_id: The file ID to read custom_llm_provider: The custom LLM provider to use for token encoding user_api_key_dict: User authentication information for file access (required for managed files) + descriptors: Rate limit descriptors already computed for this batch, so the + configured project OTPM limit can scale the no-``max_tokens`` output floor Returns: - BatchFileUsage with total_tokens and request_count + BatchFileUsage with total_tokens, output_tokens, request_count, and + per_model_usage (each row's own totals, keyed by its `body.model`) """ + descriptor_otpm_limits: Final = tuple( + int(v) + for d in (descriptors or ()) + if d.get("key") == PROJECT_OTPM_DESCRIPTOR_KEY + for rate_limit in (d.get("rate_limit"),) + for v in (rate_limit.get("tokens_per_unit") if rate_limit is not None else None,) + if v is not None + ) + # `descriptors` only ever carries the routing model's own OTPM limit + # (see `_create_batch_rate_limit_descriptors`), but a row can target + # any project-configured model. Folding in every configured model's + # OTPM limit keeps the no-`max_tokens` floor from drifting wide just + # because a row's specific model isn't known until parsed below. + project_otpm_limits: Final = ( + tuple(int(v) for v in project_otpm_limit_map.values()) + if user_api_key_dict is not None + and ( + project_otpm_limit_map := get_model_rate_limit_from_metadata( + user_api_key_dict, "project_metadata", "model_otpm_limit" + ) + ) + else () + ) + min_configured_otpm_limit: Final = min((*descriptor_otpm_limits, *project_otpm_limits), default=None) try: # Check if this is a managed file (base64 encoded unified file ID) from litellm.proxy.openai_files_endpoints.common_utils import ( @@ -545,23 +756,51 @@ class _PROXY_BatchRateLimiter(CustomLogger): # Counting stays best-effort, so a legitimate (e.g. multimodal) row # the counter can't measure is estimated, not hard-rejected. models: Final[set] = set() + # Keyed by each row's own `body.model`, so the project ITPM/OTPM + # quota for that model is charged with only its own rows' tokens, + # never the whole batch's -- see `_create_project_io_descriptors_for_models`. + per_model_usage: Final[dict[str, dict[str, int]]] = {} total_tokens = 0 + output_tokens = 0 # rebind-ok: accumulated per JSONL row in the loop below request_count = 0 for raw_line in _iter_batch_input_lines(file_content_bytes): request_count += 1 try: entry = json.loads(raw_line) except Exception: - total_tokens += _estimate_batch_entry_tokens(raw_line) + entry_total_tokens = _estimate_batch_entry_tokens(raw_line) + entry_output_tokens = self.parallel_request_limiter.no_max_tokens_output_floor( + min_configured_otpm_limit + ) + total_tokens += entry_total_tokens + output_tokens += entry_output_tokens continue + + model: str | None = (entry.get("body") or {}).get("model") if isinstance(entry, dict) else None + if model: + models.add(model) + if isinstance(entry, dict): - model = (entry.get("body") or {}).get("model") - if model: - models.add(model) + entry_output_tokens = self._estimate_entry_output_tokens(entry, min_configured_otpm_limit) + else: + entry_output_tokens = self.parallel_request_limiter.no_max_tokens_output_floor( + min_configured_otpm_limit + ) + output_tokens += entry_output_tokens + try: - total_tokens += _count_entry_tokens(entry) + entry_total_tokens = _count_entry_tokens(entry) except Exception: - total_tokens += _estimate_batch_entry_tokens(raw_line) + entry_total_tokens = _estimate_batch_entry_tokens(raw_line) + total_tokens += entry_total_tokens + + if model: + model_usage = per_model_usage.setdefault( + model, {"total_tokens": 0, "output_tokens": 0, "request_count": 0} + ) + model_usage["total_tokens"] += entry_total_tokens + model_usage["output_tokens"] += entry_output_tokens + model_usage["request_count"] += 1 # Validate every model named in the batch JSONL against the # caller's per-key model allowlist. Without this, a caller @@ -578,6 +817,8 @@ class _PROXY_BatchRateLimiter(CustomLogger): return BatchFileUsage( total_tokens=total_tokens, request_count=request_count, + output_tokens=output_tokens, + per_model_usage=per_model_usage, ) except HTTPException as e: @@ -814,6 +1055,7 @@ class _PROXY_BatchRateLimiter(CustomLogger): custom_llm_provider=custom_llm_provider, user_api_key_dict=user_api_key_dict, data=data, + descriptors=batch_rate_limit_descriptors, ) verbose_proxy_logger.debug( diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index 4492f42782c..de8834449de 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -454,7 +454,9 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): parent_otel_span=user_api_key_dict.parent_otel_span, ) - verbose_proxy_logger.debug("Atomic check+increment response: %s", json.dumps(atomic_response, indent=2)) + verbose_proxy_logger.debug( + "Atomic check+increment response: %s", json.dumps(atomic_response, indent=2, default=list) + ) if atomic_response["overall_code"] == "OVER_LIMIT": resolved_model, llm_provider = resolve_llm_provider_for_rate_limit(model) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 94ef08782d9..2d799ded752 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -8,7 +8,7 @@ import asyncio import binascii import os import uuid -from collections.abc import Callable, Sequence +from collections.abc import Callable, Mapping, Sequence, Set from contextvars import ContextVar from dataclasses import dataclass, field from datetime import datetime @@ -22,6 +22,8 @@ from typing import ( TypedDict, ) +from typing_extensions import NotRequired, ReadOnly + from litellm import DualCache from litellm._logging import verbose_proxy_logger from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, INTERNAL_CALL_ORIGIN_METADATA_KEY @@ -44,11 +46,12 @@ from litellm.proxy.common_utils.proxy_rate_limit_error import ( ) from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit from litellm.types.caching import RedisPipelineIncrementOperation -from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject +from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject, ResponseAPIUsage from litellm.types.utils import ( CallTypes, EmbeddingResponse, ModelResponse, + RerankResponse, TextCompletionResponse, Usage, ) @@ -66,6 +69,7 @@ else: Span = Any InternalUsageCache = Any + BATCH_RATE_LIMITER_SCRIPT: Final = """ local results = {} local now = tonumber(ARGV[1]) @@ -120,7 +124,8 @@ CHECK_AND_INCREMENT_BY_N_SCRIPT: Final = """ -- ARGV[(i-1)*4 + 3] = ttl_seconds (counter TTL when window resets) -- ARGV[(i-1)*4 + 4] = window_size_seconds (sliding-window length) -- --- Return on success: { 0, new_counter_1, new_counter_2, ... } +-- Return on success: +-- { 0, new_counter_1, window_start_1, new_counter_2, window_start_2, ... } -- Return on over-limit: { 1, descriptor_index, current_counter, limit } local time_reply = redis.call('TIME') local now = tonumber(time_reply[1]) @@ -157,7 +162,7 @@ for i = 1, descriptor_count do return { 1, i, current_counter, limit } end - descriptor_state[i] = { window_expired, current_counter } + descriptor_state[i] = { window_expired, current_counter, window_start } end -- Pass 2: all checks passed. Apply increments. @@ -171,8 +176,10 @@ for i = 1, descriptor_count do local window_size = tonumber(ARGV[arg_base + 3]) local window_expired = descriptor_state[i][1] + local active_window_start if window_expired then + active_window_start = now redis.call('SET', window_key, tostring(now)) redis.call('SET', counter_key, increment) redis.call('EXPIRE', window_key, window_size) @@ -181,6 +188,7 @@ for i = 1, descriptor_count do end table.insert(results, increment) else + active_window_start = tonumber(descriptor_state[i][3]) local new_counter = redis.call('INCRBY', counter_key, increment) local current_ttl = redis.call('TTL', counter_key) if current_ttl == -1 and ttl > 0 then @@ -188,11 +196,39 @@ for i = 1, descriptor_count do end table.insert(results, new_counter) end + table.insert(results, active_window_start) end return results """ +WINDOW_GUARDED_TOKEN_INCREMENT_SCRIPT: Final = """ +local results = {} +for i = 1, #KEYS, 2 do + local window_key = KEYS[i] + local counter_key = KEYS[i + 1] + local arg_base = ((i - 1) / 2) * 3 + 1 + local expected_window_start = ARGV[arg_base] + local increment = tonumber(ARGV[arg_base + 1]) + local ttl = tonumber(ARGV[arg_base + 2]) + local active_window_start = redis.call('GET', window_key) + + if active_window_start and active_window_start == expected_window_start then + local new_counter = redis.call('INCRBY', counter_key, increment) + local current_ttl = redis.call('TTL', counter_key) + if current_ttl == -1 and ttl > 0 then + redis.call('EXPIRE', counter_key, ttl) + end + table.insert(results, 1) + table.insert(results, new_counter) + else + table.insert(results, 0) + table.insert(results, tonumber(redis.call('GET', counter_key) or 0)) + end +end +return results +""" + PARALLEL_ACQUIRE_SCRIPT: Final = """ -- Atomic check-and-acquire for the max_parallel_requests concurrency gauge. -- Each gauge key is a sorted set of per-request slot ids scored by acquire @@ -297,6 +333,38 @@ DEFAULT_CHARS_PER_TOKEN: Final = 4 # (baseline floor) and to the smallest configured TPM limit (capped floor for # small per-tenant TPM caps). _TPM_FLOOR_FRACTION: Final = 4 +# Both embeddings and the Responses API put their prompt in data["input"], +# but only embeddings have no output tokens. Every "is this an embedding" +# check on data["input"] must exclude these call types, or a Responses call +# gets misclassified as an embedding and skips output-token reservation/caps. +RESPONSES_API_CALL_TYPES: Final = ("aresponses", "responses") +EMBEDDING_API_CALL_TYPES: Final = ("aembedding", "embedding") +TEXT_COMPLETION_API_CALL_TYPES: Final = ("atext_completion", "text_completion") +RERANK_API_CALL_TYPES: Final = (CallTypes.rerank.value, CallTypes.arerank.value) +GOOGLE_GENAI_NATIVE_CALL_TYPES: Final = ( + CallTypes.generate_content.value, + CallTypes.agenerate_content.value, + CallTypes.generate_content_stream.value, + CallTypes.agenerate_content_stream.value, +) +RESPONSES_API_MIN_OUTPUT_TOKENS: Final = 16 +# litellm.token_counter has no per-type handling for "input_audio" content +# blocks (unlike images, which use use_default_image_token_count) -- it +# silently contributes 0 tokens for them. When the block carries a base64 +# payload, the estimate is derived from the decoded byte count; when the +# block is a reference without a payload (or the payload is missing), this +# flat per-block floor is used instead. +DEFAULT_AUDIO_TOKEN_ESTIMATE: Final = 300 +# Conservative bytes-per-token assumption for size-based audio estimation: +# equivalent to 8 kHz mono PCM-16 (16 000 bytes/s) at 10 tokens/s. Choosing +# the lowest reasonable bitrate means we never under-reserve for higher- +# quality audio recorded at the same wall-clock duration. +_AUDIO_BYTES_PER_TOKEN: Final = 1600 +# Descriptor "key" values for project-scoped ITPM/OTPM. Distinct from +# "model_per_project" (the combined-TPM descriptor) so both can be enforced +# on the same project+model simultaneously without colliding on cache keys. +PROJECT_ITPM_DESCRIPTOR_KEY: Final = "model_per_project_itpm" +PROJECT_OTPM_DESCRIPTOR_KEY: Final = "model_per_project_otpm" # How long an acquired slot counts toward the in-flight total before it is # considered leaked (worker crashed without any release callback firing) and # pruned. Also the longest request duration the gauge can track: a request @@ -341,11 +409,24 @@ class RateLimitStatus(TypedDict): limit_remaining: int rate_limit_type: Literal["requests", "tokens", "max_parallel_requests"] descriptor_key: str + # Only populated by the atomic_check_and_increment_by_n path. A caller + # matching a status back to its descriptor must key on (descriptor_key, + # descriptor_value) when this is present, not descriptor_key alone -- + # e.g. a batch charging several models' project ITPM/OTPM in one call + # produces multiple statuses sharing the same descriptor_key. + descriptor_value: NotRequired[ReadOnly[str]] class RateLimitResponse(TypedDict): overall_code: str statuses: list[RateLimitStatus] + reservation_windows: NotRequired[ReadOnly[frozenset[tuple[str, str, Literal["redis", "local"]]]]] + + +class ReservationAwareIncrementOperation(RedisPipelineIncrementOperation): + window_key: NotRequired[str] + expected_window_start: NotRequired[str] + reservation_backend: NotRequired[Literal["redis", "local"]] class RateLimitResponseWithDescriptors(TypedDict): @@ -353,6 +434,10 @@ class RateLimitResponseWithDescriptors(TypedDict): response: RateLimitResponse +class _RateLimitDescriptorSink(Protocol): + def append(self, descriptor: RateLimitDescriptor, /) -> None: ... + + class WindowKeyMetadata(TypedDict): requests_limit: int | None tokens_limit: int | None @@ -362,6 +447,7 @@ class WindowKeyMetadata(TypedDict): class AtomicCounterMeta(TypedDict): descriptor_key: str + descriptor_value: ReadOnly[str] current_limit: int rate_limit_type: Literal["requests", "tokens"] window_key: str @@ -374,6 +460,7 @@ class AtomicCounterMeta(TypedDict): class AtomicCounterState(TypedDict): window_expired: bool current: int + window_start: ReadOnly[str] DescriptorAtomicGroup: TypeAlias = tuple[list[str], list[int], list[AtomicCounterMeta]] @@ -418,6 +505,16 @@ class RequestRateLimiterStash: reserved_tokens: int = 0 reserved_model: str | None = None reserved_scopes: frozenset[tuple[str, str]] = field(default_factory=frozenset) + itpm_reserved_tokens: int = 0 + itpm_reserved_scopes: frozenset[tuple[str, str]] = field(default_factory=frozenset) + itpm_reserved_window_identities: frozenset[tuple[str, str, Literal["redis", "local"]]] = field( + default_factory=frozenset + ) + otpm_reserved_tokens: int = 0 + otpm_reserved_scopes: frozenset[tuple[str, str]] = field(default_factory=frozenset) + otpm_reserved_window_identities: frozenset[tuple[str, str, Literal["redis", "local"]]] = field( + default_factory=frozenset + ) reservation_released: bool = False @@ -462,21 +559,13 @@ def _call_id_from_callback_kwargs(kwargs: object) -> str | None: return call_id if isinstance(call_id, str) else None -def _declared_output_budget(value: object) -> int | None: - """Coerce a declared output budget to tokens, or None when it names no budget. - - Accepts every shape the pre-existing ``int(...)`` coercion did, floats and numeric - strings included, because a budget this cannot read is a budget this cannot reserve - against, which is the bypass the caller-declared limits are checked for. - """ - if isinstance(value, (int, float)): - return int(value) - if isinstance(value, str): - try: - return int(float(value)) - except ValueError: - return None - return None +def _parse_output_cap_value(raw_value: object) -> int | None: + if isinstance(raw_value, bool) or not isinstance(raw_value, (int, float, str)): + return None + try: + return int(float(raw_value)) + except (ValueError, OverflowError): + return None class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): @@ -497,6 +586,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): self.check_and_increment_by_n_script = ( self.internal_usage_cache.dual_cache.redis_cache.async_register_script(CHECK_AND_INCREMENT_BY_N_SCRIPT) ) + self.window_guarded_token_increment_script = ( + self.internal_usage_cache.dual_cache.redis_cache.async_register_script( + WINDOW_GUARDED_TOKEN_INCREMENT_SCRIPT + ) + ) self.parallel_acquire_script = self.internal_usage_cache.dual_cache.redis_cache.async_register_script( PARALLEL_ACQUIRE_SCRIPT ) @@ -510,6 +604,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): self.batch_rate_limiter_script = None self.token_increment_script = None self.check_and_increment_by_n_script = None + self.window_guarded_token_increment_script = None self.parallel_acquire_script = None self.parallel_release_script = None self.parallel_count_script = None @@ -562,7 +657,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): return self._time_provider() @staticmethod - def _no_max_tokens_output_floor( + def no_max_tokens_output_floor( min_configured_tpm_limit: int | None, ) -> int: """Output-budget floor used when the request omits max_tokens. @@ -576,11 +671,164 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): return baseline return min(baseline, max(1, min_configured_tpm_limit // _TPM_FLOOR_FRACTION)) + @staticmethod + def _is_embedding_request(data: object, call_type: str | None) -> bool: + if call_type in EMBEDDING_API_CALL_TYPES: + return True + if call_type in RESPONSES_API_CALL_TYPES: + return False + if call_type: + return False + if not isinstance(data, dict): + return False + return data.get("input") is not None + + @staticmethod + def _translate_google_genai_native_request( + data: object, + call_type: str | None, + ) -> Mapping[str, object] | None: + contents: Final = data.get("contents") if isinstance(data, dict) else None + if ( + not isinstance(data, dict) + or call_type not in GOOGLE_GENAI_NATIVE_CALL_TYPES + or not isinstance(contents, (dict, list)) + ): + return None + from litellm.google_genai.adapters.transformation import GoogleGenAIAdapter + + config: Final = data.get("config") if "config" in data else data.get("generationConfig") + return GoogleGenAIAdapter().translate_generate_content_to_completion( + model=data.get("model") if isinstance(data.get("model"), str) else "", + contents=contents, + config=config if isinstance(config, dict) else None, + systemInstruction=data.get("systemInstruction"), + system_instruction=data.get("system_instruction"), + tools=data.get("tools"), + toolConfig=data.get("toolConfig"), + tool_config=data.get("tool_config"), + ) + + @staticmethod + def _get_explicit_output_cap(data: object, call_type: str | None) -> int | None: + if not isinstance(data, dict): + return None + if call_type in GOOGLE_GENAI_NATIVE_CALL_TYPES: + config: Final = data.get("config") if "config" in data else data.get("generationConfig") + google_cap_values: Final = tuple( + parsed + for field in ("maxOutputTokens", "max_output_tokens") + if isinstance(config, dict) + for parsed in (_parse_output_cap_value(config.get(field)),) + if parsed is not None + ) + return max(google_cap_values, default=None) + if call_type in RESPONSES_API_CALL_TYPES: + responses_cap: Final = _parse_output_cap_value(data.get("max_output_tokens")) + if responses_cap is None: + return None + return max(RESPONSES_API_MIN_OUTPUT_TOKENS, responses_cap) + if call_type in EMBEDDING_API_CALL_TYPES: + return None + fields: Final = ( + ("max_tokens", "max_completion_tokens") + if call_type + else ("max_tokens", "max_completion_tokens", "max_output_tokens") + ) + output_cap_values: Final = tuple( + parsed for field in fields for parsed in (_parse_output_cap_value(data.get(field)),) if parsed is not None + ) + return max(output_cap_values, default=None) + + @classmethod + def _has_explicit_output_cap(cls, data: object, call_type: str | None) -> bool: + """Whether the caller explicitly set an output-token cap. + + Checked via ``is not None`` (not truthiness) so an explicit 0 -- + a legitimate zero-output request -- counts as explicit. + """ + return cls._get_explicit_output_cap(data, call_type) is not None + + @staticmethod + def get_output_candidate_count(data: object, call_type: str | None = None) -> int: + if not isinstance(data, Mapping): + return 1 + config: Final = ( + (data.get("config") if "config" in data else data.get("generationConfig")) + if call_type in GOOGLE_GENAI_NATIVE_CALL_TYPES + else None + ) + candidate_values: Final = ( + data.get("n"), + data.get("best_of"), + config.get("candidateCount") if isinstance(config, dict) else None, + config.get("candidate_count") if isinstance(config, dict) else None, + ) + candidate_count = 1 # rebind-ok: running maximum across candidate-count aliases + for value in candidate_values: + try: + candidate_count = max(candidate_count, int(value or 1)) + except (TypeError, ValueError, OverflowError): + continue + return candidate_count + + @staticmethod + def _apply_implicit_output_cap( + data: object, + min_configured_limit: int | None, + call_type: str | None, + configured_output_tokens: int | None = None, + ) -> None: + """Hard-cap generation length when the request has no explicit cap. + + Guards against an unbounded response overshooting a small TPM/OTPM + budget before post-call reconciliation runs. Skips requests that + already set an explicit cap and embeddings, which have no generation + budget. The Responses API only honors ``max_output_tokens`` (its + underlying chat-completion transformation ignores ``max_tokens``), so + the cap must be written to that field for Responses call types. + + ``configured_output_tokens`` is the operator-declared per-tenant + estimate; when it exceeds the safety floor, the cap is raised to that + value instead of clamping every tenant to the same floor. + """ + if not isinstance(data, dict): + return + base_capped_floor: Final = _PROXY_MaxParallelRequestsHandler_v3.no_max_tokens_output_floor(min_configured_limit) + capped_floor: Final = ( + max(base_capped_floor, RESPONSES_API_MIN_OUTPUT_TOKENS) + if call_type in RESPONSES_API_CALL_TYPES + else base_capped_floor + ) + baseline_floor: Final = DEFAULT_MAX_TOKENS_ESTIMATE // _TPM_FLOOR_FRACTION + is_embedding: Final = _PROXY_MaxParallelRequestsHandler_v3._is_embedding_request(data, call_type) + if ( + capped_floor >= baseline_floor + or _PROXY_MaxParallelRequestsHandler_v3._has_explicit_output_cap(data, call_type) + or is_embedding + ): + return + effective_cap: Final = max(capped_floor, configured_output_tokens or 0) + if call_type in GOOGLE_GENAI_NATIVE_CALL_TYPES: + config_field: Final = "config" if "config" in data or "generationConfig" not in data else "generationConfig" + config: Final = data.get(config_field) + if config is None or isinstance(config, dict): + data[config_field] = { # rebind-ok: routed request needs cap # mutable-ok: downstream needs dict + **(config or {}), # mutable-ok: downstream native routing requires a mutable request config + "maxOutputTokens": effective_cap, + } + return + cap_field: Final = "max_output_tokens" if call_type in RESPONSES_API_CALL_TYPES else "max_tokens" + existing_cap: Final = data.get(cap_field) + if existing_cap is None or effective_cap < existing_cap: + data[cap_field] = effective_cap # rebind-ok: downstream routing requires the bounded output cap + def _estimate_tokens_for_request( self, data: dict, model: str | None = None, min_configured_tpm_limit: int | None = None, + call_type: str | None = None, configured_output_tokens: int | None = None, ) -> int: """ @@ -588,7 +836,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): upfront (input + output budget): estimated = input_tokens + max_tokens. - Supports chat (messages), completions (prompt), and embeddings (input). + Supports chat (messages), completions (prompt), embeddings (input), + and the Responses API (also `input`, disambiguated from embeddings + via ``call_type``). ``min_configured_tpm_limit`` is the smallest ``tokens_per_unit`` among the TPM-bearing descriptors this request will be charged against. When @@ -601,78 +851,108 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): floor entirely, so the reservation reflects what this tenant's model actually emits rather than one constant shared by every tenant. """ - messages = data.get("messages") - prompt: Final = data.get("prompt") - input_text: Final = data.get("input") # embeddings - - match (messages, prompt, input_text): - case (messages, _, _) if messages: - total_chars = len(get_str_from_messages(messages)) - case (_, str() as p, _): - total_chars = len(p) - case (_, list() as p, _): - total_chars = sum(len(str(item)) for item in p) - case (_, _, str() as t): - total_chars = len(t) - case (_, _, list() as t): - total_chars = sum(len(str(item)) for item in t) - case _: - total_chars = 0 - - estimated_input_tokens: Final = max(1, total_chars // DEFAULT_CHARS_PER_TOKEN) if total_chars > 0 else 0 - - # Both spellings can arrive together, e.g. a deployment-level max_tokens default under a - # client-supplied max_completion_tokens. Reserving against the larger keeps the estimate an - # upper bound on what the provider can emit, whichever one it ends up honouring. - declared_output_budgets: Final = tuple( - budget - for budget in ( - _declared_output_budget(data.get("max_tokens")), - _declared_output_budget(data.get("max_completion_tokens")), - ) - if budget is not None + estimated_input_tokens, max_tokens_estimate = self._estimate_input_and_output_tokens( + data=data, + min_configured_tpm_limit=min_configured_tpm_limit, + call_type=call_type, + configured_output_tokens=configured_output_tokens, ) - explicit_max_tokens: Final = max(declared_output_budgets) if declared_output_budgets else None - - match (explicit_max_tokens, input_text): - case (mt, _) if mt is not None: - max_tokens_estimate = int(mt) - case (_, embeddings_input) if embeddings_input: - # Embeddings have no output tokens - max_tokens_estimate = 0 - case _ if total_chars == 0 and configured_output_tokens is None: - # Fully contentless request (no messages, prompt, or input). - # Don't apply the conservative output-budget floor here — it - # would over-reserve and could push small TPM limits into a - # false 429. The caller floors at 1 so backpressure still - # applies once the counter is at limit. - max_tokens_estimate = 0 - case _: - # No max_tokens specified — reserve at least the input size with a - # conservative floor so a stream of small concurrent requests can't - # collectively bypass the limit. Cap the floor by a fraction of - # the smallest TPM limit this request will be charged against, - # so a small per-tenant TPM cap can't be tripped by the floor - # alone. - output_floor: Final = self._no_max_tokens_output_floor(min_configured_tpm_limit) - max_tokens_estimate = ( - configured_output_tokens - if configured_output_tokens is not None - else max(estimated_input_tokens, output_floor) - ) - total_estimated: Final = estimated_input_tokens + max_tokens_estimate verbose_proxy_logger.debug( - "TPM reservation estimate: input=%s, max_tokens=%s (explicit=%s), total=%s", + "TPM reservation estimate: input=%s, max_tokens=%s, total=%s", estimated_input_tokens, max_tokens_estimate, - explicit_max_tokens is not None, total_estimated, ) return total_estimated + def _estimate_input_and_output_tokens( + self, + data: object, + min_configured_tpm_limit: int | None = None, + call_type: str | None = None, + configured_output_tokens: int | None = None, + ) -> tuple[int, int]: + """ + Estimate input tokens and output (max_tokens) budget separately, so + callers needing independent ITPM/OTPM reservations (rather than one + combined TPM reservation) can use each half on its own. + + ``min_configured_tpm_limit`` is the smallest ``tokens_per_unit`` among + the TPM-bearing descriptors this request will be charged against. When + provided, the no-``max_tokens`` output-budget floor is capped at a + fraction of that limit so small TPM caps remain usable. Omit to + preserve the unconstrained floor. + + ``call_type`` disambiguates embeddings from the Responses API: both + put their prompt in ``data["input"]``, but only embeddings have no + output tokens. Unset (the default) preserves the historical + "any `input` means zero output" behavior for callers that don't have + a call type to pass. + + ``configured_output_tokens`` is the operator-declared estimate resolved + from key or team metadata. When provided it replaces the heuristic + floor entirely, so the reservation reflects what this tenant's model + actually emits rather than one constant shared by every tenant. + """ + if not isinstance(data, dict): + return 0, 0 + translated_data: Final = self._translate_google_genai_native_request(data, call_type) + estimable_data: Final = translated_data if translated_data is not None else data + selected_fields: Final[tuple[object | None, object | None, object | None]] = ( + (None, None, estimable_data.get("input")) + if call_type in RESPONSES_API_CALL_TYPES or call_type in EMBEDDING_API_CALL_TYPES + else (None, estimable_data.get("prompt"), None) + if call_type in TEXT_COMPLETION_API_CALL_TYPES + else (estimable_data.get("messages"), None, None) + if call_type + else ( + estimable_data.get("messages"), + estimable_data.get("prompt"), + estimable_data.get("input"), + ) + ) + messages, prompt, input_text = selected_fields + + total_chars: Final = ( + len(get_str_from_messages(messages)) + if isinstance(messages, list) and messages + else len(prompt) + if isinstance(prompt, str) + else sum(len(str(item)) for item in prompt) + if isinstance(prompt, list) + else len(input_text) + if isinstance(input_text, str) + else sum(len(str(item)) for item in input_text) + if isinstance(input_text, list) + else 0 + ) + + estimated_input_tokens: Final = max(1, total_chars // DEFAULT_CHARS_PER_TOKEN) if total_chars > 0 else 0 + + explicit_max_tokens: Final = self._get_explicit_output_cap(data, call_type) + is_embedding: Final = self._is_embedding_request(data, call_type) + + base_output_floor: Final = self.no_max_tokens_output_floor(min_configured_tpm_limit) + output_floor: Final = ( + max(base_output_floor, RESPONSES_API_MIN_OUTPUT_TOKENS) + if call_type in RESPONSES_API_CALL_TYPES + else base_output_floor + ) + max_tokens_estimate: Final = ( + 0 + if is_embedding or (explicit_max_tokens is None and total_chars == 0 and configured_output_tokens is None) + else explicit_max_tokens + if explicit_max_tokens is not None + else configured_output_tokens + if configured_output_tokens is not None + else max(estimated_input_tokens, output_floor) + ) + + return estimated_input_tokens, max_tokens_estimate * self.get_output_candidate_count(data, call_type) + def _is_redis_cluster(self) -> bool: """ Check if the dual cache is using Redis cluster. @@ -933,7 +1213,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): async def should_rate_limit( self, - descriptors: list[RateLimitDescriptor], + descriptors: Sequence[RateLimitDescriptor], parent_otel_span: Span | None = None, read_only: bool = False, skip_tpm_check: bool = False, @@ -1059,7 +1339,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): def _collect_windowed_keys_and_gauges( self, - descriptors: list[RateLimitDescriptor], + descriptors: Sequence[RateLimitDescriptor], skip_tpm_check: bool, ) -> tuple[list[str], dict[str, WindowKeyMetadata], list[ParallelRequestGauge]]: """ @@ -1463,6 +1743,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): meta.append( { "descriptor_key": descriptor_key, + "descriptor_value": descriptor_value, "current_limit": int(limit_value), "rate_limit_type": rlt, "window_key": window_key, @@ -1485,6 +1766,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): descriptor i, refund descriptors 0..i-1's increments. On Lua failure mid-loop, refund applied increments and fall back to in-memory. """ + if not descriptor_groups: + return RateLimitResponse( + overall_code="OK", + statuses=[], # mutable-ok: response contract requires a status list + ) applied: Final[list[list[AtomicCounterMeta]]] = [] statuses: Final[list[RateLimitStatus]] = [] raw: list[CacheCounterValue] @@ -1519,10 +1805,16 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if response["overall_code"] == "OVER_LIMIT": await self._refund_applied_descriptor_groups(applied) return response + if len(descriptor_groups) == 1: + return response applied.append(meta) statuses.extend(response["statuses"]) - return RateLimitResponse(overall_code="OK", statuses=statuses) + return RateLimitResponse( + overall_code="OK", + statuses=statuses, + reservation_windows=frozenset(), + ) async def _refund_applied_descriptor_groups( self, @@ -1585,12 +1877,14 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): limit_remaining=max(0, limit - current_counter), rate_limit_type=meta["rate_limit_type"], descriptor_key=meta["descriptor_key"], + descriptor_value=meta["descriptor_value"], ) ], ) statuses: Final[list[RateLimitStatus]] = [] - for meta, new_counter in zip(per_counter_meta, raw[1:]): + for index, meta in enumerate(per_counter_meta): + new_counter = raw[1 + index * 2] statuses.append( RateLimitStatus( code="OK", @@ -1598,9 +1892,21 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): limit_remaining=max(0, meta["current_limit"] - int(new_counter)), rate_limit_type=meta["rate_limit_type"], descriptor_key=meta["descriptor_key"], + descriptor_value=meta["descriptor_value"], ) ) - return RateLimitResponse(overall_code="OK", statuses=statuses) + return RateLimitResponse( + overall_code="OK", + statuses=statuses, + reservation_windows=frozenset( + ( + meta["counter_key"], + str(int(raw[2 + index * 2])), + "redis", + ) + for index, meta in enumerate(per_counter_meta) + ), + ) async def _atomic_check_and_increment_in_memory( self, @@ -1653,10 +1959,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): limit_remaining=max(0, meta["current_limit"] - current_counter), rate_limit_type=meta["rate_limit_type"], descriptor_key=meta["descriptor_key"], + descriptor_value=meta["descriptor_value"], ) ], ) - descriptor_state.append({"window_expired": window_expired, "current": current_counter}) + descriptor_state.append( + { # mutable-ok: local atomic-counter state is updated during pass two + "window_expired": window_expired, + "current": current_counter, + "window_start": str(now_int if window_expired else int(window_start)), + } + ) # Pass 2: apply increments. statuses: Final[list[RateLimitStatus]] = [] @@ -1684,9 +1997,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): limit_remaining=max(0, meta["current_limit"] - new_counter), rate_limit_type=meta["rate_limit_type"], descriptor_key=meta["descriptor_key"], + descriptor_value=meta["descriptor_value"], ) ) - return RateLimitResponse(overall_code="OK", statuses=statuses) + return RateLimitResponse( + overall_code="OK", + statuses=statuses, + reservation_windows=frozenset( + (meta["counter_key"], state["window_start"], "local") + for meta, state in zip(per_counter_meta, descriptor_state) + ), + ) async def reserve_tpm_tokens( self, @@ -1703,6 +2024,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): TPM-only descriptor/increment list and delegates the all-or-nothing atomicity (Lua on Redis, asyncio-locked DualCache otherwise) to the shared primitive. + + Excludes project ITPM/OTPM descriptors -- those are reserved + separately (different estimate per bucket) via ``reserve_io_tokens``. """ tpm_descriptors: Final[list[RateLimitDescriptor]] = [ RateLimitDescriptor( @@ -1714,7 +2038,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ), ) for d in descriptors - if (d.get("rate_limit") or {}).get("tokens_per_unit") is not None + if d["key"] not in (PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY) + and (d.get("rate_limit") or {}).get("tokens_per_unit") is not None # mutable-ok: optional descriptor ] if not tpm_descriptors: return RateLimitResponse(overall_code="OK", statuses=[]) @@ -1728,6 +2053,179 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): parent_otel_span=parent_otel_span, ) + async def _refund_reserved_tokens( + self, + scopes: Sequence[tuple[str, str]], + amount: int, + reservation_windows: frozenset[tuple[str, str, Literal["redis", "local"]]] = frozenset(), + parent_otel_span: Span | None = None, + ) -> None: + """ + Directly decrement previously-reserved token counters for ``scopes`` + by ``amount``. Used to roll back a reservation that already + succeeded once a *different* bucket in the same request turns out to + be over its limit (e.g. ITPM reserved fine, OTPM then hits its + limit -- the ITPM reservation must not be left inflated). + """ + if amount <= 0 or not scopes: + return + if not reservation_windows: + await self.async_increment_tokens_with_ttl_preservation( + pipeline_operations=self._build_reservation_aware_tpm_ops( + targets=scopes, + reserved_scopes=frozenset(scopes), + actual_tokens=0, + reserved_tokens=amount, + ), + parent_otel_span=parent_otel_span, + ) + return + pipeline_operations: Final = self._build_project_reservation_ops( + targets=scopes, + reserved_scopes=frozenset(scopes), + actual_tokens=0, + reserved_tokens=amount, + reservation_window_identities=reservation_windows, + ) + await self.async_increment_reservation_aware_tokens( + pipeline_operations=pipeline_operations, + parent_otel_span=parent_otel_span, + ) + + async def reserve_io_tokens( + self, + descriptors: Sequence[RateLimitDescriptor], + estimated_input_tokens: int, + estimated_output_tokens: int, + parent_otel_span: Span | None = None, + ) -> tuple[RateLimitResponse, int, int]: + """ + Reserve ``estimated_input_tokens`` against project ITPM descriptors + and ``estimated_output_tokens`` against project OTPM descriptors. + + ITPM and OTPM are reserved from different-sized estimates, so unlike + same-size TPM descriptors they can't share a single + ``atomic_check_and_increment_by_n`` call -- each bucket gets its own + all-or-nothing atomic call. If the OTPM reservation is over limit + after ITPM already succeeded, the ITPM reservation this call made is + rolled back before returning, so a partial reservation never leaks. + + Returns ``(response, itpm_reserved, otpm_reserved)`` -- the latter two + are the amounts actually reserved (0 if that bucket wasn't + configured, or if the reservation failed), for the caller to stash + for post-call reconciliation. + """ + itpm_descriptors: Final = [ # mutable-ok: atomic limiter API requires lists + d for d in descriptors if d["key"] == PROJECT_ITPM_DESCRIPTOR_KEY + ] + otpm_descriptors: Final = [ # mutable-ok: atomic limiter API requires lists + d for d in descriptors if d["key"] == PROJECT_OTPM_DESCRIPTOR_KEY + ] + + if not itpm_descriptors and not otpm_descriptors: + return RateLimitResponse(overall_code="OK", statuses=[]), 0, 0 # mutable-ok: response contract uses a list + + itpm_response: Final = ( + await self.atomic_check_and_increment_by_n( + descriptors=itpm_descriptors, + increments=[ # mutable-ok: atomic limiter API requires mutable increment records + {"tokens": estimated_input_tokens} # mutable-ok: atomic limiter increment record + for _ in itpm_descriptors + ], + parent_otel_span=parent_otel_span, + ) + if itpm_descriptors + else None + ) + if itpm_response is not None and itpm_response["overall_code"] == "OVER_LIMIT": + return itpm_response, 0, 0 + itpm_reserved: Final = estimated_input_tokens if itpm_response is not None else 0 + + if otpm_descriptors: + otpm_response: Final = await self.atomic_check_and_increment_by_n( + descriptors=otpm_descriptors, + increments=[ # mutable-ok: atomic limiter API requires mutable increment records + {"tokens": estimated_output_tokens} # mutable-ok: atomic limiter increment record + for _ in otpm_descriptors + ], + parent_otel_span=parent_otel_span, + ) + if otpm_response["overall_code"] == "OVER_LIMIT": + if itpm_reserved > 0: + await self._refund_reserved_tokens( + scopes=[ # mutable-ok: reservation rollback accepts collected scopes + (d["key"], d["value"]) for d in itpm_descriptors + ], + amount=itpm_reserved, + reservation_windows=itpm_response.get("reservation_windows", frozenset()), + parent_otel_span=parent_otel_span, + ) + return otpm_response, 0, 0 + statuses: Final = ( + [ # mutable-ok: response contract uses a list + *itpm_response["statuses"], + *otpm_response["statuses"], + ] + if itpm_response is not None + else otpm_response["statuses"] + ) + return ( + RateLimitResponse( + overall_code="OK", + statuses=statuses, + reservation_windows=( + ( + itpm_response.get("reservation_windows", frozenset()) + if itpm_response is not None + else frozenset() + ) + | otpm_response.get("reservation_windows", frozenset()) + ), + ), + itpm_reserved, + estimated_output_tokens, + ) + + assert itpm_response is not None + return itpm_response, itpm_reserved, 0 + + async def enforce_project_io_token_quota_for_frame( + self, + user_api_key_dict: UserAPIKeyAuth | None, + requested_model: str | None, + estimated_input_tokens: int, + estimated_output_tokens: int, + ) -> None: + """Reserve one WebSocket ``response.create`` frame's tokens against + the caller's project ITPM/OTPM quota. + + The Responses WebSocket connection-level pre-call hook only runs once + per connection, but a connection accepts many ``response.create`` + frames over its lifetime. Without this, a project caller could send + unlimited high-token generations after a single minimal reservation. + There is no per-frame post-call hook to reconcile against, so -- + like the batch rate limiter -- this charges the estimate immediately + and never refunds it. + """ + if user_api_key_dict is None: + return + descriptors: Final[list[RateLimitDescriptor]] = [] # mutable-ok: descriptor helper appends in place + self.add_project_io_token_rate_limit_descriptors_from_metadata( + user_api_key_dict=user_api_key_dict, + requested_model=requested_model, + descriptors=descriptors, + ) + if not descriptors: + return + response, _itpm_reserved, _otpm_reserved = await self.reserve_io_tokens( + descriptors=descriptors, + estimated_input_tokens=estimated_input_tokens, + estimated_output_tokens=estimated_output_tokens, + parent_otel_span=user_api_key_dict.parent_otel_span, + ) + if response["overall_code"] == "OVER_LIMIT": + self._handle_rate_limit_error(response, descriptors, requested_model) + def create_organization_rate_limit_descriptor( self, user_api_key_dict: UserAPIKeyAuth, requested_model: str | None = None ) -> list[RateLimitDescriptor]: @@ -2434,6 +2932,62 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) ) + def add_project_io_token_rate_limit_descriptors_from_metadata( + self, + user_api_key_dict: UserAPIKeyAuth, + requested_model: str | None, + descriptors: _RateLimitDescriptorSink, + ) -> None: + """Add project-scoped ITPM/OTPM descriptors from project_metadata. + + Enforced independently of, and alongside, the combined ``model_per_project`` + TPM descriptor above -- these give Bedrock Mantle-style separate input/output + token quotas at the project level. + """ + if requested_model is None or user_api_key_dict.project_id is None: + return + + itpm_limit_for_project_model: Final = ( + get_model_rate_limit_from_metadata(user_api_key_dict, "project_metadata", "model_itpm_limit") + or {} # mutable-ok: metadata helper returns an optional mapping + ) + otpm_limit_for_project_model: Final = ( + get_model_rate_limit_from_metadata(user_api_key_dict, "project_metadata", "model_otpm_limit") + or {} # mutable-ok: metadata helper returns an optional mapping + ) + + model_itpm_limit: Final = itpm_limit_for_project_model.get(requested_model) + model_otpm_limit: Final = otpm_limit_for_project_model.get(requested_model) + + if model_itpm_limit is None and model_otpm_limit is None: + return + + descriptor_value: Final = f"{user_api_key_dict.project_id}:{requested_model}" + if model_itpm_limit is not None: + descriptors.append( + RateLimitDescriptor( + key=PROJECT_ITPM_DESCRIPTOR_KEY, + value=descriptor_value, + rate_limit={ # mutable-ok: descriptor TypedDict requires a runtime dict + "requests_per_unit": None, + "tokens_per_unit": model_itpm_limit, + "window_size": self.window_size, + }, + ) + ) + if model_otpm_limit is not None: + descriptors.append( + RateLimitDescriptor( + key=PROJECT_OTPM_DESCRIPTOR_KEY, + value=descriptor_value, + rate_limit={ # mutable-ok: descriptor TypedDict requires a runtime dict + "requests_per_unit": None, + "tokens_per_unit": model_otpm_limit, + "window_size": self.window_size, + }, + ) + ) + def _handle_rate_limit_error( self, response: RateLimitResponse, @@ -2478,6 +3032,342 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): llm_provider=llm_provider, ) + @staticmethod + def _estimate_audio_block_tokens(block: object) -> int: + """ + Token estimate for one ``input_audio`` content block. + + When the block carries a base64 ``data`` payload, the estimate comes + from the decoded byte count (``len(b64) * 3 // 4 // _AUDIO_BYTES_PER_TOKEN``), + assuming the lowest reasonable audio bitrate so we never under-reserve + for higher-quality recordings of the same duration. + + When no payload is present (reference-only block or missing ``data``), + falls back to ``DEFAULT_AUDIO_TOKEN_ESTIMATE``. + """ + if not isinstance(block, dict): + return DEFAULT_AUDIO_TOKEN_ESTIMATE + input_audio: Final = block.get("input_audio") + b64_data: Final = input_audio.get("data") if isinstance(input_audio, dict) else None + if b64_data and isinstance(b64_data, str): + decoded_bytes: Final = len(b64_data) * 3 // 4 + return max(decoded_bytes // _AUDIO_BYTES_PER_TOKEN, DEFAULT_AUDIO_TOKEN_ESTIMATE) + return DEFAULT_AUDIO_TOKEN_ESTIMATE + + @classmethod + def _estimate_audio_content_tokens(cls, messages: object) -> int: + """ + Sum of per-block audio token estimates across all ``messages``. + Returns 0 when there are no ``input_audio`` blocks, which the caller + uses to skip the (relatively expensive) strip pass. + """ + if not isinstance(messages, list): + return 0 + return sum( + cls._estimate_audio_block_tokens(block) + for message in messages + if isinstance(message, dict) + for content in (message.get("content"),) + if isinstance(content, list) + for block in content + if isinstance(block, dict) and block.get("type") == "input_audio" + ) + + @staticmethod + def _strip_audio_content_blocks(messages: object) -> object: + """ + Drop ``input_audio`` content blocks before passing ``messages`` to + ``token_counter``, which raises ``ValueError`` on them (no per-type + handling, unlike images). The audio contribution is added back + separately via ``DEFAULT_AUDIO_TOKEN_ESTIMATE`` so the rest of the + message (text/images/tools) still gets counted accurately instead of + the whole call falling back to the cheap char-count estimate. + """ + if not isinstance(messages, list): + return messages + sanitized: Final[list[object]] = [] # mutable-ok: token_counter requires a list of message dicts + for message in messages: + if not isinstance(message, dict): + sanitized.append(message) + continue + content = message.get("content") + if not isinstance(content, list): + sanitized.append(message) + continue + filtered_content = [ # mutable-ok: token_counter requires list content blocks + block for block in content if not (isinstance(block, dict) and block.get("type") == "input_audio") + ] + sanitized.append( # mutable-ok: token_counter requires mutable message dicts + {**message, "content": filtered_content} # mutable-ok: token_counter requires message dicts + ) + return sanitized + + @staticmethod + def _responses_input_to_chat_messages(data: object) -> Sequence[object]: + """ + Convert a Responses API ``input`` (string or list of input items) into + chat-completion-style messages via the standard LiteLLM transformation + (the same one guardrails use, e.g. ``purview_dlp.py``), so multimodal + ``input_image``/``input_text`` content blocks get counted by + ``token_counter``'s ``messages`` path instead of silently contributing + zero tokens via its ``text`` path, which only joins plain strings. + """ + from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, + ) + + if not isinstance(data, dict): + return () + return LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=data.get("input") or "", + responses_api_request=data, + ) + + @staticmethod + def _count_pretokenized_embedding_input(value: object) -> int | None: + if not isinstance(value, list): + return None + if all(isinstance(token, int) for token in value): + return len(value) + if all( + isinstance(token_ids, list) and all(isinstance(token, int) for token in token_ids) for token_ids in value + ): + return sum(len(token_ids) for token_ids in value) + return None + + @staticmethod + def _rerank_input_to_text(data: Mapping[str, object]) -> str: + documents: Final = data.get("documents") + document_items: Final[Sequence[object]] = documents if isinstance(documents, list) else () # pyright: ignore[reportUnknownVariableType] # rerank documents are validated runtime JSON + input_parts: Final[tuple[object, ...]] = ( # pyright: ignore[reportUnknownVariableType] # list narrowing preserves unknown JSON element types + data.get("query"), + *document_items, + ) + return "\n".join( + str(part) # pyright: ignore[reportUnknownArgumentType] # accepted document dicts have provider-defined fields + for part in input_parts # pyright: ignore[reportUnknownVariableType] # runtime JSON list elements remain unknown after list narrowing + if isinstance(part, (str, dict)) + ) + + def _estimate_precise_input_tokens(self, data: object, model: str | None, call_type: str | None = None) -> int: + """ + Model-aware input token estimate for the project ITPM reservation, + using ``litellm.token_counter`` -- the same approach the + deployment-level itpm/otpm check uses in + ``io_token_rate_limit_check.py``. Unlike the cheap char-count + estimate the combined-TPM path uses, this accounts for image/tool + content and derives per-``input_audio``-block estimates from the + base64 payload size (assuming the lowest reasonable bitrate so + longer recordings always reserve proportionally more), so a burst + of multimodal, tool-heavy, or audio-heavy requests can't each + reserve only the one-token floor and blow past ITPM before + post-call reconciliation catches up. + + For the Responses API, ``input`` is converted to chat messages first + (via ``_responses_input_to_chat_messages``) so its own multimodal + content blocks are counted the same way; ``token_counter``'s ``text`` + argument can only see plain strings in a list, not content blocks. + + Falls back to the cheap char-count estimate if ``token_counter`` + can't resolve a tokenizer for this model (e.g. an unrecognized + custom model name) or otherwise raises -- the audio add-on still + applies on top of the fallback. + """ + from litellm import token_counter + + if not isinstance(data, dict): + return 0 + is_responses_request: Final = call_type in RESPONSES_API_CALL_TYPES + translated_request: Final = ( + None if is_responses_request else self._translate_google_genai_native_request(data, call_type) + ) + is_embedding_request: Final = self._is_embedding_request(data, call_type) + embedding_text: Final = data.get("input") if is_embedding_request else None + pretokenized_input_tokens: Final = ( + self._count_pretokenized_embedding_input(embedding_text) if is_embedding_request else None + ) + if pretokenized_input_tokens is not None: + return pretokenized_input_tokens + + prompt: Final = data.get("prompt") + fallback_text: Final = prompt if prompt is not None else data.get("input") + selected_inputs: Final[tuple[object | None, object | None, object | None, object | None]] = ( + (self._responses_input_to_chat_messages(data), None, data.get("tools"), data.get("tool_choice")) + if is_responses_request + else ( + translated_request.get("messages"), + None, + translated_request.get("tools"), + translated_request.get("tool_choice"), + ) + if translated_request is not None + else (None, embedding_text, data.get("tools"), data.get("tool_choice")) + if is_embedding_request + else (None, self._rerank_input_to_text(data), data.get("tools"), data.get("tool_choice")) + if call_type in RERANK_API_CALL_TYPES + else (None, prompt, data.get("tools"), data.get("tool_choice")) + if call_type in TEXT_COMPLETION_API_CALL_TYPES + else (data.get("messages"), fallback_text, data.get("tools"), data.get("tool_choice")) + ) + messages, selected_text, countable_tools, countable_tool_choice = selected_inputs + + audio_token_estimate: Final = self._estimate_audio_content_tokens(messages) + countable_messages: Final = self._strip_audio_content_blocks(messages) if audio_token_estimate > 0 else messages + + try: + estimate: Final = max( + 0, + int( + token_counter( + model=model or "", + messages=countable_messages, + text=selected_text, + tools=countable_tools, + tool_choice=countable_tool_choice, + use_default_image_token_count=True, + ) + ), + ) + return estimate + audio_token_estimate + except Exception: # noqa: BLE001 # tokenizer failures degrade to the cheap estimate + if call_type in RERANK_API_CALL_TYPES and isinstance(selected_text, str): + return max(0, len(selected_text) // DEFAULT_CHARS_PER_TOKEN) + estimated_input_tokens, _ = self._estimate_input_and_output_tokens(data=data, call_type=call_type) + return estimated_input_tokens + audio_token_estimate + + async def _reserve_project_io_tokens_or_raise( + self, + descriptors: Sequence[RateLimitDescriptor], + data: object, + requested_model: str | None, + user_api_key_dict: UserAPIKeyAuth, + tpm_reservation_scopes: Sequence[tuple[str, str]], + tpm_reservation_amount: int, + call_type: str | None = None, + ) -> None: + """ + Reserve project-scoped ITPM/OTPM tokens (Bedrock Mantle-style + separate input/output token buckets), independently of -- and, when + both are configured, in addition to -- the combined-TPM reservation + the caller already made. Raises (via ``_handle_rate_limit_error``) on + an over-limit reservation, first rolling back the combined-TPM + reservation named by ``tpm_reservation_scopes``/``tpm_reservation_amount`` + if one was made, so a partial reservation never leaks. + """ + if not isinstance(data, dict): + return + stash: Final = claim_request_stash_for_data(data) + io_token_descriptors: Final = [ # mutable-ok: reservation API requires descriptor lists + d for d in descriptors if d["key"] in (PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY) + ] + if not io_token_descriptors: + return + + configured_otpm_limits: Final = [ # mutable-ok: min calculation materializes validated limits + int(v) + for d in io_token_descriptors + if d["key"] == PROJECT_OTPM_DESCRIPTOR_KEY + for v in [ # mutable-ok: comprehension binds the optional descriptor value + (d.get("rate_limit") or {}).get( # mutable-ok: optional descriptor fallback + "tokens_per_unit" + ) + ] + if v is not None + ] + min_configured_otpm_limit: Final = min(configured_otpm_limits) if configured_otpm_limits else None + _, raw_estimated_output_tokens = self._estimate_input_and_output_tokens( + data=data, + min_configured_tpm_limit=min_configured_otpm_limit, + call_type=call_type, + ) + raw_estimated_input_tokens: Final = self._estimate_precise_input_tokens( + data=data, model=requested_model, call_type=call_type + ) + estimated_input_tokens: Final = max(raw_estimated_input_tokens, 1) + estimated_output_tokens: Final = ( + raw_estimated_output_tokens + if self._has_explicit_output_cap(data, call_type) + else max(raw_estimated_output_tokens, 1) + ) + + # Hard-cap generation length so an unbounded response can't overshoot + # the OTPM budget before post-call reconciliation runs, mirroring the + # combined-TPM floor cap in the caller. + self._apply_implicit_output_cap( + data=data, + min_configured_limit=min_configured_otpm_limit, + call_type=call_type, + ) + + io_response, itpm_reserved, otpm_reserved = await self.reserve_io_tokens( + descriptors=io_token_descriptors, + estimated_input_tokens=estimated_input_tokens, + estimated_output_tokens=estimated_output_tokens, + parent_otel_span=user_api_key_dict.parent_otel_span, + ) + + if io_response["overall_code"] == "OVER_LIMIT": + # A combined-TPM reservation may have already succeeded above for + # this same request; refund it too, or its counter stays inflated + # until the window's TTL expires. Mark it released so the + # ProxyRateLimitError we're about to raise doesn't get refunded + # a second time when async_post_call_failure_hook sees the same + # (still-stashed) reservation and refunds it again. + if tpm_reservation_amount > 0: + await self._refund_reserved_tokens( + scopes=tpm_reservation_scopes, + amount=tpm_reservation_amount, + parent_otel_span=user_api_key_dict.parent_otel_span, + ) + stash.reservation_released = True + acquisition: Final = stash.parallel_slot + if acquisition is not None: + await self._release_parallel_request_slots( + acquisition=acquisition, + parent_otel_span=user_api_key_dict.parent_otel_span, + ) + stash.parallel_slot = None + self._handle_rate_limit_error( + response=io_response, + descriptors=descriptors, + requested_model=requested_model, + ) + + if itpm_reserved > 0: + itpm_scopes: Final = tuple( + (d["key"], d["value"]) for d in io_token_descriptors if d["key"] == PROJECT_ITPM_DESCRIPTOR_KEY + ) + stash.itpm_reserved_tokens = itpm_reserved + stash.itpm_reserved_scopes = frozenset(itpm_scopes) + stash.itpm_reserved_window_identities = frozenset( + (counter_key, window_start, backend) + for counter_key, window_start, backend in io_response.get("reservation_windows", frozenset()) + if "model_per_project_itpm" in counter_key + ) + if otpm_reserved > 0: + otpm_scopes: Final = tuple( + (d["key"], d["value"]) for d in io_token_descriptors if d["key"] == PROJECT_OTPM_DESCRIPTOR_KEY + ) + stash.otpm_reserved_tokens = otpm_reserved + stash.otpm_reserved_scopes = frozenset(otpm_scopes) + stash.otpm_reserved_window_identities = frozenset( + (counter_key, window_start, backend) + for counter_key, window_start, backend in io_response.get("reservation_windows", frozenset()) + if "model_per_project_otpm" in counter_key + ) + + if stash.rate_limit_response is not None: + stash.rate_limit_response["statuses"].extend(io_response["statuses"]) + elif io_response["statuses"]: + stash.rate_limit_response = io_response + + verbose_proxy_logger.debug( + "ITPM/OTPM tokens reserved: itpm=%s, otpm=%s for model %s", + itpm_reserved, + otpm_reserved, + requested_model, + ) + async def async_pre_call_hook( self, user_api_key_dict: UserAPIKeyAuth, @@ -2550,6 +3440,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): requested_model=requested_model, descriptors=descriptors, ) + self.add_project_io_token_rate_limit_descriptors_from_metadata( + user_api_key_dict=user_api_key_dict, + requested_model=requested_model, + descriptors=descriptors, + ) # Org Level Rate Limits descriptors.extend(self.create_organization_rate_limit_descriptor(user_api_key_dict, requested_model)) @@ -2565,7 +3460,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): # in-flight request would pre-inflate the :tokens counter by 1, # shrinking the effective TPM budget by N and causing # false-positive 429s under bursts. When reservation is disabled, - # this pass enforces TPM directly from the post-call counters. + # this pass enforces TPM directly from the post-call counters -- + # except for project ITPM/OTPM descriptors, which are excluded + # then because _reserve_project_io_tokens_or_raise below charges + # them unconditionally and counting them here too would + # double-charge every request. parallel_counter_keys: Final = [ self.create_rate_limit_keys(d["key"], d["value"], "max_parallel_requests") for d in descriptors @@ -2573,8 +3472,15 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ] parallel_slot_id: Final = uuid.uuid4().hex if parallel_counter_keys else None + first_pass_descriptors: Final = ( + descriptors + if self.tpm_reservation_enabled + else tuple( + d for d in descriptors if d["key"] not in (PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY) + ) + ) response: Final = await self.should_rate_limit( - descriptors=descriptors, + descriptors=first_pass_descriptors, parent_otel_span=user_api_key_dict.parent_otel_span, skip_tpm_check=self.tpm_reservation_enabled, parallel_slot_id=parallel_slot_id, @@ -2606,32 +3512,39 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): configured_tpm_limits: Final = [ int(v) for d in descriptors + if d["key"] not in (PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY) for v in [(d.get("rate_limit") or {}).get("tokens_per_unit")] if v is not None ] has_tpm_limits: Final = bool(configured_tpm_limits) + # Populated on a successful combined-TPM reservation below, so the + # project ITPM/OTPM block further down can roll it back if a + # different bucket in the same request subsequently hits its + # limit. Stays empty/0 whenever no combined-TPM reservation was + # made (or it was over limit, in which case execution never + # reaches the ITPM/OTPM block -- `_handle_rate_limit_error` raises). + tpm_reservation_scopes: Sequence[tuple[str, str]] = () # rebind-ok: set after successful reservation + tpm_reservation_amount = 0 # rebind-ok: set after successful reservation + if has_tpm_limits and self.tpm_reservation_enabled: min_configured_tpm_limit: Final = min(configured_tpm_limits) - # When the configured TPM cap is small enough to constrain the - # no-max_tokens floor, also hard-cap the model output via - # data["max_tokens"] so concurrent unbounded generations can't - # spend past the limit before post-call reconciliation runs. - # Skip when the request already sets max_tokens or has no - # generation budget at all (embeddings). - capped_floor: Final = self._no_max_tokens_output_floor(min_configured_tpm_limit) - baseline_floor: Final = DEFAULT_MAX_TOKENS_ESTIMATE // _TPM_FLOOR_FRACTION - has_explicit_max_tokens: Final = ( - data.get("max_tokens") is not None or data.get("max_completion_tokens") is not None - ) - is_embedding: Final = data.get("input") is not None configured_output_tokens: Final = get_estimated_output_tokens( user_api_key_dict=user_api_key_dict, model_name=requested_model, ) - if capped_floor < baseline_floor and not has_explicit_max_tokens and not is_embedding: - data["max_tokens"] = max(capped_floor, configured_output_tokens or 0) + + # When the configured TPM cap is small enough to constrain the + # no-max_tokens floor, also hard-cap the model output so + # concurrent unbounded generations can't spend past the limit + # before post-call reconciliation runs. + self._apply_implicit_output_cap( + data=data, + min_configured_limit=min_configured_tpm_limit, + call_type=call_type, + configured_output_tokens=configured_output_tokens, + ) # Floor at 1 token so contentless requests (/responses, # tool-call continuations, empty messages) still flow @@ -2645,6 +3558,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): data=data, model=requested_model, min_configured_tpm_limit=min_configured_tpm_limit, + call_type=call_type, configured_output_tokens=configured_output_tokens, ), 1, @@ -2691,8 +3605,16 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): stash.reserved_scopes = frozenset( (d["key"], d["value"]) for d in descriptors - if (d.get("rate_limit") or {}).get("tokens_per_unit") is not None + if d["key"] not in (PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY) + and (d.get("rate_limit") or {}).get( # mutable-ok: optional descriptor fallback + "tokens_per_unit" + ) + is not None ) + tpm_reservation_scopes = tuple( # rebind-ok: record successful reservation scopes + stash.reserved_scopes + ) + tpm_reservation_amount = estimated_tokens # rebind-ok: record successful reservation amount # Merge TPM statuses into the stored rate-limit response # so x-ratelimit-{key}-remaining-tokens / -limit-tokens @@ -2706,6 +3628,15 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): verbose_proxy_logger.debug( "TPM tokens reserved: %s for model %s", estimated_tokens, requested_model ) + await self._reserve_project_io_tokens_or_raise( + descriptors=descriptors, + data=data, + requested_model=requested_model, + user_api_key_dict=user_api_key_dict, + tpm_reservation_scopes=tpm_reservation_scopes, + tpm_reservation_amount=tpm_reservation_amount, + call_type=call_type, + ) def _create_pipeline_operations( self, @@ -2782,7 +3713,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): return total_tokens @staticmethod - def _aggregate_only_total_tokens(usage: Usage | dict | None) -> int: + def _aggregate_only_total_tokens(usage: Usage | ResponseAPIUsage | Mapping[str, object] | None) -> int: """Total for usage that carries no input/output split, else 0. A source that can only report one number for the whole request (a @@ -2792,24 +3723,43 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): uncharged, which is how pass-through traffic slips past a TPM limit it is supposed to share. """ - if isinstance(usage, Usage): - prompt_tokens, completion_tokens, total_tokens = ( - usage.prompt_tokens or 0, - usage.completion_tokens or 0, - usage.total_tokens or 0, - ) - elif isinstance(usage, dict): - prompt_tokens, completion_tokens, total_tokens = ( - usage.get("prompt_tokens") or 0, - usage.get("completion_tokens") or 0, + if usage is None: + return 0 + token_counts: Final = ( + (usage.prompt_tokens or 0, usage.completion_tokens or 0, usage.total_tokens or 0) + if isinstance(usage, Usage) + else (usage.input_tokens or 0, usage.output_tokens or 0, usage.total_tokens or 0) + if isinstance(usage, ResponseAPIUsage) + else ( + usage.get("prompt_tokens") or usage.get("input_tokens") or 0, + usage.get("completion_tokens") or usage.get("output_tokens") or 0, usage.get("total_tokens") or 0, ) - else: - return 0 - if prompt_tokens or completion_tokens: + ) + prompt_tokens, completion_tokens, total_tokens = token_counts + if prompt_tokens or completion_tokens or not isinstance(total_tokens, int): return 0 return total_tokens + @staticmethod + def _response_usage( + response_obj: object, + ) -> Usage | ResponseAPIUsage | Mapping[str, object] | None: + if isinstance(response_obj, (Usage, ResponseAPIUsage)): + return response_obj + if isinstance( + response_obj, + (ModelResponse, EmbeddingResponse, TextCompletionResponse, BaseLiteLLMOpenAIResponseObject), + ): + usage: Final = getattr(response_obj, "usage", None) + return usage if isinstance(usage, (Usage, ResponseAPIUsage, dict)) else None + if isinstance(response_obj, dict): + nested_usage: Final = response_obj.get("usage") + if isinstance(nested_usage, (Usage, ResponseAPIUsage, dict)): + return nested_usage + return response_obj + return None + async def _execute_token_increment_script( self, pipeline_operations: list["RedisPipelineIncrementOperation"], @@ -2885,6 +3835,116 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): litellm_parent_otel_span=parent_otel_span, ) + async def _apply_local_window_guarded_token_increments( + self, + operations: Sequence[ReservationAwareIncrementOperation], + parent_otel_span: Span | None = None, + ) -> None: + async with self._check_and_increment_lock: + for operation in operations: + window_key = operation.get("window_key") + expected_window_start = operation.get("expected_window_start") + if window_key is None or expected_window_start is None: + continue + active_window_start = await self.internal_usage_cache.async_get_cache( + key=window_key, + litellm_parent_otel_span=parent_otel_span, + local_only=True, + ) + if active_window_start is None or str(active_window_start) != expected_window_start: + continue + current_counter = ( + await self.internal_usage_cache.async_get_cache( + key=operation["key"], + litellm_parent_otel_span=parent_otel_span, + local_only=True, + ) + or 0 + ) + await self.internal_usage_cache.async_set_cache( + key=operation["key"], + value=float(current_counter) + operation["increment_value"], + ttl=operation["ttl"], + litellm_parent_otel_span=parent_otel_span, + local_only=True, + ) + + async def _apply_redis_window_guarded_token_increments( + self, + operations: Sequence[ReservationAwareIncrementOperation], + parent_otel_span: Span | None = None, + ) -> None: + for operation in operations: + window_key = operation.get("window_key") + expected_window_start = operation.get("expected_window_start") + if window_key is None or expected_window_start is None: + continue + if self.window_guarded_token_increment_script is not None: + try: + await self.window_guarded_token_increment_script( + keys=[ # mutable-ok: Redis script interface requires a key list + window_key, + operation["key"], + ], + args=[ # mutable-ok: Redis script interface requires an argument list + expected_window_start, + operation["increment_value"], + operation["ttl"] or 0, + ], + ) + continue + except Exception as e: # noqa: BLE001 # Redis failures use the plain increment fallback + verbose_proxy_logger.warning( + "Window-guarded token adjustment failed for %s: %s", + operation["key"], + e, + ) + if operation["increment_value"] > 0: + await self.internal_usage_cache.async_increment_cache( + key=operation["key"], + value=operation["increment_value"], + litellm_parent_otel_span=parent_otel_span, + ttl=operation["ttl"], + ) + + async def async_increment_reservation_aware_tokens( + self, + pipeline_operations: Sequence[ReservationAwareIncrementOperation], + parent_otel_span: Span | None = None, + ) -> None: + for operation in pipeline_operations: + if operation.get("window_key") is None or operation.get("expected_window_start") is None: + await self.internal_usage_cache.async_increment_cache( + key=operation["key"], + value=operation["increment_value"], + litellm_parent_otel_span=parent_otel_span, + ttl=operation["ttl"], + ) + local_guarded_operations: Final = tuple( + operation + for operation in pipeline_operations + if operation.get("window_key") is not None + and operation.get("expected_window_start") is not None + and operation.get("reservation_backend") == "local" + ) + redis_guarded_operations: Final = tuple( + operation + for operation in pipeline_operations + if operation.get("window_key") is not None + and operation.get("expected_window_start") is not None + and operation.get("reservation_backend") != "local" + ) + if local_guarded_operations: + await self._apply_local_window_guarded_token_increments( + operations=local_guarded_operations, + parent_otel_span=parent_otel_span, + ) + if redis_guarded_operations: + await self._apply_redis_window_guarded_token_increments( + operations=redis_guarded_operations, + parent_otel_span=parent_otel_span, + ) + def get_rate_limit_type(self) -> Literal["output", "input", "total"]: from litellm.proxy.proxy_server import general_settings @@ -2914,6 +3974,164 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): merged[f"{prefix}-limit-{status['rate_limit_type']}"] = status["current_limit"] return merged + @staticmethod + def _resolve_rerank_token_usage(response_obj: object) -> tuple[int, int, bool] | None: + if not isinstance(response_obj, RerankResponse) or response_obj.meta is None: + return None + + rerank_tokens: Final = response_obj.meta.get("tokens") # pyright: ignore[reportUnknownMemberType] # TypedDict's optional generic metadata widens get overloads + if rerank_tokens is not None: + input_tokens: Final = rerank_tokens.get("input_tokens") or 0 # pyright: ignore[reportUnknownMemberType] # token fields are typed integers despite the generic get overload + output_tokens: Final = rerank_tokens.get("output_tokens") or 0 # pyright: ignore[reportUnknownMemberType] # token fields are typed integers despite the generic get overload + if input_tokens or output_tokens: + return max(0, input_tokens), max(0, output_tokens), True + + billed_units: Final = response_obj.meta.get("billed_units") # pyright: ignore[reportUnknownMemberType] # TypedDict's optional generic metadata widens get overloads + if billed_units is not None: + total_tokens: Final = billed_units.get("total_tokens") or 0 # pyright: ignore[reportUnknownMemberType] # billed total is a typed integer despite the generic get overload + if total_tokens: + return max(0, total_tokens), 0, True + return None + + def _resolve_io_token_reconcile_usage( + self, + response_obj: object, + ) -> tuple[int, int, bool]: + """ + Resolve ``(billable_input_tokens, completion_tokens, usage_resolved)`` + for ITPM/OTPM reconciliation. Cache-read tokens are excluded from + billable input -- Bedrock Mantle doesn't count them toward ITPM -- + but they're untouched everywhere else (cost/usage logging still sees + the full prompt token count). + """ + rerank_usage: Final = self._resolve_rerank_token_usage(response_obj) + if rerank_usage is not None: + return rerank_usage + + usage: Final = self._response_usage(response_obj) + + if isinstance(usage, Usage): + prompt_tokens: Final = usage.prompt_tokens or 0 + completion_tokens: Final = usage.completion_tokens or 0 + cached_tokens: Final = ( + getattr(usage.prompt_tokens_details, "cached_tokens", 0) or 0 + if usage.prompt_tokens_details is not None + else 0 + ) + if prompt_tokens == 0 and completion_tokens == 0: + return 0, 0, False + return max(0, prompt_tokens - cached_tokens), completion_tokens, True + + if isinstance(usage, ResponseAPIUsage): + response_input_tokens: Final = usage.input_tokens or 0 + response_output_tokens: Final = usage.output_tokens or 0 + response_cached_tokens: Final = ( + usage.input_tokens_details.cached_tokens or 0 if usage.input_tokens_details is not None else 0 + ) + if response_input_tokens == 0 and response_output_tokens == 0: + return 0, 0, False + return max(0, response_input_tokens - response_cached_tokens), response_output_tokens, True + + if isinstance(usage, Mapping): + raw_prompt_tokens: Final = usage.get("prompt_tokens") or usage.get("input_tokens") or 0 + raw_completion_tokens: Final = usage.get("completion_tokens") or usage.get("output_tokens") or 0 + mapped_prompt_tokens: Final = raw_prompt_tokens if isinstance(raw_prompt_tokens, int) else 0 + mapped_completion_tokens: Final = raw_completion_tokens if isinstance(raw_completion_tokens, int) else 0 + prompt_details: Final = usage.get("prompt_tokens_details") or usage.get("input_tokens_details") + raw_cached_tokens: Final = ( + (prompt_details.get("cached_tokens", 0) if isinstance(prompt_details, dict) else 0) + or usage.get("cache_read_input_tokens") + or 0 + ) + mapped_cached_tokens: Final = raw_cached_tokens if isinstance(raw_cached_tokens, int) else 0 + if mapped_prompt_tokens == 0 and mapped_completion_tokens == 0: + return 0, 0, False + return max(0, mapped_prompt_tokens - mapped_cached_tokens), mapped_completion_tokens, True + + return 0, 0, False + + def _build_io_token_reservation_ops( + self, + kwargs: object, + response_obj: object, + ) -> Sequence[RedisPipelineIncrementOperation]: + """ + Reconcile project ITPM/OTPM reservations to actual usage on success: + ITPM to billable input tokens, OTPM to actual completion tokens. + Reuses ``_build_reservation_aware_tpm_ops``'s delta pattern -- ITPM/OTPM + are stored in the same ":tokens" cache bucket as combined TPM, just + under distinct scope keys, so the reservation-aware increment math is + identical; only the usage fields being reconciled against differ. + """ + if not isinstance(kwargs, dict): + return () + stash: Final = get_request_stash_for_call(_call_id_from_callback_kwargs(kwargs)) + if stash is None: + return () + + itpm_reserved: Final = stash.itpm_reserved_tokens + otpm_reserved: Final = stash.otpm_reserved_tokens + if itpm_reserved <= 0 and otpm_reserved <= 0: + return () + + response_usage: Final = self._resolve_io_token_reconcile_usage(response_obj) + combined_usage: Final = self._resolve_io_token_reconcile_usage(kwargs.get("combined_usage_object")) + aggregate_total: Final = self._aggregate_only_total_tokens( + self._response_usage(response_obj) + ) or self._aggregate_only_total_tokens(self._response_usage(kwargs.get("combined_usage_object"))) + + if not response_usage[2] and not combined_usage[2] and aggregate_total <= 0 and not stash.reservation_released: + return () + resolved_usage: Final = ( + response_usage + if response_usage[2] + else combined_usage + if combined_usage[2] + else (aggregate_total, aggregate_total, True) + if aggregate_total > 0 + else (itpm_reserved, otpm_reserved, False) + ) + billable_input, completion_tokens, _ = resolved_usage + + if stash.reservation_released or ( + not stash.itpm_reserved_window_identities and not stash.otpm_reserved_window_identities + ): + return self._build_reservation_aware_tpm_ops( + targets=tuple(stash.itpm_reserved_scopes), + reserved_scopes=frozenset() if stash.reservation_released else stash.itpm_reserved_scopes, + actual_tokens=billable_input, + reserved_tokens=0 if stash.reservation_released else itpm_reserved, + ) + self._build_reservation_aware_tpm_ops( + targets=tuple(stash.otpm_reserved_scopes), + reserved_scopes=frozenset() if stash.reservation_released else stash.otpm_reserved_scopes, + actual_tokens=completion_tokens, + reserved_tokens=0 if stash.reservation_released else otpm_reserved, + ) + + itpm_ops: Final[Sequence[ReservationAwareIncrementOperation]] = ( + self._build_project_reservation_ops( + targets=tuple(stash.itpm_reserved_scopes), + reserved_scopes=frozenset() if stash.reservation_released else stash.itpm_reserved_scopes, + actual_tokens=billable_input, + reserved_tokens=itpm_reserved, + reservation_window_identities=stash.itpm_reserved_window_identities, + ) + if itpm_reserved > 0 + else () + ) + otpm_ops: Final[Sequence[ReservationAwareIncrementOperation]] = ( + self._build_project_reservation_ops( + targets=tuple(stash.otpm_reserved_scopes), + reserved_scopes=frozenset() if stash.reservation_released else stash.otpm_reserved_scopes, + actual_tokens=completion_tokens, + reserved_tokens=otpm_reserved, + reservation_window_identities=stash.otpm_reserved_window_identities, + ) + if otpm_reserved > 0 + else () + ) + return tuple((*itpm_ops, *otpm_ops)) + def _collect_tpm_scope_targets( self, standard_logging_metadata: dict[str, Any], @@ -2978,8 +4196,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): def _build_reservation_aware_tpm_ops( self, - targets: list[tuple[str, str]], - reserved_scopes: frozenset[tuple[str, str]], + targets: Sequence[tuple[str, str]], + reserved_scopes: Set[tuple[str, str]], actual_tokens: int, reserved_tokens: int, ) -> list[RedisPipelineIncrementOperation]: @@ -3012,6 +4230,66 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) return ops + def _build_project_reservation_op( + self, + scope: tuple[str, str], + reserved_scopes: Set[tuple[str, str]], + actual_tokens: int, + reserved_tokens: int, + reservation_window_identities: frozenset[tuple[str, str, Literal["redis", "local"]]], + ) -> ReservationAwareIncrementOperation | None: + scope_key, scope_value = scope + is_reserved_scope: Final = scope in reserved_scopes + increment: Final = actual_tokens - reserved_tokens if is_reserved_scope else actual_tokens + if increment == 0: + return None + counter_key: Final = self.create_rate_limit_keys(scope_key, scope_value, "tokens") + window_identity: Final = next( + ( + (window_start, backend) + for identity_counter_key, window_start, backend in reservation_window_identities + if identity_counter_key == counter_key + ), + None, + ) + if not is_reserved_scope or window_identity is None: + return ReservationAwareIncrementOperation( + key=counter_key, + increment_value=increment, + ttl=self.window_size, + ) + return ReservationAwareIncrementOperation( + key=counter_key, + increment_value=increment, + ttl=self.window_size, + window_key=f"{{{scope_key}:{scope_value}}}:window", + expected_window_start=window_identity[0], + reservation_backend=window_identity[1], + ) + + def _build_project_reservation_ops( + self, + targets: Sequence[tuple[str, str]], + reserved_scopes: Set[tuple[str, str]], + actual_tokens: int, + reserved_tokens: int, + reservation_window_identities: frozenset[tuple[str, str, Literal["redis", "local"]]], + ) -> tuple[ReservationAwareIncrementOperation, ...]: + return tuple( + operation + for scope in targets + if ( + operation := self._build_project_reservation_op( + scope=scope, + reserved_scopes=reserved_scopes, + actual_tokens=actual_tokens, + reserved_tokens=reserved_tokens, + reservation_window_identities=reservation_window_identities, + ) + ) + is not None + ) + def _build_success_event_pipeline_operations( self, kwargs: Any, @@ -3134,12 +4412,26 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): response_obj=response_obj, rate_limit_type=rate_limit_type, ) - if pipeline_operations: await self.async_increment_tokens_with_ttl_preservation( pipeline_operations=pipeline_operations, parent_otel_span=litellm_parent_otel_span, ) + io_token_operations: Final = self._build_io_token_reservation_ops( + kwargs=kwargs, + response_obj=response_obj, + ) + if io_token_operations: + if isinstance(io_token_operations, list): + await self.async_increment_tokens_with_ttl_preservation( + pipeline_operations=io_token_operations, + parent_otel_span=litellm_parent_otel_span, + ) + else: + await self.async_increment_reservation_aware_tokens( + pipeline_operations=io_token_operations, + parent_otel_span=litellm_parent_otel_span, + ) except Exception as e: verbose_proxy_logger.exception("Error in rate limit success event: %s", e) @@ -3232,9 +4524,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): # already released it (proxy-level rejection that also bubbles up # here as an LLM-error callback). max_parallel_requests is its # own counter and is always decremented per call. - reserved_tokens = 0 - if stash is not None and not stash.reservation_released: - reserved_tokens = stash.reserved_tokens + reserved_tokens, itpm_reserved, otpm_reserved = ( + (0, 0, 0) + if stash is None or stash.reservation_released + else (stash.reserved_tokens, stash.itpm_reserved_tokens, stash.otpm_reserved_tokens) + ) + if stash is not None and reserved_tokens > 0: verbose_proxy_logger.debug("Releasing reserved TPM tokens on failure: %s", reserved_tokens) # Refund only against the scopes the reservation actually @@ -3251,12 +4546,64 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) ) + # Refund project ITPM/OTPM reservations the same way -- full + # refund, since a failed call has no billable usage to reconcile + # against. + itpm_operations: Final = ( + self._build_project_reservation_ops( + targets=tuple(stash.itpm_reserved_scopes), + reserved_scopes=stash.itpm_reserved_scopes, + actual_tokens=0, + reserved_tokens=itpm_reserved, + reservation_window_identities=stash.itpm_reserved_window_identities, + ) + if stash is not None and itpm_reserved > 0 and stash.itpm_reserved_window_identities + else self._build_reservation_aware_tpm_ops( + targets=tuple(stash.itpm_reserved_scopes), + reserved_scopes=stash.itpm_reserved_scopes, + actual_tokens=0, + reserved_tokens=itpm_reserved, + ) + if stash is not None and itpm_reserved > 0 + else () + ) + + otpm_operations: Final = ( + self._build_project_reservation_ops( + targets=tuple(stash.otpm_reserved_scopes), + reserved_scopes=stash.otpm_reserved_scopes, + actual_tokens=0, + reserved_tokens=otpm_reserved, + reservation_window_identities=stash.otpm_reserved_window_identities, + ) + if stash is not None and otpm_reserved > 0 and stash.otpm_reserved_window_identities + else self._build_reservation_aware_tpm_ops( + targets=tuple(stash.otpm_reserved_scopes), + reserved_scopes=stash.otpm_reserved_scopes, + actual_tokens=0, + reserved_tokens=otpm_reserved, + ) + if stash is not None and otpm_reserved > 0 + else () + ) + if pipeline_operations: await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline( increment_list=pipeline_operations, litellm_parent_otel_span=litellm_parent_otel_span, ) - if stash is not None and reserved_tokens > 0: + for project_operations in (itpm_operations, otpm_operations): + if isinstance(project_operations, list): + await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline( + increment_list=project_operations, + litellm_parent_otel_span=litellm_parent_otel_span, + ) + elif project_operations: + await self.async_increment_reservation_aware_tokens( + pipeline_operations=project_operations, + parent_otel_span=litellm_parent_otel_span, + ) + if stash is not None and (reserved_tokens > 0 or itpm_reserved > 0 or otpm_reserved > 0): stash.reservation_released = True except Exception as e: verbose_proxy_logger.exception("Error in rate limit failure event: %s", e) @@ -3334,19 +4681,19 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): traceback_str: str | None = None, ) -> None: """ - Release the parallel-request slot and any TPM reservation when the - request is rejected after the pre-call hook acquired them but before - the LLM call ran (e.g. a downstream guardrail/auth hook raised). - Without this, those resources are stranded — async_log_failure_event - is a litellm completion-level callback and never fires for proxy-side - rejections, so a leaked slot would occupy the gauge for the full - PARALLEL_REQUEST_SLOT_TTL_SECONDS. + Release the parallel-request slot and any TPM/ITPM/OTPM reservation + when the request is rejected after the pre-call hook acquired them + but before the LLM call ran (e.g. a downstream guardrail/auth hook + raised). Without this, those resources are stranded — + async_log_failure_event is a litellm completion-level callback and + never fires for proxy-side rejections, so a leaked slot would occupy + the gauge for the full PARALLEL_REQUEST_SLOT_TTL_SECONDS. Idempotent: the slot release clears the stashed acquisition (and slot - removal is a no-op ZREM on a second run), and the TPM refund is - guarded by the stash's ``reservation_released`` flag — if both this - hook and async_log_failure_event end up running in the same flow, only - the first release/refund applies. + removal is a no-op ZREM on a second run), and the TPM/ITPM/OTPM + refund is guarded by the stash's ``reservation_released`` flag — if + both this hook and async_log_failure_event end up running in the same + flow, only the first release/refund applies. """ try: stash: Final = get_request_stash() @@ -3362,23 +4709,80 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if stash.reservation_released: return reserved_tokens: Final = stash.reserved_tokens - if reserved_tokens <= 0: + itpm_reserved: Final = stash.itpm_reserved_tokens + otpm_reserved: Final = stash.otpm_reserved_tokens + if reserved_tokens <= 0 and itpm_reserved <= 0 and otpm_reserved <= 0: return - ops: Final = self._build_reservation_aware_tpm_ops( - targets=list(stash.reserved_scopes), - reserved_scopes=stash.reserved_scopes, - actual_tokens=0, - reserved_tokens=reserved_tokens, - ) - if ops: - verbose_proxy_logger.debug( - "Releasing reserved TPM tokens on proxy-level rejection: %s", reserved_tokens + combined_ops: Final = ( + self._build_reservation_aware_tpm_ops( + targets=tuple(stash.reserved_scopes), + reserved_scopes=stash.reserved_scopes, + actual_tokens=0, + reserved_tokens=reserved_tokens, ) + if reserved_tokens > 0 + else () + ) + itpm_ops: Final = ( + self._build_project_reservation_ops( + targets=tuple(stash.itpm_reserved_scopes), + reserved_scopes=stash.itpm_reserved_scopes, + actual_tokens=0, + reserved_tokens=itpm_reserved, + reservation_window_identities=stash.itpm_reserved_window_identities, + ) + if itpm_reserved > 0 and stash.itpm_reserved_window_identities + else self._build_reservation_aware_tpm_ops( + targets=tuple(stash.itpm_reserved_scopes), + reserved_scopes=stash.itpm_reserved_scopes, + actual_tokens=0, + reserved_tokens=itpm_reserved, + ) + if itpm_reserved > 0 + else () + ) + otpm_ops: Final = ( + self._build_project_reservation_ops( + targets=tuple(stash.otpm_reserved_scopes), + reserved_scopes=stash.otpm_reserved_scopes, + actual_tokens=0, + reserved_tokens=otpm_reserved, + reservation_window_identities=stash.otpm_reserved_window_identities, + ) + if otpm_reserved > 0 and stash.otpm_reserved_window_identities + else self._build_reservation_aware_tpm_ops( + targets=tuple(stash.otpm_reserved_scopes), + reserved_scopes=stash.otpm_reserved_scopes, + actual_tokens=0, + reserved_tokens=otpm_reserved, + ) + if otpm_reserved > 0 + else () + ) + if combined_ops or itpm_ops or otpm_ops: + verbose_proxy_logger.debug( + "Releasing reserved tokens on proxy-level rejection: tpm=%s, itpm=%s, otpm=%s", + reserved_tokens, + itpm_reserved, + otpm_reserved, + ) + if combined_ops: await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline( - increment_list=ops, + increment_list=combined_ops, litellm_parent_otel_span=user_api_key_dict.parent_otel_span, ) + for project_ops in (itpm_ops, otpm_ops): + if isinstance(project_ops, list): + await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline( + increment_list=project_ops, + litellm_parent_otel_span=user_api_key_dict.parent_otel_span, + ) + elif project_ops: + await self.async_increment_reservation_aware_tokens( + pipeline_operations=project_ops, + parent_otel_span=user_api_key_dict.parent_otel_span, + ) stash.reservation_released = True except Exception as e: verbose_proxy_logger.exception("Error releasing TPM reservation on post-call failure: %s", e) diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 8edb3b42dce..781fe264eb8 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -594,13 +594,16 @@ def _build_aggregated_where_clause( sql_params.append(adjusted_end) p += 1 - # Optional entity filter + # Optional entity filter; an empty list must match nothing, not everything if entity_id is not None: if isinstance(entity_id, list): - placeholders = ", ".join(f"${p + i}" for i in range(len(entity_id))) - sql_conditions.append(f'"{entity_id_field}" IN ({placeholders})') - sql_params.extend(entity_id) - p += len(entity_id) + if entity_id: + placeholders = ", ".join(f"${p + i}" for i in range(len(entity_id))) + sql_conditions.append(f'"{entity_id_field}" IN ({placeholders})') + sql_params.extend(entity_id) + p += len(entity_id) + else: + sql_conditions.append("FALSE") else: sql_conditions.append(f'"{entity_id_field}" = ${p}') sql_params.append(entity_id) diff --git a/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py b/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py index 49c0135ff10..8e8545a51cc 100644 --- a/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py @@ -57,20 +57,37 @@ def _model_table(prisma_client: PrismaClient) -> _ModelTableClient: return ModelRepository(prisma_client).table -def validate_models_exist(model_names: list[str], llm_router: "Router | None") -> tuple[bool, list[str]]: +def validate_models_exist(model_names: Sequence[str], llm_router: "Router | None") -> tuple[bool, Sequence[str]]: """ Validate that all requested model names exist in the router. Checks only exact model name matches. Returns: - Tuple[bool, List[str]]: (all_valid, missing_models) + (all_valid, missing_models) """ if llm_router is None: return False, model_names - router_model_names: Final = set(llm_router.get_model_names()) - missing: Final = [m for m in model_names if m not in router_model_names] - return (len(missing) == 0, missing) + router_model_names: Final = frozenset(llm_router.get_model_names()) + missing: Final = tuple(m for m in model_names if m not in router_model_names) + return (not missing, missing) + + +async def _missing_models_after_read_through( + model_names: Sequence[str], llm_router: "Router | None" +) -> tuple[str, ...]: + from litellm.proxy import proxy_server + from litellm.proxy.common_utils.registry_read_through import ( + model_registry_read_through, + ) + + _, missing = validate_models_exist(model_names=model_names, llm_router=llm_router) + if not missing: + return () + for name in missing: + await model_registry_read_through.attempt(name) + _, still_missing = validate_models_exist(model_names=model_names, llm_router=proxy_server.llm_router) + return tuple(still_missing) def add_access_group_to_deployment(model_info: dict[str, Any], access_group: str) -> tuple[dict[str, Any], bool]: @@ -101,13 +118,21 @@ def _raise_http_if_reload_degraded_serving( before: frozenset[str], written_models: Sequence[tuple[str, object]], access_group: str, + still_desired: frozenset[str] | None, + live_after: frozenset[str] | None, ) -> None: """Same verdict as the model-write endpoints, expressed through this file's HTTPException error convention, with the metadata-only obligation: these writes change group membership, not the models themselves, so a row that was already not serving before the reload is never blamed here; only a model this reload stopped serving is reported.""" - missing, collateral = reload_serving_verdict(before=before, written_models=written_models, written_must_serve=False) + missing, collateral = reload_serving_verdict( + before=before, + written_models=written_models, + written_must_serve=False, + still_desired=still_desired, + live_after=live_after, + ) gone: Final = tuple(dict.fromkeys((*missing, *collateral))) if not gone: return @@ -390,12 +415,12 @@ async def create_model_group( # Validate model_names exist in router (only if using model_names path) if not use_model_ids and has_model_names: assert data.model_names is not None - all_valid, missing_models = validate_models_exist( + missing_models: Final = await _missing_models_after_read_through( model_names=data.model_names, llm_router=llm_router, ) - if not all_valid: + if missing_models: raise HTTPException( status_code=400, detail={"error": f"Model(s) not found: {', '.join(missing_models)}"}, @@ -439,11 +464,13 @@ async def create_model_group( live_before_reload: Final = live_model_ids_snapshot() - await clear_cache() + reload_outcome: Final = await clear_cache() _raise_http_if_reload_degraded_serving( before=live_before_reload, written_models=updated_pairs, access_group=data.access_group, + still_desired=reload_outcome.still_desired, + live_after=reload_outcome.live_after, ) verbose_proxy_logger.info( @@ -654,12 +681,12 @@ async def update_access_group( # Validation: Check if all new models exist (only if using model_names path) if not use_model_ids and has_model_names: assert data.model_names is not None - all_valid, missing_models = validate_models_exist( + missing_models: Final = await _missing_models_after_read_through( model_names=data.model_names, llm_router=llm_router, ) - if not all_valid: + if missing_models: raise HTTPException( status_code=400, detail={"error": f"Model(s) not found: {', '.join(missing_models)}"}, @@ -699,11 +726,13 @@ async def update_access_group( # Clear cache and reload models to pick up the access group changes live_before_reload: Final = live_model_ids_snapshot() - await clear_cache() + reload_outcome: Final = await clear_cache() _raise_http_if_reload_degraded_serving( before=live_before_reload, written_models=list({**dict(stripped_pairs), **dict(updated_pairs)}.items()), access_group=access_group, + still_desired=reload_outcome.still_desired, + live_after=reload_outcome.live_after, ) verbose_proxy_logger.info( @@ -801,11 +830,13 @@ async def delete_access_group( # Clear cache and reload models to pick up the access group changes live_before_reload: Final = live_model_ids_snapshot() - await clear_cache() + reload_outcome: Final = await clear_cache() _raise_http_if_reload_degraded_serving( before=live_before_reload, written_models=removed_pairs, access_group=access_group, + still_desired=reload_outcome.still_desired, + live_after=reload_outcome.live_after, ) verbose_proxy_logger.info( diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 635767f4db7..c5ab7f1fc63 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -50,7 +50,7 @@ from litellm.proxy.vector_store_endpoints.utils import ( get_litellm_managed_vector_store, is_allowed_to_call_vector_store_endpoint, ) -from litellm.secret_managers.main import get_secret_str +from litellm.secret_managers.main import get_secret_str, str_to_bool from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, @@ -1016,15 +1016,21 @@ async def bedrock_proxy_route( raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.") aws_region_name: Final = litellm.utils.get_secret(secret_name="AWS_REGION_NAME") - if _is_bedrock_agent_runtime_route(endpoint=endpoint): # handle bedrock agents - base_target_url: Final = f"https://bedrock-agent-runtime.{aws_region_name}.amazonaws.com" - else: + if not _is_bedrock_agent_runtime_route(endpoint=endpoint): return await bedrock_llm_proxy_route( endpoint=endpoint, request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, ) + + if _is_bedrock_agent_runtime_passthrough_disabled(): + raise HTTPException( + status_code=403, + detail="bedrock-agent-runtime pass-through is disabled on this proxy.", + ) + + base_target_url: Final = f"https://bedrock-agent-runtime.{aws_region_name}.amazonaws.com" encoded_endpoint = httpx.URL(endpoint).path # Ensure endpoint starts with '/' for proper URL construction @@ -1292,6 +1298,15 @@ def _is_bedrock_agent_runtime_route(endpoint: str) -> bool: return False +def _is_bedrock_agent_runtime_passthrough_disabled() -> bool: + from litellm.proxy.proxy_server import general_settings + + setting: Final = general_settings.get("disable_bedrock_agent_runtime_passthrough") + if isinstance(setting, str): + return str_to_bool(setting) is True + return setting is True + + @router.api_route( "/assemblyai/{endpoint:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"], diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py index 1c8bce28454..5f6489a69ca 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py @@ -229,6 +229,25 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): verbose_proxy_logger.warning("Error calculating image editing cost: %s", e) return 0.0 + @staticmethod + def _calculate_embeddings_cost( + litellm_model_response: EmbeddingResponse, + model: str, + custom_llm_provider: str, + ) -> float: + try: + return litellm.completion_cost( + completion_response=litellm_model_response, + model=model, + custom_llm_provider=custom_llm_provider, + call_type="aembedding", + ) + except Exception as e: # noqa: BLE001 # completion_cost raises bare Exception for unmapped models; cost failure must never drop the spend log + verbose_proxy_logger.warning( + "Error calculating embeddings cost for model %s, logging spend with cost 0: %s", model, e + ) + return 0.0 + @staticmethod def _build_responses_api_response_and_cost( model: str, @@ -351,11 +370,10 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): model_response_object=EmbeddingResponse(), response_type="embedding", ) - response_cost = litellm.completion_cost( - completion_response=litellm_model_response, + response_cost = OpenAIPassthroughLoggingHandler._calculate_embeddings_cost( + litellm_model_response=litellm_model_response, model=model, custom_llm_provider=custom_llm_provider, - call_type="aembedding", ) litellm_model_response._hidden_params["response_cost"] = response_cost elif is_image_generation: @@ -471,6 +489,12 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): except Exception as e: verbose_proxy_logger.error("Error in OpenAI passthrough cost tracking: %s", e) + if not is_chat_completions: + unbilled_result: Final[PassThroughEndpointLoggingTypedDict] = { + "result": None, + "kwargs": kwargs, + } + return unbilled_result # Fall back to base handler without cost tracking base_handler = OpenAIPassthroughLoggingHandler() return base_handler.passthrough_chat_handler( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 56036713fa9..be1914ea50c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4120,6 +4120,31 @@ def _swap_in_model_cost_map(new_model_cost_map: dict) -> int: return fetched_model_count +def should_load_db_object(object_type: str | SupportedDBObjectType) -> bool: + """ + Check if an object type should be loaded from the database based on general_settings.supported_db_objects. + + Args: + object_type: Type of object to check (e.g., SupportedDBObjectType.MODELS, "models", etc.) + + Returns: + True if the object should be loaded, False otherwise + """ + supported_db_objects: Final = general_settings.get("supported_db_objects", None) + + if supported_db_objects is None: + return True + + if not isinstance(supported_db_objects, list): + verbose_proxy_logger.warning( + "supported_db_objects is not a list, got %s. Loading all objects.", type(supported_db_objects) + ) + return True + + object_type_str: Final = str(object_type) + return any(str(obj) == object_type_str for obj in supported_db_objects) + + class ProxyConfig: """ Abstraction class on top of config loading/updating logic. Gives us one place to control all config updating logic. @@ -6522,36 +6547,7 @@ class ProxyConfig: return config def _should_load_db_object(self, object_type: str | SupportedDBObjectType) -> bool: - """ - Check if an object type should be loaded from the database based on general_settings.supported_db_objects. - - Args: - object_type: Type of object to check (e.g., SupportedDBObjectType.MODELS, "models", etc.) - - Returns: - True if the object should be loaded, False otherwise - """ - global general_settings - - # Get the supported_db_objects configuration - supported_db_objects: Final = general_settings.get("supported_db_objects", None) - - # If supported_db_objects is not set, load all objects (default behavior) - if supported_db_objects is None: - return True - - # If supported_db_objects is set, only load specified objects - if not isinstance(supported_db_objects, list): - verbose_proxy_logger.warning( - "supported_db_objects is not a list, got %s. Loading all objects.", type(supported_db_objects) - ) - return True - - # Convert object_type to string for comparison (handles both str and enum) - object_type_str: Final = str(object_type) - - # Check if the object type is in the list (supports both str and enum values) - return any(str(obj) == object_type_str for obj in supported_db_objects) + return should_load_db_object(object_type=object_type) async def _get_models_from_db(self, prisma_client: PrismaClient) -> list | None: """ @@ -7094,38 +7090,40 @@ class ProxyConfig: async def _init_guardrails_in_db(self, prisma_client: PrismaClient): from litellm.proxy.guardrails.guardrail_registry import ( + GUARDRAIL_RECONCILE_LOCK, IN_MEMORY_GUARDRAIL_HANDLER, Guardrail, GuardrailRegistry, ) try: - guardrails_in_db: Final[list[Guardrail]] = await GuardrailRegistry.get_all_guardrails_from_db( - prisma_client=prisma_client - ) - verbose_proxy_logger.debug("guardrails from the DB %s", str(guardrails_in_db)) - db_guardrail_ids: Final[set] = set() - for guardrail in guardrails_in_db: - guardrail_id = guardrail.get("guardrail_id") - if guardrail_id: - db_guardrail_ids.add(guardrail_id) - try: - IN_MEMORY_GUARDRAIL_HANDLER.sync_guardrail_from_db( - guardrail=cast(Guardrail, guardrail), - ) - except Exception as e: # noqa: BLE001 # one unloadable row must not stop the remaining guardrails - verbose_proxy_logger.error( - "litellm.proxy.proxy_server.py::ProxyConfig:_init_guardrails_in_db - " - "skipping guardrail '%s' (ID: %s): %s: %s", - guardrail.get("guardrail_name"), - guardrail_id, - type(e).__name__, - e, - ) + async with GUARDRAIL_RECONCILE_LOCK: + guardrails_in_db: Final[list[Guardrail]] = await GuardrailRegistry.get_all_guardrails_from_db( + prisma_client=prisma_client + ) + verbose_proxy_logger.debug("guardrails from the DB %s", str(guardrails_in_db)) + db_guardrail_ids: Final[set] = set() + for guardrail in guardrails_in_db: + guardrail_id = guardrail.get("guardrail_id") + if guardrail_id: + db_guardrail_ids.add(guardrail_id) + try: + IN_MEMORY_GUARDRAIL_HANDLER.sync_guardrail_from_db( + guardrail=cast(Guardrail, guardrail), + ) + except Exception as e: # noqa: BLE001 # one unloadable row must not stop the remaining guardrails + verbose_proxy_logger.error( + "litellm.proxy.proxy_server.py::ProxyConfig:_init_guardrails_in_db - " + "skipping guardrail '%s' (ID: %s): %s: %s", + guardrail.get("guardrail_name"), + guardrail_id, + type(e).__name__, + e, + ) - # Drop in-memory DB-backed entries whose row was deleted on another - # pod. Config-loaded entries are never touched. - IN_MEMORY_GUARDRAIL_HANDLER.reconcile_db_guardrails(db_guardrail_ids=db_guardrail_ids) + # Drop in-memory DB-backed entries whose row was deleted on another + # pod. Config-loaded entries are never touched. + IN_MEMORY_GUARDRAIL_HANDLER.reconcile_db_guardrails(db_guardrail_ids=db_guardrail_ids) except Exception as e: verbose_proxy_logger.exception("litellm.proxy.proxy_server.py::ProxyConfig:_init_guardrails_in_db - %s", e) @@ -7278,13 +7276,17 @@ class ProxyConfig: ) async def _init_agents_in_db(self, prisma_client: PrismaClient): + from litellm.proxy.agent_endpoints.agent_registry import ( + AGENT_RECONCILE_LOCK, + ) from litellm.proxy.agent_endpoints.agent_registry import ( global_agent_registry as AGENT_REGISTRY, ) try: - db_agents: Final = await AGENT_REGISTRY.get_all_agents_from_db(prisma_client=prisma_client) - AGENT_REGISTRY.load_agents_from_db_and_config(db_agents=db_agents) + async with AGENT_RECONCILE_LOCK: + db_agents: Final = await AGENT_REGISTRY.get_all_agents_from_db(prisma_client=prisma_client) + AGENT_REGISTRY.load_agents_from_db_and_config(db_agents=db_agents) except Exception as e: verbose_proxy_logger.exception("litellm.proxy.proxy_server.py::ProxyConfig:_init_agents_in_db - %s", e) @@ -10254,11 +10256,9 @@ async def embeddings( """ global proxy_logging_obj - data: Any = {} + data: Final = await _read_request_body(request=request) + base_llm_response_processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - # Use shared request body reading helper (same as chat/completions) - data = await _read_request_body(request=request) - ### HANDLE TOKEN ARRAY INPUT DECODING ### # This must happen BEFORE base_process_llm_request() since it modifies the input router_model_names: Final = llm_router.model_names if llm_router is not None else [] @@ -10302,10 +10302,6 @@ async def embeddings( if hasattr(user_api_key_dict, "agent_id") and user_api_key_dict.agent_id is not None: data["metadata"]["agent_id"] = user_api_key_dict.agent_id - # Use unified request processor (same as chat/completions and responses) - base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data) - - # Process the request with all optimizations (shared sessions, network tuning, etc.) response: Final = await base_llm_response_processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, @@ -10327,8 +10323,6 @@ async def embeddings( return response except Exception as e: - # Use unified error handler - base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data) raise await base_llm_response_processor._handle_llm_api_exception( e=e, user_api_key_dict=user_api_key_dict, diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index c85325b6fa9..91a0c68fd58 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -146,7 +146,8 @@ ROUTE_ENDPOINT_MAPPING: Final = { class ProxyModelNotFoundError(HTTPException): - def __init__(self, route: str, model_name: str): + def __init__(self, route: str, model_name: str, retryable_with_model_read_through: bool = True): + self.retryable_with_model_read_through: Final = retryable_with_model_read_through detail: Final = { "error": f"{route}: Invalid model name passed in model={model_name}. Call `/v1/models` to view available models for your key." } @@ -320,112 +321,150 @@ async def add_shared_session_to_data(data: dict) -> None: pass +RouteType = Literal[ + "acompletion", + "atext_completion", + "aembedding", + "aimage_generation", + "aspeech", + "atranscription", + "amoderation", + "arerank", + "aresponses", + "aget_responses", + "adelete_responses", + "acancel_responses", + "acompact_responses", + "acreate_response_reply", + "alist_input_items", + "_arealtime", # private function for realtime API + "acreate_realtime_client_secret", + "arealtime_calls", + "acreate_realtime_transcription_session", + "_aresponses_websocket", # private function for responses WebSocket mode + "aimage_edit", + "agenerate_content", + "agenerate_content_stream", + "allm_passthrough_route", + "acreate_batch", + "aretrieve_batch", + "alist_batches", + "afile_content", + "afile_retrieve", + "acreate_fine_tuning_job", + "acancel_fine_tuning_job", + "alist_fine_tuning_jobs", + "aretrieve_fine_tuning_job", + "avector_store_search", + "avector_store_create", + "avector_store_retrieve", + "avector_store_list", + "avector_store_update", + "avector_store_delete", + "avector_store_file_create", + "avector_store_file_list", + "avector_store_file_retrieve", + "avector_store_file_content", + "avector_store_file_update", + "avector_store_file_delete", + "aocr", + "asearch", + "avideo_generation", + "avideo_list", + "avideo_status", + "avideo_content", + "avideo_remix", + "avideo_create_character", + "avideo_get_character", + "avideo_edit", + "avideo_extension", + "acreate_container", + "alist_containers", + "aretrieve_container", + "adelete_container", + "aupload_container_file", + "alist_container_files", + "aretrieve_container_file", + "adelete_container_file", + "aretrieve_container_file_content", + "acreate_skill", + "alist_skills", + "aget_skill", + "adelete_skill", + "aingest", + "anthropic_messages", + "acreate_interaction", + "aget_interaction", + "adelete_interaction", + "acancel_interaction", + "acreate_agent", + "alist_agents", + "aget_agent", + "adelete_agent", + "alist_agent_versions", + "asend_message", + "call_mcp_tool", + "acancel_batch", + "afile_delete", + "acreate_eval", + "alist_evals", + "aget_eval", + "aupdate_eval", + "adelete_eval", + "acancel_eval", + "acreate_run", + "alist_runs", + "aget_run", + "acancel_run", + "adelete_run", +] + + async def route_request( data: dict, llm_router: LitellmRouter | None, user_model: str | None, - route_type: Literal[ - "acompletion", - "atext_completion", - "aembedding", - "aimage_generation", - "aspeech", - "atranscription", - "amoderation", - "arerank", - "aresponses", - "aget_responses", - "adelete_responses", - "acancel_responses", - "acompact_responses", - "acreate_response_reply", - "alist_input_items", - "_arealtime", # private function for realtime API - "acreate_realtime_client_secret", - "arealtime_calls", - "acreate_realtime_transcription_session", - "_aresponses_websocket", # private function for responses WebSocket mode - "aimage_edit", - "agenerate_content", - "agenerate_content_stream", - "allm_passthrough_route", - "acreate_batch", - "aretrieve_batch", - "alist_batches", - "afile_content", - "afile_retrieve", - "acreate_fine_tuning_job", - "acancel_fine_tuning_job", - "alist_fine_tuning_jobs", - "aretrieve_fine_tuning_job", - "avector_store_search", - "avector_store_create", - "avector_store_retrieve", - "avector_store_list", - "avector_store_update", - "avector_store_delete", - "avector_store_file_create", - "avector_store_file_list", - "avector_store_file_retrieve", - "avector_store_file_content", - "avector_store_file_update", - "avector_store_file_delete", - "aocr", - "asearch", - "avideo_generation", - "avideo_list", - "avideo_status", - "avideo_content", - "avideo_remix", - "avideo_create_character", - "avideo_get_character", - "avideo_edit", - "avideo_extension", - "acreate_container", - "alist_containers", - "aretrieve_container", - "adelete_container", - "aupload_container_file", - "alist_container_files", - "aretrieve_container_file", - "adelete_container_file", - "aretrieve_container_file_content", - "acreate_skill", - "alist_skills", - "aget_skill", - "adelete_skill", - "aingest", - "anthropic_messages", - "acreate_interaction", - "aget_interaction", - "adelete_interaction", - "acancel_interaction", - "acreate_agent", - "alist_agents", - "aget_agent", - "adelete_agent", - "alist_agent_versions", - "asend_message", - "call_mcp_tool", - "acancel_batch", - "afile_delete", - "acreate_eval", - "alist_evals", - "aget_eval", - "aupdate_eval", - "adelete_eval", - "acancel_eval", - "acreate_run", - "alist_runs", - "aget_run", - "acancel_run", - "adelete_run", - ], + route_type: RouteType, user_api_key_dict: UserAPIKeyAuth | None = None, ): """ Common helper to route the request """ + try: + return await _route_request_single_attempt( + data=data, + llm_router=llm_router, + user_model=user_model, + route_type=route_type, + user_api_key_dict=user_api_key_dict, + ) + except ProxyModelNotFoundError as e: + requested_model: Final = data.get("model", "") + if not e.retryable_with_model_read_through or not isinstance(requested_model, str) or not requested_model: + raise + from litellm.proxy import proxy_server + from litellm.proxy.common_utils.registry_read_through import ( + model_registry_read_through, + ) + + if not await model_registry_read_through.attempt(requested_model): + raise + return await _route_request_single_attempt( + data=data, + llm_router=proxy_server.llm_router, + user_model=user_model, + route_type=route_type, + user_api_key_dict=user_api_key_dict, + ) + + +async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited provider coroutines; the inferred union keeps route_request's callers typed + data: dict, # mutable-ok: request body is the proxy-wide mutable dict contract shared with route_request + llm_router: LitellmRouter | None, + user_model: str | None, + route_type: RouteType, + user_api_key_dict: UserAPIKeyAuth | None = None, +): raise_if_required_body_param_missing(route_type=route_type, data=data) await add_shared_session_to_data(data) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 2ad7180bd5f..a743526e975 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -40,7 +40,7 @@ from litellm.proxy._types import ( from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.types.guardrails import GuardrailEventHooks from litellm.types.proxy.model_listing import ModelInfoResponse -from litellm.types.utils import CallTypes, CallTypesLiteral, ModelInfo +from litellm.types.utils import CallTypes, CallTypesLiteral, ModelInfo, Usage try: from litellm_enterprise.enterprise_callbacks.send_emails.base_email import ( @@ -403,6 +403,120 @@ def _exception_changes_request_flow(exc: BaseException) -> bool: return isinstance(exc, (SensitiveDataRouteException, ModifyResponseException)) +def _prompt_block_text(block: object) -> str: + if isinstance(block, str): + return block + if not isinstance(block, dict): + return "" + block_text: Final = block.get("text") + return block_text if isinstance(block_text, str) else "" + + +def _system_prompt_text(system_input: object) -> str: + if isinstance(system_input, str): + return system_input + if not isinstance(system_input, list): + return "" + return "".join(_prompt_block_text(block) for block in system_input) + + +def _count_request_input_tokens(model: str, request_input: object, system_input: object) -> int: + system_text: Final = _system_prompt_text(system_input) + system_tokens: Final = litellm.token_counter(model=model, text=system_text) if system_text else 0 + if isinstance(request_input, str): + return system_tokens + litellm.token_counter(model=model, text=request_input) + if not isinstance(request_input, list) or not request_input: + return system_tokens + text_entries: Final = tuple(entry for entry in request_input if isinstance(entry, str)) + if len(text_entries) == len(request_input): + return system_tokens + litellm.token_counter(model=model, text="".join(text_entries)) + return system_tokens + litellm.token_counter( + model=model, messages=request_input, use_default_image_token_count=True + ) + + +def _estimate_dispatched_failure_usage(model: str, request_input: object, system_input: object) -> Usage | None: + """A request that failed after dispatch consumed provider-billed input + tokens, but no provider usage ever came back. Estimate the input side with + the same tokenizer fallback interrupted streams use, so the spend log's + failure row records what was sent instead of zero.""" + try: + input_tokens: Final = _count_request_input_tokens( + model=model, request_input=request_input, system_input=system_input + ) + except Exception: + return None + if input_tokens <= 0: + return None + return Usage(prompt_tokens=input_tokens, completion_tokens=0, total_tokens=input_tokens) + + +_INPUT_ESTIMABLE_CALL_TYPES: Final = frozenset( + call_type.value + for call_type in ( + CallTypes.completion, + CallTypes.acompletion, + CallTypes.text_completion, + CallTypes.atext_completion, + CallTypes.anthropic_messages, + CallTypes.aanthropic_messages, + CallTypes.responses, + CallTypes.aresponses, + CallTypes.embedding, + CallTypes.aembedding, + CallTypes.moderation, + CallTypes.amoderation, + CallTypes.image_generation, + CallTypes.aimage_generation, + CallTypes.speech, + CallTypes.aspeech, + CallTypes.rerank, + CallTypes.arerank, + CallTypes.generate_content, + CallTypes.agenerate_content, + CallTypes.generate_content_stream, + CallTypes.agenerate_content_stream, + ) +) + + +def _failure_usage_to_lift( + model_call_details: Mapping[str, object], + request_body: Mapping[str, object], + dispatched: bool, +) -> tuple[object, object] | None: + """A stream that broke mid-flight still billed the provider for the chunks + already delivered; the streaming handler stashes that recovered usage and + cost in model_call_details, so prefer it. Otherwise a request that was + dispatched to a provider and failed without upstream usage gets an + estimated input-side Usage with zero cost. The raw request body backfills + the system prompt when the SDK bridges an endpoint (e.g. /v1/messages on a + chat-completions provider) without filling optional_params. Returns the + (combined_usage_object, response_cost) pair to lift, or None.""" + recovered_usage: Final = model_call_details.get("combined_usage_object") + if recovered_usage is not None: + return recovered_usage, model_call_details.get("response_cost") + if not dispatched or model_call_details.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL): + return None + if str(model_call_details.get("call_type")) not in _INPUT_ESTIMABLE_CALL_TYPES: + return None + optional_params: Final = model_call_details.get("optional_params") + dispatched_system: Final = ( + (optional_params.get("system") or optional_params.get("instructions")) + if isinstance(optional_params, dict) + else None + ) + system_input: Final = dispatched_system or request_body.get("system") or request_body.get("instructions") + estimated_usage: Final = _estimate_dispatched_failure_usage( + model=str(model_call_details.get("model") or ""), + request_input=model_call_details.get("messages"), + system_input=system_input, + ) + if estimated_usage is None: + return None + return estimated_usage, 0.0 + + @dataclass(frozen=True) class _CallbackCapabilities: """Cached per-hook capability flags derived from ``litellm.callbacks``. @@ -2190,15 +2304,19 @@ class ProxyLogging: if _first_handoff is not None: request_data["first_api_call_start_time"] = _first_handoff - # A stream that broke mid-flight still billed the provider for the - # chunks already delivered; the streaming handler stashes that - # recovered usage and cost here. Lift them onto request_data so the + # Lift recovered partial-stream usage, or an estimated input-side + # usage for a dispatched failure, onto request_data so the # failure-path spend callbacks (which run after the logging object - # is popped) record the real partial spend instead of zero. - _recovered_usage: Final = _model_call_details.get("combined_usage_object") - if _recovered_usage is not None: - request_data["combined_usage_object"] = _recovered_usage - request_data["response_cost"] = _model_call_details.get("response_cost") + # is popped) record real token counts instead of zero. + _usage_to_lift: Final = _failure_usage_to_lift( + model_call_details=_model_call_details, + request_body=request_data, + dispatched=_first_handoff is not None, + ) + if _usage_to_lift is not None: + _lifted_usage, _lifted_cost = _usage_to_lift + request_data["combined_usage_object"] = _lifted_usage + request_data["response_cost"] = _lifted_cost # Remove before callbacks iterate — not serialisable request_data.pop("litellm_logging_obj", None) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 25e5fcb6976..e678fba2852 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -20,7 +20,7 @@ from litellm.constants import ( LITELLM_MAX_STREAMING_DURATION_SECONDS, STREAM_SSE_DONE_STRING, ) -from litellm.exceptions import MidStreamFallbackError +from litellm.exceptions import MidStreamFallbackError, RateLimitError from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -50,6 +50,16 @@ if TYPE_CHECKING: ) +class ProjectQuotaCallback(Protocol): + async def enforce_project_io_token_quota_for_frame( + self, + user_api_key_dict: UserAPIKeyAuth | None, + requested_model: str | None, + estimated_input_tokens: int, + estimated_output_tokens: int, + ) -> None: ... + + @lru_cache(maxsize=1) def _get_openai_response_types(): from litellm.types.llms import openai as openai_types @@ -1326,6 +1336,84 @@ def _build_synthetic_response_events( from litellm._logging import verbose_logger +# Conservative per-frame output-token floor used when a response.create +# frame omits max_output_tokens, so a project OTPM quota can't be bypassed +# by simply never declaring an output cap. +_FRAME_NO_MAX_OUTPUT_TOKENS_FLOOR: Final = 1024 + +# Rough chars-per-token ratio for estimating a frame's input tokens without +# resolving a real per-model tokenizer, matching the conservative estimate +# the proxy's own rate limiter uses for the same purpose. +_FRAME_CHARS_PER_TOKEN_ESTIMATE: Final = 4 + + +def _extract_frame_quota_estimate_inputs(msg_obj: Mapping[str, object]) -> tuple[int, int | None]: + """Extract a rough input-token count and any explicit max_output_tokens + from a ``response.create`` frame, handling both wire shapes: + flat: {"type": "response.create", "input": ..., "max_output_tokens": ...} + nested: {"type": "response.create", "response": {"input": ..., "max_output_tokens": ...}} + """ + nested: Final = msg_obj.get("response") + params: Final[Mapping[str, object]] = ( + nested + if _is_json_object(nested) and nested + else MappingProxyType( # mutable-ok: immediately frozen filtered frame + {k: v for k, v in msg_obj.items() if k != "type"} + ) + ) + text_parts: Final[list[str]] = [] # mutable-ok: local accumulator built in one pass, not shared + pending: Final[list[object]] = [ # mutable-ok: explicit worklist avoids recursion + params.get("input"), + params.get("instructions"), + ] + while pending: + value = pending.pop() + if isinstance(value, str): + text_parts.append(value) + elif _is_json_array(value): + for item in value: + if isinstance(item, str): + text_parts.append(item) + elif _is_json_object(item): + pending.append(item.get("content")) + pending.append(item.get("text")) + total_chars: Final = sum(len(part) for part in text_parts) + estimated_input_tokens: Final = max(1, total_chars // _FRAME_CHARS_PER_TOKEN_ESTIMATE) if total_chars else 0 + + max_output_tokens: Final = params.get("max_output_tokens") + return estimated_input_tokens, max_output_tokens if isinstance(max_output_tokens, int) else None + + +async def _enforce_frame_project_quota( + quota_callbacks: Sequence[ProjectQuotaCallback], + user_api_key_dict: UserAPIKeyAuth | None, + model: str | None, + raw_message: str, +) -> None: + """Charge one response.create frame's estimated tokens against every + registered project ITPM/OTPM quota callback, in isolation from PII + masking / logging so a malformed frame still reaches those callbacks.""" + if not quota_callbacks: + return + try: + msg_obj = json.loads(raw_message) + except (json.JSONDecodeError, TypeError): + return + if not _is_json_object(msg_obj) or msg_obj.get("type") != "response.create": + return + estimated_input_tokens, explicit_max_output_tokens = _extract_frame_quota_estimate_inputs(msg_obj) + estimated_output_tokens: Final = ( + explicit_max_output_tokens if explicit_max_output_tokens is not None else _FRAME_NO_MAX_OUTPUT_TOKENS_FLOOR + ) + for callback in quota_callbacks: + await callback.enforce_project_io_token_quota_for_frame( + user_api_key_dict=user_api_key_dict, + requested_model=model, + estimated_input_tokens=estimated_input_tokens, + estimated_output_tokens=estimated_output_tokens, + ) + + RESPONSES_WS_LOGGED_EVENT_TYPES: Final = [ "response.created", "response.completed", @@ -1360,6 +1448,7 @@ class ResponsesWebSocketStreaming: first_message: str | None = None, guardrail_callbacks: list[Any] | None = None, output_guardrail_callbacks: list[PresidioGuardrailCallback] | None = None, + quota_callbacks: Sequence[ProjectQuotaCallback] | None = None, authorized_model: str | None = None, ): self.websocket = websocket @@ -1372,6 +1461,7 @@ class ResponsesWebSocketStreaming: self.first_message = first_message self.guardrail_callbacks: list[Any] = guardrail_callbacks or [] self.output_guardrail_callbacks: list[PresidioGuardrailCallback] = output_guardrail_callbacks or [] + self.quota_callbacks: tuple[ProjectQuotaCallback, ...] = tuple(quota_callbacks) if quota_callbacks else () # Model name authorized at connection time; enforced on every # response.create frame to prevent deployment-substitution attacks. self.authorized_model: str | None = authorized_model @@ -1781,10 +1871,39 @@ class ResponsesWebSocketStreaming: return json.dumps(evt_obj) if modified else response_str + async def _enforce_or_reject_frame(self, message: str) -> bool: + """Run the per-frame project quota check. + + On rejection, sends an ``error`` event to the client and reports that + the frame must be dropped instead of forwarded, so the connection + stays open for the client to retry once the window resets. + """ + try: + await _enforce_frame_project_quota( + self.quota_callbacks, self.user_api_key_dict, self.authorized_model, message + ) + except RateLimitError as e: + try: + await self.websocket.send_text( + json.dumps( # mutable-ok: WebSocket wire payload requires JSON objects + { # mutable-ok: WebSocket wire payload requires JSON objects + "type": "error", + "error": { # mutable-ok: nested WebSocket error object + "type": "rate_limit_exceeded", + "message": str(e), + }, + } + ) + ) + except Exception: # noqa: BLE001, S110 # client may already be gone + pass + return False + return True + async def client_to_backend(self) -> None: """Forward response.create events from client to backend.""" try: - if self.first_message is not None: + if self.first_message is not None and await self._enforce_or_reject_frame(self.first_message): masked_first: Final = await self._mask_response_create(self.first_message) self._store_input(masked_first) self._store_event(masked_first) @@ -1792,6 +1911,8 @@ class ResponsesWebSocketStreaming: while True: message = await self.websocket.receive_text() + if not await self._enforce_or_reject_frame(message): + continue masked = await self._mask_response_create(message) self._store_input(masked) self._store_event(masked) @@ -1871,6 +1992,7 @@ class ManagedResponsesWebSocketHandler: timeout: float | None = None, custom_llm_provider: str | None = None, first_message: str | None = None, + quota_callbacks: Sequence[ProjectQuotaCallback] | None = None, **kwargs: object, ) -> None: self.websocket = websocket @@ -1887,6 +2009,7 @@ class ManagedResponsesWebSocketHandler: self.custom_llm_provider = custom_llm_provider self._connection_provider = self._resolve_provider(model) or custom_llm_provider self.first_message = first_message + self.quota_callbacks: tuple[ProjectQuotaCallback, ...] = tuple(quota_callbacks) if quota_callbacks else () # Carry through safe pass-through kwargs (e.g. extra_headers) self.extra_kwargs: dict[str, object] = {k: v for k, v in kwargs.items() if k not in _MANAGED_WS_SKIP_KWARGS} # In-memory session history: response_id → full accumulated message list. @@ -2292,6 +2415,14 @@ class ManagedResponsesWebSocketHandler: verbose_logger.debug("ManagedResponsesWS: error sending warmup ack: %s", exc) return + try: + await _enforce_frame_project_quota( + self.quota_callbacks, self.user_api_key_dict, self.model_group or self.model, raw_message + ) + except RateLimitError as e: + await self._send_error(str(e), error_type="rate_limit_exceeded") + return + call_kwargs: Final = self._build_base_call_kwargs(msg_obj) call_kwargs["stream"] = True diff --git a/litellm/router.py b/litellm/router.py index c5881960c80..00c6c4c8b6f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8573,11 +8573,9 @@ class Router: Returns: - The added/updated deployment """ + _deployment_model_id: Final = deployment.model_info.id or "" + _deployment_on_router: Final[Deployment | None] = self.get_deployment(model_id=_deployment_model_id) try: - # check if deployment already exists - _deployment_model_id: Final = deployment.model_info.id or "" - - _deployment_on_router: Final[Deployment | None] = self.get_deployment(model_id=_deployment_model_id) if _deployment_on_router is not None: # deployment with this model_id exists on the router if ( @@ -8628,10 +8626,31 @@ class Router: deployment.model_info.id, e, ) + self._restore_deployment_after_failed_upsert( + previous_deployment=_deployment_on_router, model_id=_deployment_model_id + ) return None else: raise e + def _restore_deployment_after_failed_upsert(self, previous_deployment: Deployment | None, model_id: str) -> None: + if previous_deployment is None or self.has_model_id(model_id): + return + try: + self.add_deployment(deployment=previous_deployment) + verbose_router_logger.info( + "Restored deployment %s (id=%s); it keeps serving its previous configuration.", + previous_deployment.model_name, + model_id, + ) + except Exception as restore_error: # noqa: BLE001 # best-effort restore: a second failure must not abort the reload + verbose_router_logger.warning( + "Could not restore previously served deployment %s (id=%s) after the failed upsert: %s", + previous_deployment.model_name, + model_id, + restore_error, + ) + @staticmethod def _backend_cost_map_keys(model: str, custom_llm_provider: str | None) -> tuple[str, ...]: """The ``litellm.model_cost`` keys a deployment's shared backend info is registered under.""" @@ -11463,8 +11482,10 @@ class Router: deployment the strategy was registered from via its (model_name, tags) pair. - With tag filtering enabled, strategies that all carry real tags matching - none of the request's do not capture it when the name also has plain + With tag filtering enabled, router-wide or by the request's + enable_tag_filtering (which the proxy sets from key/team + router_settings), strategies that all carry real tags matching none of + the request's do not capture it when the name also has plain deployments: returning None hands the request to ordinary tag-aware deployment selection. """ @@ -11487,8 +11508,9 @@ class Router: for tagged in candidates: if "default" in tagged.tags: return tagged + request_scoped_filtering: Final = request_kwargs.get("enable_tag_filtering") is True if ( - self.enable_tag_filtering + (self.enable_tag_filtering or request_scoped_filtering) and all(tagged.tags for tagged in candidates) and self._model_name_has_plain_deployments(model) ): diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 63bc5203417..3c9a4097321 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -253,6 +253,7 @@ def get_fallback_model_group(fallbacks: list[Any], model_group: str) -> tuple[li PROVIDER_SCOPED_RESOURCE_KEYS: Final = ("input_file_id", "training_file") +PROVIDER_SCOPED_CREATION_FUNCTION_NAMES: Final = frozenset({"_acreate_file"}) def _get_fallback_target_model_group(fallback_entry: str | Mapping[str, object]) -> str | None: @@ -274,6 +275,18 @@ def references_provider_scoped_resource(kwargs: Mapping[str, object]) -> bool: return any(kwargs.get(key) for key in PROVIDER_SCOPED_RESOURCE_KEYS) +def creates_provider_scoped_resource(kwargs: Mapping[str, object]) -> bool: + """ + True when the request creates a resource that will live under one provider's credentials. + + A file uploaded for batches or fine-tuning is stored in the account of the deployment + that handled it, and its id is only usable against the model group the caller named. + Letting the upload fall back to a different model group silently stores the file with + the wrong provider, and every later use of the returned id fails. + """ + return getattr(kwargs.get("original_function"), "__name__", None) in PROVIDER_SCOPED_CREATION_FUNCTION_NAMES + + async def run_async_fallback( *args: tuple[Any], litellm_router: LitellmRouter, @@ -322,7 +335,9 @@ async def run_async_fallback( metadata_variable_name: Final = _get_router_metadata_variable_name( function_name=getattr(kwargs.get("original_function"), "__name__", None) ) - same_model_group_only: Final = references_provider_scoped_resource(kwargs) + same_model_group_only: Final = references_provider_scoped_resource(kwargs) or creates_provider_scoped_resource( + kwargs + ) # Read out of kwargs and narrowed here rather than declared as a parameter: every caller # reaches this function by spreading a loosely-typed kwargs dict, so a declared parameter # would carry an annotation that no call site can actually be checked against. diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index aeeeca21d3b..d09503cdc4d 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -224,6 +224,14 @@ class MCPServer(BaseModel): JWT) but forwards the caller's separate upstream ``Authorization`` unchanged, minting nothing.""" return self.auth_type == MCPAuth.oauth_delegate + @property + def is_client_forwarded_token(self) -> bool: + """True for the two modes whose upstream credential is the caller's own bearer, forwarded + unchanged: the gateway mints nothing for them and holds no OAuth client identity, so a + discovered ``authorization_url`` / ``token_url`` enriches only the gateway's own OAuth front + door and is never a precondition for opening a session.""" + return self.is_true_passthrough or self.is_oauth_delegate + @property def is_dcr_bridge(self) -> bool: """True when this client-forwarded-token server serves the gateway-hosted DCR front door @@ -231,7 +239,7 @@ class MCPServer(BaseModel): authorize, and token relays) instead of relaying the upstream's own OAuth discovery verbatim. ``dcr_bridge`` is rejected on every other auth type at create, update, and config load, so the mode gate here only defends rows edited outside those paths.""" - return bool(self.dcr_bridge) and (self.is_true_passthrough or self.is_oauth_delegate) + return bool(self.dcr_bridge) and self.is_client_forwarded_token @property def requires_per_user_auth(self) -> bool: @@ -248,7 +256,7 @@ class MCPServer(BaseModel): if self.needs_user_oauth_token: return True - if self.is_true_passthrough or self.is_oauth_delegate: + if self.is_client_forwarded_token: return True # PAT passthrough: auth_type is none but extra_headers includes auth headers diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 6f6f317b257..5b925e610e7 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1631,8 +1631,20 @@ class PromptTokensDetailsWrapper( def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) + extra_fields: Final = self.model_extra + nested_cache_creation_input_tokens: Final = ( + extra_fields.get("cache_creation_input_tokens") if extra_fields is not None else None + ) self.cache_write_tokens = ( - self.cache_write_tokens if self.cache_write_tokens is not None else self.cache_creation_tokens + self.cache_write_tokens + if self.cache_write_tokens is not None + else ( + self.cache_creation_tokens + if self.cache_creation_tokens is not None + else ( + nested_cache_creation_input_tokens if isinstance(nested_cache_creation_input_tokens, int) else None + ) + ) ) if self.character_count is None: del self.character_count diff --git a/litellm/utils.py b/litellm/utils.py index 1c880ee9521..a7b70c4129a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -65,6 +65,7 @@ from litellm.constants import ( DEFAULT_EMBEDDING_PARAM_VALUES, DEFAULT_MAX_LRU_CACHE_SIZE, DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT, + DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, DEFAULT_TRIM_RATIO, FUNCTION_DEFINITION_TOKEN_COUNT, INITIAL_RETRY_DELAY, @@ -7638,12 +7639,20 @@ def validate_and_fix_openai_tools(tools: list | None) -> list[dict] | None: def validate_and_fix_thinking_param( - thinking: AnthropicThinkingParam | None, + thinking: AnthropicThinkingParam | bool | None, ) -> AnthropicThinkingParam | None: """ - Normalizes camelCase keys in the thinking param to snake_case. + Coerces bool thinking values (True becomes enabled with the default medium budget, False becomes None) + and normalizes camelCase keys in the thinking param to snake_case. Handles clients that send budgetTokens instead of budget_tokens. """ + if thinking is True: + return cast( + "AnthropicThinkingParam", + {"type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET}, + ) + if thinking is False: + return None if thinking is None or not isinstance(thinking, dict): return thinking normalized: Final = dict(thinking) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 409022016b0..07f9027313b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -14456,6 +14456,25 @@ "supports_tool_choice": true, "supports_output_config": true }, + "databricks/databricks-claude-opus-4-6": { + "input_cost_per_token": 5.00003e-06, + "input_dbu_cost_per_token": 7.1429e-05, + "litellm_provider": "databricks", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 2.5000010000000002e-05, + "output_dbu_cost_per_token": 0.000357143, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "databricks/databricks-claude-sonnet-4": { "input_cost_per_token": 2.9999900000000002e-06, "input_dbu_cost_per_token": 4.2857e-05, @@ -14513,6 +14532,25 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "databricks/databricks-claude-sonnet-4-6": { + "input_cost_per_token": 2.9999900000000002e-06, + "input_dbu_cost_per_token": 4.2857e-05, + "litellm_provider": "databricks", + "max_input_tokens": 1000000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 1.5000020000000002e-05, + "output_dbu_cost_per_token": 0.000214286, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "databricks/databricks-gemini-2-5-flash": { "input_cost_per_token": 3.0001999999999996e-07, "input_dbu_cost_per_token": 4.285999999999999e-06, @@ -14547,6 +14585,74 @@ "supports_function_calling": true, "supports_tool_choice": true }, + "databricks/databricks-gemini-3-1-flash-lite": { + "input_cost_per_token": 3.1248e-07, + "input_dbu_cost_per_token": 4.464e-06, + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 1.87502e-06, + "output_dbu_cost_per_token": 2.6786e-05, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supports_function_calling": true, + "supports_tool_choice": true + }, + "databricks/databricks-gemini-3-1-pro": { + "input_cost_per_token": 2.49998e-06, + "input_dbu_cost_per_token": 3.5714e-05, + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 1.5000020000000002e-05, + "output_dbu_cost_per_token": 0.000214286, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supports_function_calling": true, + "supports_tool_choice": true + }, + "databricks/databricks-gemini-3-flash": { + "input_cost_per_token": 6.2503e-07, + "input_dbu_cost_per_token": 8.929e-06, + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 3.74997e-06, + "output_dbu_cost_per_token": 5.3571e-05, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supports_function_calling": true, + "supports_tool_choice": true + }, + "databricks/databricks-gemini-3-pro": { + "input_cost_per_token": 2.49998e-06, + "input_dbu_cost_per_token": 3.5714e-05, + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 1.5000020000000002e-05, + "output_dbu_cost_per_token": 0.000214286, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supports_function_calling": true, + "supports_tool_choice": true + }, "databricks/databricks-gemma-3-12b": { "input_cost_per_token": 1.5000999999999998e-07, "input_dbu_cost_per_token": 2.1429999999999996e-06, @@ -14592,6 +14698,126 @@ "output_dbu_cost_per_token": 0.000142857, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving" }, + "databricks/databricks-gpt-5-1-codex-max": { + "input_cost_per_token": 1.24999e-06, + "input_dbu_cost_per_token": 1.7857e-05, + "litellm_provider": "databricks", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 9.999990000000002e-06, + "output_dbu_cost_per_token": 0.000142857, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving" + }, + "databricks/databricks-gpt-5-1-codex-mini": { + "input_cost_per_token": 2.4997e-07, + "input_dbu_cost_per_token": 3.571e-06, + "litellm_provider": "databricks", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 1.99997e-06, + "output_dbu_cost_per_token": 2.8571e-05, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving" + }, + "databricks/databricks-gpt-5-2": { + "input_cost_per_token": 1.75e-06, + "input_dbu_cost_per_token": 2.5e-05, + "litellm_provider": "databricks", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 1.4e-05, + "output_dbu_cost_per_token": 0.0002, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving" + }, + "databricks/databricks-gpt-5-2-codex": { + "input_cost_per_token": 1.75e-06, + "input_dbu_cost_per_token": 2.5e-05, + "litellm_provider": "databricks", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 1.4e-05, + "output_dbu_cost_per_token": 0.0002, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving" + }, + "databricks/databricks-gpt-5-3-codex": { + "input_cost_per_token": 1.75e-06, + "input_dbu_cost_per_token": 2.5e-05, + "litellm_provider": "databricks", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 1.4e-05, + "output_dbu_cost_per_token": 0.0002, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving" + }, + "databricks/databricks-gpt-5-4": { + "input_cost_per_token": 2.49998e-06, + "input_dbu_cost_per_token": 3.5714e-05, + "litellm_provider": "databricks", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 1.5000020000000002e-05, + "output_dbu_cost_per_token": 0.000214286, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving" + }, + "databricks/databricks-gpt-5-4-mini": { + "input_cost_per_token": 7.4998e-07, + "input_dbu_cost_per_token": 1.0714e-05, + "litellm_provider": "databricks", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 4.50002e-06, + "output_dbu_cost_per_token": 6.4286e-05, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving" + }, + "databricks/databricks-gpt-5-4-nano": { + "input_cost_per_token": 1.9999e-07, + "input_dbu_cost_per_token": 2.857e-06, + "litellm_provider": "databricks", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 1.24999e-06, + "output_dbu_cost_per_token": 1.7857e-05, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving" + }, "databricks/databricks-gpt-5-mini": { "input_cost_per_token": 2.4997000000000006e-07, "input_dbu_cost_per_token": 3.571e-06, diff --git a/pyproject.toml b/pyproject.toml index 275343ccef6..ffbc96eefb9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm" -version = "1.98.0" +version = "1.99.0" description = "Library to easily interface with LLM API providers" readme = "README.md" requires-python = ">=3.10, <3.15" @@ -67,8 +67,8 @@ proxy = [ "azure-identity>=1.25.2,<2.0", "azure-storage-blob>=12.28.0,<13.0", "mcp>=1.28.1,<2.0", - "litellm-proxy-extras==0.4.86", - "litellm-enterprise==0.1.56", + "litellm-proxy-extras==0.4.87", + "litellm-enterprise==0.1.57", "RestrictedPython>=8.1,<9.0", "rich>=13.9.4,<14.0", "InquirerPy>=0.3.4,<1.0", @@ -306,7 +306,7 @@ members = ["enterprise", "litellm-proxy-extras"] profile = "black" [tool.commitizen] -version = "1.98.0" +version = "1.99.0" version_files = [ "pyproject.toml:^version", ] diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index 6882479a344..096e039b8aa 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -12,7 +12,7 @@ "limit": 2017 }, "ANN202": { - "limit": 855 + "limit": 854 }, "ANN204": { "limit": 711 diff --git a/tests/batches_tests/test_batch_rate_limits.py b/tests/batches_tests/test_batch_rate_limits.py index 0baff6c17be..bb41c1137da 100644 --- a/tests/batches_tests/test_batch_rate_limits.py +++ b/tests/batches_tests/test_batch_rate_limits.py @@ -27,6 +27,16 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import ( from litellm.proxy.utils import InternalUsageCache +def _build_batch_limiter() -> _PROXY_BatchRateLimiter: + internal_usage_cache = InternalUsageCache(dual_cache=DualCache()) + return _PROXY_BatchRateLimiter( + internal_usage_cache=internal_usage_cache, + parallel_request_limiter=_PROXY_MaxParallelRequestsHandler_v3( + internal_usage_cache=internal_usage_cache + ), + ) + + def get_expected_batch_file_usage(file_path: str) -> tuple[int, int]: """ Helper function to calculate expected request count and token count from a batch JSONL file. @@ -69,10 +79,7 @@ async def test_batch_rate_limits(): """ litellm._turn_on_debug() CUSTOM_LLM_PROVIDER = "openai" - BATCH_LIMITER = _PROXY_BatchRateLimiter( - internal_usage_cache=None, - parallel_request_limiter=None, - ) + BATCH_LIMITER = _build_batch_limiter() file_name = "openai_batch_completions.jsonl" _current_dir = os.path.dirname(os.path.abspath(__file__)) @@ -580,10 +587,7 @@ async def test_batch_rate_limiter_without_user_context(tmp_path): CUSTOM_LLM_PROVIDER = "openai" # Setup - BATCH_LIMITER = _PROXY_BatchRateLimiter( - internal_usage_cache=None, - parallel_request_limiter=None, - ) + BATCH_LIMITER = _build_batch_limiter() # Create a simple batch file batch_content = """{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hello"}]}}""" diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml index ebbfd3415a5..b50551ec105 100644 --- a/tests/e2e/coverage_registry/reliability.yaml +++ b/tests/e2e/coverage_registry/reliability.yaml @@ -18,6 +18,16 @@ - {id: reliability.routing.usage_based.picks_under_tpm, module: reliability, tier: P0, behavior: routing, variant: usage_based, assertions: [picks_under_tpm], exercised_on: [chat_completions, messages], source: "router_strategy/lowest_tpm_rpm_v2.py", rationale: "Routes to lowest-TPM deployment; prevents over-allocation"} - {id: reliability.routing.least_busy.picks_lowest_traffic, module: reliability, tier: P1, behavior: routing, variant: least_busy, assertions: [picks_lowest_traffic], exercised_on: [chat_completions, messages], source: "router_strategy/least_busy.py", rationale: "Fewest in-flight requests"} - {id: reliability.routing.complexity_llm_classifier.routes_by_llm_tier, module: reliability, tier: P1, behavior: routing, variant: complexity_llm_classifier, assertions: [routes_by_llm_tier], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py", fail_before_fix: proven, rationale: "v2 auto-router LLM complexity classifier runs over the proxy and routes by semantic tier instead of silently crashing on absent litellm_metadata and falling back to heuristic scoring"} +- {id: reliability.routing.tagged_marker.request_tag_selects_marker, module: reliability, tier: P0, behavior: routing, variant: tagged_marker, assertions: [request_tag_selects_marker], exercised_on: [chat_completions], source: "litellm/router.py:11445", rationale: "Tagged request selects the tagged strategy marker under a shared model_name instead of the plain deployment registered first (GitHub issue #36619)"} +- {id: reliability.routing.tagged_marker.untagged_request_served_by_plain_deployment, module: reliability, tier: P0, behavior: routing, variant: tagged_marker, assertions: [untagged_request_served_by_plain_deployment], exercised_on: [chat_completions, messages, responses], source: "litellm/router.py:11445", rationale: "Untagged requests to a shared model_name are served by the plain deployment on every call, never captured or errored by the tagged marker (GitHub issue #36620)"} +- {id: reliability.routing.tagged_marker.header_tag_selects_marker, module: reliability, tier: P0, behavior: routing, variant: tagged_marker, assertions: [header_tag_selects_marker], exercised_on: [messages], source: "litellm/router.py:11445", rationale: "A request tagged only via the x-litellm-tags header selects the tagged marker on Anthropic-native /v1/messages (GitHub issue #36621)"} +- {id: reliability.routing.tagged_marker.untagged_tier_deployments_still_served, module: reliability, tier: P1, behavior: routing, variant: tagged_marker, assertions: [untagged_tier_deployments_still_served], exercised_on: [chat_completions, messages], source: "litellm/router_strategy/tag_based_routing.py:433", rationale: "Routing tags the marker consumed no longer constrain deployment selection inside the routed tier group, so untagged tier deployments serve the rewrite (GitHub issue #36621)"} +- {id: reliability.routing.tagged_marker.tag_semantics_stay_strict, module: reliability, tier: P1, behavior: routing, variant: tagged_marker, assertions: [tag_semantics_stay_strict], exercised_on: [chat_completions], source: "litellm/router_strategy/tag_based_routing.py:299", rationale: "Tag consumption must not loosen strict semantics: a tagged call aimed straight at an untagged deployment still gets the 401 tags-configuration denial"} +- {id: reliability.routing.tagged_marker.responses_input_routes_through_marker, module: reliability, tier: P0, behavior: routing, variant: tagged_marker, assertions: [responses_input_routes_through_marker], exercised_on: [responses], source: "litellm/router.py:11489", rationale: "Tagged /v1/responses (header or litellm_metadata.tags, string or list input) routes through the marker to its tier, extending the GitHub issues #36620/#36621 tag split to the Responses surface"} +- {id: reliability.routing.tagged_marker.alias_connection_params_stay_with_tier, module: reliability, tier: P0, behavior: routing, variant: tagged_marker, assertions: [alias_connection_params_stay_with_tier], exercised_on: [chat_completions], source: "litellm/router.py:11567", rationale: "An api_key or api_base on the marker alias is never forwarded onto the routed request; the tier deployment calls its provider with its own credential (GitHub PR #36626)"} +- {id: reliability.routing.semantic_auto_router.responses_input_routed, module: reliability, tier: P0, behavior: routing, variant: semantic_auto_router, assertions: [responses_input_routed], exercised_on: [responses], source: "litellm/router_strategy/auto_router/auto_router.py:131", fail_before_fix: proven, rationale: "/v1/responses input is resolved into messages for the semantic auto-router pre-routing hook instead of failing 400 Unmapped LLM provider auto_router (GitHub PR #37333)"} +- {id: reliability.routing.strategy_alias.custom_pricing_ignored, module: reliability, tier: P1, behavior: routing, variant: strategy_alias, assertions: [custom_pricing_ignored], exercised_on: [chat_completions], source: "litellm/router.py:11489", rationale: "Custom pricing on a strategy-router alias never prices the routed request; spend logs at the routed tier deployment's own rate (GitHub PR #36691)"} +- {id: reliability.routing.complexity_heuristic.scores_current_ask_only, module: reliability, tier: P1, behavior: routing, variant: complexity_heuristic, assertions: [scores_current_ask_only], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py:942", rationale: "The heuristic complexity classifier scores the caller's current ask only, so a keyword-heavy agent system prompt cannot inflate the tier (GitHub PR #36721)"} - {id: reliability.cache.exact.returns_cached, module: reliability, tier: P1, behavior: cache, variant: exact, assertions: [returns_cached], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/caching.py", rationale: "Response cache returns cached on exact match"} - {id: reliability.cache.prompt_caching_model_select.returns_cached, module: reliability, tier: P1, behavior: cache, variant: prompt_caching_model_select, assertions: [returns_cached], exercised_on: [chat_completions], source: "router_utils/prompt_caching_cache.py", rationale: "Selects model supporting prompt caching for cacheable prefix"} - {id: reliability.circuit_breaker.redis.trips_then_recovers, module: reliability, tier: P0, behavior: circuit_breaker, variant: redis, assertions: [trips_then_recovers], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/redis_cache.py:99", rationale: "Redis breaker CLOSED->OPEN->HALF_OPEN; guards all cache/rate-limit ops"} diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py index f4db88b1e19..cb6fc7a01e5 100644 --- a/tests/e2e/e2e_http.py +++ b/tests/e2e/e2e_http.py @@ -75,6 +75,8 @@ class NetworkError(BaseModel): class UnauthorizedError(BaseModel): kind: Literal["unauthorized"] = "unauthorized" + # litellm 401s for key auth, model access, and tag routing alike, so keep the body to tell them apart. + body: str = "" class RateLimitedError(BaseModel): @@ -289,7 +291,7 @@ def _classify[R: BaseModel]( resp: requests.Response, response_type: type[R] ) -> Result[R]: if resp.status_code == 401: - return UnauthorizedError() + return UnauthorizedError(body=resp.text) if resp.status_code == 429: return RateLimitedError(body=resp.text) if not resp.ok: diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 734e63a94e6..df5cb841fad 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -73,6 +73,7 @@ class KeyGenerateBody(BaseModel): allowed_passthrough_routes: list[str] | None = None metadata: KeyMetadata | None = None object_permission: ObjectPermission | None = None + router_settings: "RouterSettingsOverride | None" = None class KeyGenerateResponse(BaseModel): @@ -234,16 +235,18 @@ class ChatBody(BaseModel): class RouterSettingsOverride(BaseModel): - """Per-request `router_settings_override` in a /chat/completions body: the - reliability knobs (fallbacks by trigger, retry count) the reliability suite - drives per call instead of via static router config. Serialized exclude_none, so - an override sets only the strategies a test exercises. Each fallbacks map is - model_name -> the ordered fallback model_names to try.""" + """Router settings a test scopes below the global config: sent per request as + `router_settings_override` in a /chat/completions body (the reliability suite's + fallback and retry knobs) or stored on a key as `router_settings` at + /key/generate (the auto-router suite's tag filtering switch). Serialized + exclude_none, so an override sets only the knobs a test exercises. Each + fallbacks map is model_name -> the ordered fallback model_names to try.""" fallbacks: list[dict[str, list[str]]] | None = None context_window_fallbacks: list[dict[str, list[str]]] | None = None content_policy_fallbacks: list[dict[str, list[str]]] | None = None num_retries: int | None = None + enable_tag_filtering: bool | None = None class ReliabilityChatBody(ChatBody): @@ -744,6 +747,10 @@ class LiteLLMParamsBody(BaseModel): extra_headers: dict[str, str] | None = None use_in_pass_through: bool | None = None complexity_router_config: dict[str, object] | None = None + auto_router_config: str | None = None + auto_router_default_model: str | None = None + auto_router_embedding_model: str | None = None + tags: list[str] | None = None mock_response: str | None = None timeout: float | None = None tpm: int | None = None diff --git a/tests/e2e/router/test_auto_router_regressions_e2e.py b/tests/e2e/router/test_auto_router_regressions_e2e.py new file mode 100644 index 00000000000..35ba2c8d3d1 --- /dev/null +++ b/tests/e2e/router/test_auto_router_regressions_e2e.py @@ -0,0 +1,620 @@ +"""Live e2e regression pins for strategy-router (auto-router) routing. + +A strategy marker (an ``auto_router/complexity_router`` deployment) and a plain +deployment can share one ``model_name``, split by tags once +``enable_tag_filtering`` is on: tagged requests route through the marker to its +tier models, untagged requests go to the plain deployment. That split, and the +strategy-router alias behaviors around it, regressed repeatedly; each test here +pins one fixed behavior: + +- GitHub issue #36619: a tagged request selects the tagged marker under a + shared name even when a plain deployment was registered first. +- GitHub issue #36620: untagged requests keep being served by the plain + deployment on every call, never captured or 400'd by the tagged marker. +- GitHub issue #36621: a request tagged via the ``x-litellm-tags`` header + routes through the marker even when the tier deployments carry no tags + (the marker consumes the routing tags before deployment selection), while a + tagged call aimed straight at an untagged deployment stays denied. +- GitHub issues #36620/#36621 on /v1/responses: the same tag split holds for + string and list input, whether the tag arrives in litellm_metadata or the + x-litellm-tags header. +- GitHub PR #37333: /v1/responses input is resolved into messages for a + semantic ``auto_router`` deployment's pre-routing hook; such requests used + to fail with 400 "Unmapped LLM provider auto_router" because only chat + messages fed the route matcher. +- GitHub PR #36691: custom pricing on the marker alias never prices the routed + request; spend logs at the routed tier deployment's own rate. +- GitHub PR #36721: the heuristic complexity classifier scores the caller's + current ask only, so a large agent system prompt cannot inflate the tier. +- GitHub PR #36626: connection params on the marker alias (``api_key``, + ``api_base``) stay with the alias; the routed tier calls its provider with + its own credentials. + +Every deployment is registered via /model/new (stage has no static config for +these) and ``enable_tag_filtering`` is enabled through key-level +``router_settings`` on the keys the tag tests mint, so the switch rides only +this module's own requests and the rest of the suite is never filtered. +The served deployment is always read back from the spend log's ``model``, +which stores either the registered alias or the provider-prefixed form. +""" + +import json +import os +from collections.abc import Iterator +from dataclasses import dataclass +from typing import Final + +import pytest +from pydantic import BaseModel, ConfigDict, Field + +from e2e_config import unique_marker +from e2e_http import AnthropicHeaders, AuthHeaders, UnauthorizedError, unwrap +from lifecycle import ResourceManager +from models import ( + AnthropicMessagesBody, + AnthropicMessagesResponse, + ChatBody, + ChatMessage, + ChatMetadata, + KeyGenerateBody, + LiteLLMParamsBody, + RouterSettingsOverride, + SpendLogRow, +) +from proxy_client import ProxyClient + +pytestmark = pytest.mark.e2e + +PLAIN_MODEL = "anthropic/claude-sonnet-5" +CHEAP_MODEL = "anthropic/claude-haiku-4-5" +STRONG_MODEL = "openai/gpt-5.6" +MAX_TOKENS = 16 +TAG_DENIAL_MESSAGE = "Not allowed to access model due to tags configuration" +PLAIN_SERVED = frozenset({PLAIN_MODEL, "claude-sonnet-5"}) +CHEAP_SERVED = frozenset({CHEAP_MODEL, "claude-haiku-4-5"}) +EMBEDDING_MODEL = "openai/text-embedding-3-small" +SEMANTIC_ROUTE_UTTERANCE = "summarize this quarterly revenue report into three bullet points" + +KEYWORD_HEAVY_SYSTEM_PROMPT = ( + "You are the principal architecture assistant for a distributed systems platform. " + "Analyze every request step by step: design the algorithm, prove its correctness, " + "evaluate time and space complexity, and reason about concurrency, consistency, and " + "fault tolerance tradeoffs. When asked, refactor and debug multi-threaded code, " + "optimize database query plans, derive mathematical proofs, and explain the theorem " + "or lemma behind each optimization. Think through edge cases rigorously before answering. " +) * 4 + + +class TaggedAuthHeaders(AuthHeaders): + x_litellm_tags: str | None = Field(default=None, serialization_alias="x-litellm-tags") + + +class TaggedAnthropicHeaders(AnthropicHeaders): + x_litellm_tags: str | None = Field(default=None, serialization_alias="x-litellm-tags") + + +class ResponsesTagMetadata(BaseModel): + tags: list[str] + + +class ResponsesInputItem(BaseModel): + role: str + content: str + + +class ResponsesBody(BaseModel): + model: str + input: str | list[ResponsesInputItem] + max_output_tokens: int | None = None + litellm_metadata: ResponsesTagMetadata | None = None + + +class ResponsesApiResponse(BaseModel): + """Minimal /v1/responses answer shape; routing is proven from spend logs, + so only the fields the assertions read are modeled.""" + + model_config = ConfigDict(extra="allow") + id: str | None = None + status: str | None = None + model: str | None = None + + +@dataclass(frozen=True, slots=True) +class TagSplitDeployments: + """Scenario A mirrors the customer-shaped config from GitHub issue #36619: + plain deployment registered first, tier deployment and marker both tagged. + Scenario B flips both axes for GitHub issue #36621: marker registered first + and its tier deployment left untagged, so routing depends neither on + registration order nor on tier deployments carrying tags.""" + + tag_a: str + shared_a: str + tier_a: str + tag_b: str + shared_b: str + tier_b: str + + +@dataclass(frozen=True, slots=True) +class ZeroPricedAlias: + alias: str + tier: str + + +@dataclass(frozen=True, slots=True) +class HeuristicSplit: + alias: str + cheap: str + strong: str + + +@dataclass(frozen=True, slots=True) +class SemanticAutoRouter: + marker: str + target: str + fallback: str + embedding: str + + +@dataclass(frozen=True, slots=True) +class CredentialedAlias: + alias: str + tier: str + + +def _provider_key(env_var: str) -> str: + return os.environ.get(env_var) or f"os.environ/{env_var}" + + +def _uniform_tier_config(tier_model: str) -> dict[str, object]: + return { + "classifier_type": "heuristic", + "tiers": {"SIMPLE": tier_model, "MEDIUM": tier_model, "COMPLEX": tier_model, "REASONING": tier_model}, + } + + +def _key_for( + proxy: ProxyClient, resources: ResourceManager, models: list[str], tag_filtering: bool = False +) -> str: + key: Final = proxy.generate_key( + KeyGenerateBody( + models=models, + user_id="e2e-auto-router-regressions", + router_settings=RouterSettingsOverride(enable_tag_filtering=True) if tag_filtering else None, + ) + ) + resources.defer(lambda: proxy.delete_key(key)) + return key + + +def _hello_chat_body(model: str, tags: list[str] | None = None) -> ChatBody: + return ChatBody( + model=model, + messages=[ChatMessage(role="user", content=f"say hello {unique_marker()}")], + max_tokens=MAX_TOKENS, + metadata=ChatMetadata(tags=tags) if tags is not None else None, + ) + + +def _hello_messages_body(model: str) -> AnthropicMessagesBody: + return AnthropicMessagesBody( + model=model, + messages=[ChatMessage(role="user", content=f"say hello {unique_marker()}")], + max_tokens=MAX_TOKENS, + ) + + +def _assert_served_only_by(rows: list[SpendLogRow], allowed: frozenset[str], context: str) -> None: + served: Final = tuple(row.model for row in rows) + assert served and all(model in allowed for model in served), ( + f"{context}: expected every request to be served by one of {sorted(allowed)}, spend logs show {served}" + ) + + +@pytest.fixture(scope="module") +def split(proxy: ProxyClient) -> Iterator[TagSplitDeployments]: + marker: Final = unique_marker() + deployments: Final = TagSplitDeployments( + tag_a=f"e2e-split-a-{marker}", + shared_a=f"e2e-autoroute-a-{marker}", + tier_a=f"e2e-tier-a-{marker}", + tag_b=f"e2e-split-b-{marker}", + shared_b=f"e2e-autoroute-b-{marker}", + tier_b=f"e2e-tier-b-{marker}", + ) + anthropic_key: Final = _provider_key("ANTHROPIC_API_KEY") + marker_params_a: Final = LiteLLMParamsBody( + model="auto_router/complexity_router", + complexity_router_config=_uniform_tier_config(deployments.tier_a), + tags=[deployments.tag_a], + ) + marker_params_b: Final = LiteLLMParamsBody( + model="auto_router/complexity_router", + complexity_router_config=_uniform_tier_config(deployments.tier_b), + tags=[deployments.tag_b], + ) + registrations: Final[tuple[tuple[str, LiteLLMParamsBody], ...]] = ( + (deployments.shared_a, LiteLLMParamsBody(model=PLAIN_MODEL, api_key=anthropic_key)), + (deployments.tier_a, LiteLLMParamsBody(model=CHEAP_MODEL, api_key=anthropic_key, tags=[deployments.tag_a])), + (deployments.shared_a, marker_params_a), + (deployments.shared_b, marker_params_b), + (deployments.tier_b, LiteLLMParamsBody(model=CHEAP_MODEL, api_key=anthropic_key)), + (deployments.shared_b, LiteLLMParamsBody(model=PLAIN_MODEL, api_key=anthropic_key)), + ) + created: Final = tuple(proxy.create_model(name, params) for name, params in registrations) + try: + yield deployments + finally: + for model_id in created: + proxy.delete_model(model_id) + + +@pytest.fixture(scope="module") +def zero_priced_alias(proxy: ProxyClient) -> Iterator[ZeroPricedAlias]: + marker: Final = unique_marker() + named: Final = ZeroPricedAlias(alias=f"e2e-priced-alias-{marker}", tier=f"e2e-priced-tier-{marker}") + alias_params: Final = LiteLLMParamsBody( + model="auto_router/complexity_router", + complexity_router_config=_uniform_tier_config(named.tier), + input_cost_per_token=0.0, + output_cost_per_token=0.0, + ) + registrations: Final[tuple[tuple[str, LiteLLMParamsBody], ...]] = ( + (named.tier, LiteLLMParamsBody(model=CHEAP_MODEL, api_key=_provider_key("ANTHROPIC_API_KEY"))), + (named.alias, alias_params), + ) + created: Final = tuple(proxy.create_model(name, params) for name, params in registrations) + try: + yield named + finally: + for model_id in created: + proxy.delete_model(model_id) + + +@pytest.fixture(scope="module") +def heuristic_split(proxy: ProxyClient) -> Iterator[HeuristicSplit]: + marker: Final = unique_marker() + named: Final = HeuristicSplit( + alias=f"e2e-heuristic-router-{marker}", + cheap=f"e2e-heuristic-cheap-{marker}", + strong=f"e2e-heuristic-strong-{marker}", + ) + config: Final[dict[str, object]] = { + "classifier_type": "heuristic", + "token_thresholds": {"simple": 15, "complex": 400}, + "tiers": {"SIMPLE": named.cheap, "MEDIUM": named.strong, "COMPLEX": named.strong, "REASONING": named.strong}, + } + registrations: Final[tuple[tuple[str, LiteLLMParamsBody], ...]] = ( + (named.cheap, LiteLLMParamsBody(model=CHEAP_MODEL, api_key=_provider_key("ANTHROPIC_API_KEY"))), + (named.strong, LiteLLMParamsBody(model=STRONG_MODEL, api_key=_provider_key("OPENAI_API_KEY"))), + (named.alias, LiteLLMParamsBody(model="auto_router/complexity_router", complexity_router_config=config)), + ) + created: Final = tuple(proxy.create_model(name, params) for name, params in registrations) + try: + yield named + finally: + for model_id in created: + proxy.delete_model(model_id) + + +@pytest.fixture(scope="module") +def semantic_auto_router(proxy: ProxyClient) -> Iterator[SemanticAutoRouter]: + marker: Final = unique_marker() + named: Final = SemanticAutoRouter( + marker=f"e2e-semantic-router-{marker}", + target=f"e2e-semantic-target-{marker}", + fallback=f"e2e-semantic-fallback-{marker}", + embedding=f"e2e-semantic-embedding-{marker}", + ) + router_config: Final = json.dumps( + {"routes": [{"name": named.target, "utterances": [SEMANTIC_ROUTE_UTTERANCE], "score_threshold": 0.3}]} + ) + marker_params: Final = LiteLLMParamsBody( + model=f"auto_router/{named.marker}", + auto_router_config=router_config, + auto_router_default_model=named.fallback, + auto_router_embedding_model=named.embedding, + ) + registrations: Final[tuple[tuple[str, LiteLLMParamsBody], ...]] = ( + (named.embedding, LiteLLMParamsBody(model=EMBEDDING_MODEL, api_key=_provider_key("OPENAI_API_KEY"))), + (named.target, LiteLLMParamsBody(model=CHEAP_MODEL, api_key=_provider_key("ANTHROPIC_API_KEY"))), + (named.fallback, LiteLLMParamsBody(model=PLAIN_MODEL, api_key=_provider_key("ANTHROPIC_API_KEY"))), + (named.marker, marker_params), + ) + created: Final = tuple(proxy.create_model(name, params) for name, params in registrations) + try: + yield named + finally: + for model_id in created: + proxy.delete_model(model_id) + + +@pytest.fixture(scope="module") +def credentialed_alias(proxy: ProxyClient) -> Iterator[CredentialedAlias]: + marker: Final = unique_marker() + named: Final = CredentialedAlias(alias=f"e2e-cred-alias-{marker}", tier=f"e2e-cred-tier-{marker}") + alias_params: Final = LiteLLMParamsBody( + model="auto_router/complexity_router", + complexity_router_config=_uniform_tier_config(named.tier), + api_key=f"sk-alias-never-used-{marker}", + ) + registrations: Final[tuple[tuple[str, LiteLLMParamsBody], ...]] = ( + (named.tier, LiteLLMParamsBody(model=CHEAP_MODEL, api_key=_provider_key("ANTHROPIC_API_KEY"))), + (named.alias, alias_params), + ) + created: Final = tuple(proxy.create_model(name, params) for name, params in registrations) + try: + yield named + finally: + for model_id in created: + proxy.delete_model(model_id) + + +class TestTagSplitRouting: + @pytest.mark.covers("reliability.routing.tagged_marker.request_tag_selects_marker") + def test_body_tagged_chat_routes_through_the_marker_to_its_tier( + self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments + ) -> None: + """Pins GitHub issue #36619: with tag filtering on, a chat request whose + body metadata tags match the tagged marker under a shared model name is + answered by the marker's tier deployment, not by the plain deployment + that was registered under the name first.""" + key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True) + chat: Final = unwrap(proxy.chat(key, _hello_chat_body(split.shared_a, tags=[split.tag_a]))) + assert chat.choices, "tagged chat through the shared name returned no choices" + rows: Final = proxy.poll_logs_for_key(key, min_rows=1) + _assert_served_only_by(rows, CHEAP_SERVED | {split.tier_a}, "body-tagged chat on the shared name") + + @pytest.mark.covers("reliability.routing.tagged_marker.untagged_request_served_by_plain_deployment") + def test_untagged_chat_is_always_served_by_the_plain_deployment( + self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments + ) -> None: + """Pins GitHub issue #36620: untagged chat requests to the shared name + succeed on every call and are all served by the plain deployment; the + tagged marker never captures them, so no intermittent auto-router + errors and no tier hijacking.""" + key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True) + for _ in range(5): + chat = unwrap(proxy.chat(key, _hello_chat_body(split.shared_a))) + assert chat.choices, "untagged chat through the shared name returned no choices" + rows: Final = proxy.poll_logs_for_key(key, min_rows=5) + _assert_served_only_by(rows, PLAIN_SERVED | {split.shared_a}, "untagged chat on the shared name") + + @pytest.mark.covers("reliability.routing.tagged_marker.untagged_request_served_by_plain_deployment") + def test_untagged_messages_is_served_by_the_plain_deployment( + self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments + ) -> None: + """Pins GitHub issue #36620 on the /v1/messages surface: an untagged + Anthropic-native request to the shared name is served by the plain + deployment, not captured by the tagged marker.""" + key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True) + answer: Final = unwrap(proxy.messages(key, _hello_messages_body(split.shared_a))) + assert answer.content or answer.choices, "untagged /v1/messages returned neither content nor choices" + rows: Final = proxy.poll_logs_for_key(key, min_rows=1) + _assert_served_only_by(rows, PLAIN_SERVED | {split.shared_a}, "untagged /v1/messages on the shared name") + + +class TestUntaggedTierDeployments: + @pytest.mark.covers("reliability.routing.tagged_marker.header_tag_selects_marker") + def test_header_tagged_messages_routes_through_the_marker_to_an_untagged_tier( + self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments + ) -> None: + """Pins GitHub issue #36621: a /v1/messages request tagged only via the + x-litellm-tags header selects the tagged marker, and the rewrite still + lands on the tier deployment even though that deployment carries no + tags, because the marker consumed the routing tags.""" + key: Final = _key_for(proxy, resources, [split.shared_b, split.tier_b], tag_filtering=True) + headers: Final = TaggedAnthropicHeaders(authorization=f"Bearer {key}", x_litellm_tags=split.tag_b) + answer: Final = unwrap( + proxy.transport.post( + "/v1/messages", + headers=headers, + json=_hello_messages_body(split.shared_b), + response_type=AnthropicMessagesResponse, + ) + ) + assert answer.content or answer.choices, "header-tagged /v1/messages returned neither content nor choices" + rows: Final = proxy.poll_logs_for_key(key, min_rows=1) + _assert_served_only_by(rows, CHEAP_SERVED | {split.tier_b}, "header-tagged /v1/messages on the shared name") + + @pytest.mark.covers("reliability.routing.tagged_marker.untagged_tier_deployments_still_served") + def test_body_tagged_chat_reaches_the_untagged_tier_after_marker_rewrite( + self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments + ) -> None: + """Pins the tag-consumption half of GitHub issue #36621: after the + tagged marker rewrites the request to its tier model, the consumed + routing tags no longer constrain deployment selection, so the untagged + tier deployment serves the request instead of a strict-tag denial.""" + key: Final = _key_for(proxy, resources, [split.shared_b, split.tier_b], tag_filtering=True) + chat: Final = unwrap(proxy.chat(key, _hello_chat_body(split.shared_b, tags=[split.tag_b]))) + assert chat.choices, "body-tagged chat through the marker-first shared name returned no choices" + rows: Final = proxy.poll_logs_for_key(key, min_rows=1) + _assert_served_only_by(rows, CHEAP_SERVED | {split.tier_b}, "body-tagged chat with untagged tier") + + @pytest.mark.covers("reliability.routing.tagged_marker.tag_semantics_stay_strict") + def test_tagged_call_straight_at_an_untagged_deployment_stays_denied( + self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments + ) -> None: + """The tag-consumption fix must not loosen strict tag semantics: a + tagged request aimed directly at an untagged deployment (no marker + involved) is still rejected with the 401 tags-configuration error.""" + key: Final = _key_for(proxy, resources, [split.tier_b], tag_filtering=True) + result: Final = proxy.chat(key, _hello_chat_body(split.tier_b, tags=[split.tag_b])) + assert isinstance(result, UnauthorizedError), ( + f"expected the tagged direct call to an untagged deployment to be denied with 401, got {result}" + ) + assert TAG_DENIAL_MESSAGE in result.body, ( + f"expected the denial to come from tag routing, got a 401 reading {result.body[:300]}" + ) + + +class TestResponsesApiTagRouting: + @pytest.mark.covers("reliability.routing.tagged_marker.responses_input_routes_through_marker") + def test_header_tagged_responses_with_string_input_routes_to_the_tier( + self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments + ) -> None: + """Pins the /v1/responses surface of the tag split (GitHub issues + #36620/#36621): a /v1/responses request with string input, tagged via + the x-litellm-tags header, succeeds and routes through the tagged + marker to its tier.""" + key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True) + headers: Final = TaggedAuthHeaders(authorization=f"Bearer {key}", x_litellm_tags=split.tag_a) + body: Final = ResponsesBody( + model=split.shared_a, input=f"say hello {unique_marker()}", max_output_tokens=64 + ) + answer: Final = unwrap( + proxy.transport.post("/v1/responses", headers=headers, json=body, response_type=ResponsesApiResponse) + ) + assert answer.id, "header-tagged /v1/responses returned no response id" + rows: Final = proxy.poll_logs_for_key(key, min_rows=1) + _assert_served_only_by(rows, CHEAP_SERVED | {split.tier_a}, "header-tagged /v1/responses string input") + + @pytest.mark.covers("reliability.routing.tagged_marker.responses_input_routes_through_marker") + def test_body_tagged_responses_with_list_input_routes_to_the_tier( + self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments + ) -> None: + """Pins the body-tag and list-input combination of the same split: + /v1/responses with litellm_metadata.tags and structured input items + routes through the tagged marker to its tier.""" + key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True) + body: Final = ResponsesBody( + model=split.shared_a, + input=[ResponsesInputItem(role="user", content=f"say hello {unique_marker()}")], + max_output_tokens=64, + litellm_metadata=ResponsesTagMetadata(tags=[split.tag_a]), + ) + answer: Final = unwrap( + proxy.transport.post( + "/v1/responses", + headers=proxy.transport.bearer(key), + json=body, + response_type=ResponsesApiResponse, + ) + ) + assert answer.id, "body-tagged /v1/responses returned no response id" + rows: Final = proxy.poll_logs_for_key(key, min_rows=1) + _assert_served_only_by(rows, CHEAP_SERVED | {split.tier_a}, "body-tagged /v1/responses list input") + + @pytest.mark.covers("reliability.routing.tagged_marker.untagged_request_served_by_plain_deployment") + def test_untagged_responses_is_served_by_the_plain_deployment( + self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments + ) -> None: + """Pins the untagged half of the /v1/responses tag split: an untagged + request to the shared name is served by the plain deployment, matching + the chat and messages surfaces.""" + key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True) + body: Final = ResponsesBody( + model=split.shared_a, input=f"say hello {unique_marker()}", max_output_tokens=64 + ) + answer: Final = unwrap( + proxy.transport.post( + "/v1/responses", + headers=proxy.transport.bearer(key), + json=body, + response_type=ResponsesApiResponse, + ) + ) + assert answer.id, "untagged /v1/responses returned no response id" + rows: Final = proxy.poll_logs_for_key(key, min_rows=1) + _assert_served_only_by(rows, PLAIN_SERVED | {split.shared_a}, "untagged /v1/responses on the shared name") + + +class TestStrategyAliasPricing: + @pytest.mark.covers("reliability.routing.strategy_alias.custom_pricing_ignored") + def test_zero_priced_alias_still_logs_spend_at_the_tier_rate( + self, proxy: ProxyClient, resources: ResourceManager, zero_priced_alias: ZeroPricedAlias + ) -> None: + """Pins GitHub PR #36691: custom pricing registered on a strategy-router + alias never prices the routed request. The alias here carries explicit + zero pricing, so any zero-spend row would prove the alias pricing was + applied; the routed tier deployment's real rate must produce spend > 0.""" + key: Final = _key_for(proxy, resources, [zero_priced_alias.alias, zero_priced_alias.tier]) + chat: Final = unwrap(proxy.chat(key, _hello_chat_body(zero_priced_alias.alias))) + assert chat.choices, "chat through the zero-priced alias returned no choices" + rows: Final = proxy.poll_logs_for_key( + key, min_rows=1, predicate=lambda logged: all((row.spend or 0.0) > 0.0 for row in logged) + ) + _assert_served_only_by(rows, CHEAP_SERVED | {zero_priced_alias.tier}, "chat through the zero-priced alias") + priced: Final = tuple((row.model, row.spend) for row in rows) + assert all((row.spend or 0.0) > 0.0 for row in rows), ( + f"expected spend at the tier deployment's own rate, got zero-spend rows: {priced}" + ) + + +class TestComplexityHeuristicScope: + @pytest.mark.covers("reliability.routing.complexity_heuristic.scores_current_ask_only") + def test_trivial_ask_behind_keyword_heavy_system_prompt_stays_on_the_cheap_tier( + self, proxy: ProxyClient, resources: ResourceManager, heuristic_split: HeuristicSplit + ) -> None: + """Pins GitHub PR #36721: the heuristic complexity classifier scores the + caller's current ask alone. The trivial ask scores SIMPLE on its own, + while the accompanying ~2KB agent system prompt is packed with enough + reasoning and complexity keywords that scoring the combined text lands + in REASONING; only ask-only scoring keeps this on the cheap tier.""" + key: Final = _key_for( + proxy, resources, [heuristic_split.alias, heuristic_split.cheap, heuristic_split.strong] + ) + body: Final = ChatBody( + model=heuristic_split.alias, + messages=[ + ChatMessage(role="system", content=KEYWORD_HEAVY_SYSTEM_PROMPT), + ChatMessage(role="user", content=f"hi {unique_marker()}"), + ], + max_tokens=MAX_TOKENS, + ) + chat: Final = unwrap(proxy.chat(key, body)) + assert chat.choices, "chat through the heuristic router returned no choices" + rows: Final = proxy.poll_logs_for_key(key, min_rows=1) + _assert_served_only_by( + rows, CHEAP_SERVED | {heuristic_split.cheap}, "trivial ask behind a keyword-heavy system prompt" + ) + + +class TestSemanticAutoRouterResponses: + @pytest.mark.covers("reliability.routing.semantic_auto_router.responses_input_routed") + def test_responses_input_reaches_the_semantic_auto_router( + self, proxy: ProxyClient, resources: ResourceManager, semantic_auto_router: SemanticAutoRouter + ) -> None: + """Pins GitHub PR #37333: /v1/responses input is resolved into messages + for the semantic auto-router's pre-routing hook, so the marker embeds + the input, matches its route, and the target deployment serves the + request; before the fix the hook saw no messages and the request + failed with 400 "Unmapped LLM provider auto_router".""" + key: Final = _key_for( + proxy, + resources, + [semantic_auto_router.marker, semantic_auto_router.target, semantic_auto_router.fallback], + ) + body: Final = ResponsesBody( + model=semantic_auto_router.marker, input=SEMANTIC_ROUTE_UTTERANCE, max_output_tokens=64 + ) + answer: Final = unwrap( + proxy.transport.post( + "/v1/responses", + headers=proxy.transport.bearer(key), + json=body, + response_type=ResponsesApiResponse, + ) + ) + assert answer.id, "/v1/responses through the semantic auto-router returned no response id" + rows: Final = proxy.poll_logs_for_key(key, min_rows=1) + _assert_served_only_by( + rows, CHEAP_SERVED | {semantic_auto_router.target}, "semantic auto-router /v1/responses string input" + ) + + +class TestAliasParamForwarding: + @pytest.mark.covers("reliability.routing.tagged_marker.alias_connection_params_stay_with_tier") + def test_alias_api_key_never_overrides_the_tier_credential( + self, proxy: ProxyClient, resources: ResourceManager, credentialed_alias: CredentialedAlias + ) -> None: + """Pins GitHub PR #36626: an api_key set on the marker alias entry is + never forwarded onto the routed request, so the tier deployment calls + its provider with its own credential. Before the fix the alias's key + was copied into the request, overriding the tier's credential, and + every routed call failed provider auth.""" + key: Final = _key_for(proxy, resources, [credentialed_alias.alias, credentialed_alias.tier]) + chat: Final = unwrap(proxy.chat(key, _hello_chat_body(credentialed_alias.alias))) + assert chat.choices, "chat through the credentialed alias returned no choices" + rows: Final = proxy.poll_logs_for_key(key, min_rows=1) + _assert_served_only_by(rows, CHEAP_SERVED | {credentialed_alias.tier}, "chat through the credentialed alias") diff --git a/tests/e2e/ui/tests/mcp/mcpTools.spec.ts b/tests/e2e/ui/tests/mcp/mcpTools.spec.ts index edaeab196aa..225ca8b9449 100644 --- a/tests/e2e/ui/tests/mcp/mcpTools.spec.ts +++ b/tests/e2e/ui/tests/mcp/mcpTools.spec.ts @@ -63,7 +63,7 @@ test.describe("MCP Tools", () => { // The form is generated from the tool's inputSchema, so `repoName` proves the schema // round-tripped through the proxy instead of the panel falling back to a generic field. - const repoInput = page.locator('input[id="repoName"]'); + const repoInput = page.getByLabel(/repoName/); await expect(repoInput).toBeVisible(); await repoInput.fill(TOOL_ARG_REPO); diff --git a/tests/e2e/ui/tests/modelsPage/addModel.spec.ts b/tests/e2e/ui/tests/modelsPage/addModel.spec.ts index 1b11ea69f97..bd5373e1569 100644 --- a/tests/e2e/ui/tests/modelsPage/addModel.spec.ts +++ b/tests/e2e/ui/tests/modelsPage/addModel.spec.ts @@ -338,8 +338,8 @@ test.describe("Add Model", () => { await page.getByRole("button", { name: "Add Model" }).last().click(); - // Scope to antd's notification container so a stale toast can't satisfy this. - await expect(page.locator(".ant-notification").getByText("created successfully").last()).toBeVisible({ + // Scope to the toast container so a stale toast can't satisfy this. + await expect(page.locator("[data-sonner-toast]").getByText("created successfully").last()).toBeVisible({ timeout: 15_000, }); diff --git a/tests/e2e/ui/tests/modelsPage/clearCustomPricing.spec.ts b/tests/e2e/ui/tests/modelsPage/clearCustomPricing.spec.ts index e67dcb96f36..c532641b238 100644 --- a/tests/e2e/ui/tests/modelsPage/clearCustomPricing.spec.ts +++ b/tests/e2e/ui/tests/modelsPage/clearCustomPricing.spec.ts @@ -85,9 +85,9 @@ test.describe("Clear custom pricing on a deployment", () => { const inputCost = page.getByPlaceholder("Enter input cost"); const outputCost = page.getByPlaceholder("Enter output cost"); // Both cache fields share the same placeholder ("Defaults to Input Cost if blank"), - // so disambiguate via the Form.Item id (AntD assigns the `name` prop as input id). - const cacheReadCost = page.locator("#cache_read_cost"); - const cacheWriteCost = page.locator("#cache_write_cost"); + // so disambiguate via their labels. + const cacheReadCost = page.getByLabel(/Cache Read Cost/); + const cacheWriteCost = page.getByLabel(/Cache Write Cost/); await inputCost.waitFor({ timeout: 15_000 }); for (const field of [inputCost, outputCost, cacheReadCost, cacheWriteCost]) { await field.click({ clickCount: 3 }); diff --git a/tests/e2e/ui/tests/proxy-admin/keys.spec.ts b/tests/e2e/ui/tests/proxy-admin/keys.spec.ts index d9b0f959c9f..004fedb3263 100644 --- a/tests/e2e/ui/tests/proxy-admin/keys.spec.ts +++ b/tests/e2e/ui/tests/proxy-admin/keys.spec.ts @@ -36,9 +36,8 @@ test.describe("Proxy Admin - Keys", () => { // Wait for the key creation modal await expect(page.getByText("Key Ownership")).toBeVisible({ timeout: 10_000 }); - // Fill key name (has data-testid="base-input" in the built UI) const keyName = `e2e-admin-key-${Date.now()}`; - await page.getByTestId("base-input").fill(keyName); + await page.getByLabel(/Key Name/).fill(keyName); // Select team const teamSelect = page.getByTestId("team-dropdown").getByRole("combobox"); @@ -192,7 +191,7 @@ test.describe("Proxy Admin - Keys", () => { await expect(page.getByText("Key Ownership")).toBeVisible({ timeout: 10_000 }); const keyName = `e2e-admin-allproxy-${Date.now()}`; - await page.getByTestId("base-input").fill(keyName); + await page.getByLabel(/Key Name/).fill(keyName); // No team selection — leave team dropdown empty so the key is owned by the admin user @@ -220,7 +219,7 @@ test.describe("Proxy Admin - Keys", () => { await expect(page.getByText("Key Ownership")).toBeVisible({ timeout: 10_000 }); const keyName = `e2e-admin-specific-${Date.now()}`; - await page.getByTestId("base-input").fill(keyName); + await page.getByLabel(/Key Name/).fill(keyName); // Open the model multi-select and pick a single specific model. Use // getByRole("option", ...) to avoid the strict-mode collision between diff --git a/tests/e2e/ui/tests/proxy-admin/teams.spec.ts b/tests/e2e/ui/tests/proxy-admin/teams.spec.ts index d7c8eb6237e..172fde173df 100644 --- a/tests/e2e/ui/tests/proxy-admin/teams.spec.ts +++ b/tests/e2e/ui/tests/proxy-admin/teams.spec.ts @@ -79,7 +79,7 @@ test.describe("Proxy Admin - Teams", () => { await expect(modal).toBeVisible({ timeout: 5_000 }); // The email field is a Select — type to search, then select from dropdown - await modal.locator(".ant-select").first().click(); + await modal.getByRole("combobox").first().click(); await page.keyboard.type("invitable@test.local"); // Wait for the option to appear, then select via keyboard (avoids viewport issues) diff --git a/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts b/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts index d71d5e6c0fe..0ef74e71529 100644 --- a/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts +++ b/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts @@ -66,7 +66,7 @@ test.describe("Team Admin", () => { // Use a dedicated invitee user so this doesn't race with the proxy-admin // "Invite a user" test that adds invitable@test.local to the same team. - await modal.locator(".ant-select").first().click(); + await modal.getByRole("combobox").first().click(); await page.keyboard.type("invitable-team@test.local"); const emailOption = page.getByRole("option", { name: "invitable-team@test.local" }).first(); @@ -136,7 +136,7 @@ test.describe("Team Admin", () => { await expect(page.getByText("Key Ownership")).toBeVisible({ timeout: 10_000 }); const keyName = `e2e-team-admin-key-${Date.now()}`; - await page.getByTestId("base-input").fill(keyName); + await page.getByLabel(/Key Name/).fill(keyName); // Team selector — same locator pattern as the proxy-admin keys test. const teamSelect = page.getByTestId("team-dropdown").getByRole("combobox"); diff --git a/tests/image_gen_tests/test_image_generation.py b/tests/image_gen_tests/test_image_generation.py index f0a73325afa..33dcdbb57a5 100644 --- a/tests/image_gen_tests/test_image_generation.py +++ b/tests/image_gen_tests/test_image_generation.py @@ -269,7 +269,7 @@ class TestAimlImageGeneration(BaseImageGenTest): class TestGoogleImageGen(BaseImageGenTest): def get_base_image_generation_call_args(self) -> dict: - return {"model": "gemini/imagen-4.0-generate-001"} + return {"model": "gemini/gemini-3.1-flash-image"} @pytest.mark.skip(reason="Runwayml image generation API only tested locally") diff --git a/tests/llm_translation/test_groq.py b/tests/llm_translation/test_groq.py index cf4be9e801e..c720f818eaf 100644 --- a/tests/llm_translation/test_groq.py +++ b/tests/llm_translation/test_groq.py @@ -20,7 +20,7 @@ from litellm.llms.groq.chat.transformation import ( class TestGroq(BaseLLMChatTest): def get_base_completion_call_args(self) -> dict: return { - "model": "groq/llama-3.3-70b-versatile", + "model": "groq/openai/gpt-oss-120b", } def test_tool_call_no_arguments(self, tool_call_no_arguments): diff --git a/tests/local_testing/test_model_alias_map.py b/tests/local_testing/test_model_alias_map.py index 14c1de2f6a7..9ef0448e7c6 100644 --- a/tests/local_testing/test_model_alias_map.py +++ b/tests/local_testing/test_model_alias_map.py @@ -15,7 +15,7 @@ from litellm import completion, embedding litellm.set_verbose = True -model_alias_map = {"good-model": "groq/llama-3.1-8b-instant"} +model_alias_map = {"good-model": "groq/openai/gpt-oss-120b"} def test_model_alias_map(caplog): @@ -34,7 +34,7 @@ def test_model_alias_map(caplog): if rec.levelname == "ERROR" and rec.name.startswith("LiteLLM"): pytest.fail(f"Unexpected litellm ERROR log: {rec.getMessage()}") - assert "llama-3.1-8b-instant" in response.model + assert "gpt-oss-120b" in response.model except litellm.ServiceUnavailableError: pass except Exception as e: diff --git a/tests/local_testing/test_router.py b/tests/local_testing/test_router.py index af1a052e86d..7bc29517f8c 100644 --- a/tests/local_testing/test_router.py +++ b/tests/local_testing/test_router.py @@ -120,7 +120,7 @@ async def test_router_provider_wildcard_routing(): print("response 2 = ", response2) response3 = await router.acompletion( - model="groq/llama-3.1-8b-instant", + model="groq/openai/gpt-oss-120b", messages=[{"role": "user", "content": "hello"}], ) diff --git a/tests/local_testing/test_router_batch_completion.py b/tests/local_testing/test_router_batch_completion.py index f7a1b41ca29..bb9e1851c61 100644 --- a/tests/local_testing/test_router_batch_completion.py +++ b/tests/local_testing/test_router_batch_completion.py @@ -44,7 +44,7 @@ async def test_batch_completion_multiple_models(mode): { "model_name": "groq-llama", "litellm_params": { - "model": "groq/llama-3.1-8b-instant", + "model": "groq/openai/gpt-oss-120b", }, }, ] @@ -143,7 +143,7 @@ async def test_batch_completion_fastest_response_streaming(): { "model_name": "groq-llama", "litellm_params": { - "model": "groq/llama-3.1-8b-instant", + "model": "groq/openai/gpt-oss-120b", }, }, ] @@ -179,7 +179,7 @@ async def test_batch_completion_multiple_models_multiple_messages(): { "model_name": "groq-llama", "litellm_params": { - "model": "groq/llama-3.1-8b-instant", + "model": "groq/openai/gpt-oss-120b", }, }, ] diff --git a/tests/local_testing/test_stream_chunk_builder.py b/tests/local_testing/test_stream_chunk_builder.py index 38e04b93f18..664fd936205 100644 --- a/tests/local_testing/test_stream_chunk_builder.py +++ b/tests/local_testing/test_stream_chunk_builder.py @@ -871,7 +871,7 @@ def load_env(): } LLAMA3_3 = { "messages": messages, - "model": "groq/llama-3.3-70b-versatile", + "model": "groq/openai/gpt-oss-120b", "api_base": "https://api.groq.com/openai/v1", "temperature": 0.0, "tools": tools, diff --git a/tests/logging_callback_tests/test_logging_redaction_e2e_test.py b/tests/logging_callback_tests/test_logging_redaction_e2e_test.py index d56d5e51b04..3b42595b959 100644 --- a/tests/logging_callback_tests/test_logging_redaction_e2e_test.py +++ b/tests/logging_callback_tests/test_logging_redaction_e2e_test.py @@ -317,110 +317,69 @@ async def test_redaction_responses_api_stream(): @pytest.mark.asyncio async def test_redaction_responses_api_with_reasoning_summary(): """Test that reasoning summary in ResponsesAPIResponse output is properly redacted""" + import litellm from litellm.litellm_core_utils.redact_messages import perform_redaction - # Create a simple mock object with output items that have reasoning summaries - class MockResponsesAPIResponse: - def __init__(self): - self.output = [ - # Reasoning item with summary - type( - "obj", - (object,), + response = litellm.ResponsesAPIResponse( + id="resp_123", + created_at=1234567890, + output=[ + { + "type": "reasoning", + "id": "rs_123", + "summary": [ { - "type": "reasoning", - "id": "rs_123", - "summary": [ - type( - "obj", - (object,), - { - "text": "This is a detailed reasoning summary that should be redacted", - "type": "summary_text", - }, - )() - ], - }, - )(), - # Message item with content - type( - "obj", - (object,), + "type": "summary_text", + "text": "This is a detailed reasoning summary that should be redacted", + } + ], + }, + { + "type": "message", + "id": "msg_123", + "status": "completed", + "role": "assistant", + "content": [ { - "type": "message", - "id": "msg_123", - "content": [ - type( - "obj", - (object,), - { - "text": "This is the actual message content", - "type": "output_text", - }, - )() - ], - }, - )(), - ] - self.reasoning = {"effort": "low", "summary": "auto"} + "type": "output_text", + "text": "This is the actual message content", + "annotations": [], + } + ], + }, + ], + reasoning={"effort": "low", "summary": "auto"}, + ) - # Mock as ResponsesAPIResponse so perform_redaction recognizes it - mock_response = MockResponsesAPIResponse() - mock_response.__class__.__name__ = "ResponsesAPIResponse" + model_call_details = { + "messages": [{"role": "user", "content": "test"}], + "prompt": "test prompt", + "input": "test input", + } - # Patch isinstance to recognize our mock as ResponsesAPIResponse - import litellm + redacted_result = perform_redaction(model_call_details, response) - original_isinstance = isinstance + assert isinstance( + redacted_result, litellm.ResponsesAPIResponse + ), "Redaction should preserve the ResponsesAPIResponse type" - def patched_isinstance(obj, cls): - if ( - cls == litellm.ResponsesAPIResponse - and obj.__class__.__name__ == "ResponsesAPIResponse" - ): - return True - return original_isinstance(obj, cls) + reasoning_item = redacted_result.output[0] + assert ( + reasoning_item.summary[0].text == "redacted-by-litellm" + ), "Reasoning summary text should be redacted" - import builtins + message_item = redacted_result.output[1] + assert ( + message_item.content[0].text == "redacted-by-litellm" + ), "Message content text should be redacted" - builtins.isinstance = patched_isinstance + assert ( + redacted_result.reasoning is None + ), "Top-level reasoning field should be None" - try: - model_call_details = { - "messages": [{"role": "user", "content": "test"}], - "prompt": "test prompt", - "input": "test input", - } - - # Perform redaction - redacted_result = perform_redaction(model_call_details, mock_response) - - # Verify reasoning summary text is redacted - reasoning_item = redacted_result.output[0] - assert ( - reasoning_item.summary[0].text == "redacted-by-litellm" - ), "Reasoning summary text should be redacted" - - # Verify message content is also redacted - message_item = redacted_result.output[1] - assert ( - message_item.content[0].text == "redacted-by-litellm" - ), "Message content text should be redacted" - - # Verify top-level reasoning field is removed - assert ( - redacted_result.reasoning is None - ), "Top-level reasoning field should be None" - - # Verify input messages are redacted - assert ( - model_call_details["messages"][0]["content"] == "redacted-by-litellm" - ), "Input messages should be redacted" - - print("✓ Reasoning summary redaction test passed") - finally: - # Restore original isinstance - builtins.isinstance = original_isinstance + assert ( + model_call_details["messages"][0]["content"] == "redacted-by-litellm" + ), "Input messages should be redacted" @pytest.mark.asyncio diff --git a/tests/ocr_tests/test_ocr_azure_document_intelligence.py b/tests/ocr_tests/test_ocr_azure_document_intelligence.py index 09c21842ad7..5736bd797e3 100644 --- a/tests/ocr_tests/test_ocr_azure_document_intelligence.py +++ b/tests/ocr_tests/test_ocr_azure_document_intelligence.py @@ -62,7 +62,7 @@ class TestAzureDocumentIntelligencePagesParam: return AzureDocumentIntelligenceOCRConfig() def test_get_supported_ocr_params_includes_pages_and_features(self, cfg): - assert cfg.get_supported_ocr_params("prebuilt-layout") == ["pages", "features"] + assert cfg.get_supported_ocr_params("prebuilt-layout") == ["pages", "features", "req_format"] def test_map_ocr_params_mistral_zero_based_int_list(self, cfg): mapped = cfg.map_ocr_params({"pages": [0, 1, 2]}, {}, "prebuilt-layout") diff --git a/tests/pass_through_unit_tests/test_pass_through_unit_tests.py b/tests/pass_through_unit_tests/test_pass_through_unit_tests.py index c263b8ce381..77fb924c085 100644 --- a/tests/pass_through_unit_tests/test_pass_through_unit_tests.py +++ b/tests/pass_through_unit_tests/test_pass_through_unit_tests.py @@ -413,35 +413,54 @@ async def test_pass_through_request_logging_failure_with_stream( assert response.body == b'{"mock": "response"}' +PROTOCOL_CONSTRAINED_PASS_THROUGH_ROUTES = { + "/comprehendmedical": {"POST"}, + "/comprehendmedical/{operation}": {"POST"}, +} + + def test_pass_through_routes_support_all_methods(): """ - Test that all pass-through routes support GET, POST, PUT, DELETE, PATCH methods + A pass-through route fronts a whole provider API, so narrowing its method + set turns a request the upstream would have accepted into a 405. The + exceptions are providers whose wire protocol admits only one method: Amazon + Comprehend Medical speaks AWS JSON 1.1, which is POST-only, so there is no + other method to forward. """ - # Import the routers from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( router as llm_router, ) - # Expected HTTP methods expected_methods = {"GET", "POST", "PUT", "DELETE", "PATCH"} - # Function to check routes in a router def check_router_methods(router): for route in router.routes: if isinstance(route, APIRoute): - # Get path and methods for this route path = route.path methods = set(route.methods) - print("supported methods for route", path, "are", methods) - # Assert all expected methods are supported + allowed = PROTOCOL_CONSTRAINED_PASS_THROUGH_ROUTES.get(path, expected_methods) assert ( - methods == expected_methods - ), f"Route {path} does not support all methods. Supported: {methods}, Expected: {expected_methods}" + methods == allowed + ), f"Route {path} does not support all methods. Supported: {methods}, Expected: {allowed}" - # Check both routers check_router_methods(llm_router) +def test_protocol_constrained_pass_through_exemptions_are_not_stale(): + """ + The exemption list above weakens the method contract, so it must not + outlive the routes it covers: a renamed or deleted route has to fail here + rather than sit in the list silently exempting nothing. + """ + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + router as llm_router, + ) + + registered_paths = {route.path for route in llm_router.routes if isinstance(route, APIRoute)} + unmatched = set(PROTOCOL_CONSTRAINED_PASS_THROUGH_ROUTES) - registered_paths + assert not unmatched, f"Exempted pass-through routes no longer exist: {sorted(unmatched)}" + + def test_is_bedrock_agent_runtime_route(): """ Test that _is_bedrock_agent_runtime_route correctly identifies bedrock agent runtime endpoints diff --git a/tests/search_tests/test_tinyfish_search.py b/tests/search_tests/test_tinyfish_search.py index aca28544513..becb8287a29 100644 --- a/tests/search_tests/test_tinyfish_search.py +++ b/tests/search_tests/test_tinyfish_search.py @@ -35,11 +35,16 @@ MOCK_TINYFISH_RESPONSE = { def _make_mock_response( - json_data: dict, status_code: int = 200, request_url: str | None = None + json_data: dict, + status_code: int = 200, + request_url: str | None = None, + headers: dict | None = None, ) -> MagicMock: mock = MagicMock() mock.status_code = status_code mock.json.return_value = json_data + # httpx.Headers normalizes keys to lowercase — mirror production behavior. + mock.headers = httpx.Headers(headers or {}) if request_url: mock.request = MagicMock() mock.request.url = httpx.URL(request_url) @@ -163,7 +168,7 @@ class TestTinyfishSearch: @pytest.mark.asyncio async def test_fetch_param_round_trip(self): - # End-to-end check: caller passes `fetch=...` (JSON-encoded tf-fetch + # End-to-end check: caller passes `fetch=...` (JSON-encoded fetch # config); param reaches TinyFish on the request side and the nested # `fetch` object on each result surfaces back to the SearchResult on the # response side. No LiteLLM-side support code is required. @@ -235,6 +240,58 @@ class TestTinyfishSearch: assert result.results[0].title == "Result 0" assert result.results[2].title == "Result 2" + @pytest.mark.asyncio + async def test_top_level_extras_surface_end_to_end(self): + # Envelope extras (`query`, `total_results`, `page`) must survive the + # full asearch dispatch — proves LiteLLM's entry-point plumbing outside + # our transformer doesn't accidentally strip them. + os.environ["TINYFISH_API_KEY"] = "sk-tinyfish-test" + + mock_response = _make_mock_response(MOCK_TINYFISH_RESPONSE) + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get", + new_callable=AsyncMock, + ) as mock_get: + mock_get.return_value = mock_response + + response = await litellm.asearch( + query="web automation tools", + search_provider="tinyfish", + ) + + assert getattr(response, "query", None) == "web automation tools" + assert getattr(response, "total_results", None) == 2 + assert getattr(response, "page", None) == 0 + + @pytest.mark.asyncio + async def test_response_headers_surface_end_to_end(self): + # Response headers must land on `_hidden_params` after the full + # asearch dispatch (both raw and sanitized channels). + os.environ["TINYFISH_API_KEY"] = "sk-tinyfish-test" + + mock_response = _make_mock_response( + MOCK_TINYFISH_RESPONSE, + headers={"X-Request-ID": "req-e2e-1"}, + ) + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get", + new_callable=AsyncMock, + ) as mock_get: + mock_get.return_value = mock_response + + response = await litellm.asearch( + query="test", + search_provider="tinyfish", + ) + + raw = response._hidden_params["headers"] + add = response._hidden_params["additional_headers"] + # httpx lowercases; both channels agree on the value. + assert raw["x-request-id"] == "req-e2e-1" + assert add["llm_provider-x-request-id"] == "req-e2e-1" + @pytest.mark.asyncio async def test_empty_results(self): os.environ["TINYFISH_API_KEY"] = "sk-tinyfish-test" diff --git a/tests/test_litellm/batches/test_batch_utils.py b/tests/test_litellm/batches/test_batch_utils.py index 64b3df4180b..f7067053acb 100644 --- a/tests/test_litellm/batches/test_batch_utils.py +++ b/tests/test_litellm/batches/test_batch_utils.py @@ -959,8 +959,14 @@ async def test_handle_completed_vertex_batch_computes_cost_usage_and_models(monk litellm_params={"vertex_project": "proj-1", "vertex_location": "us-central1"}, ) + pricing = litellm.model_cost["vertex_ai/gemini-3.6-flash"] + batch_input = pricing["input_cost_per_token_batches"] + batch_output = pricing["output_cost_per_token_batches"] + + assert batch_input < pricing["input_cost_per_token"] + assert batch_output < pricing["output_cost_per_token"] assert result.cost > 0 - assert result.cost == pytest.approx(30 * 7.5e-07 + 15 * 3.75e-06) + assert result.cost == pytest.approx(30 * batch_input + 15 * batch_output) assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (30, 15, 45) assert result.models == ["gemini-3.6-flash", "gemini-3.6-flash"] assert result.successful_requests == 2 diff --git a/tests/test_litellm/conftest.py b/tests/test_litellm/conftest.py index 0dc8f56f3ce..c0644c88291 100644 --- a/tests/test_litellm/conftest.py +++ b/tests/test_litellm/conftest.py @@ -100,6 +100,12 @@ def isolate_host_aws_config(monkeypatch, isolated_aws_credentials_dir): monkeypatch.delenv("AWS_DEFAULT_REGION", raising=False) +@pytest.fixture(scope="function", autouse=True) +def isolate_host_proxy_base_url(monkeypatch): + """Prevent a host PROXY_BASE_URL from outranking request-derived URLs during unit tests.""" + monkeypatch.delenv("PROXY_BASE_URL", raising=False) + + def _run_coroutine_if_needed(result): if not asyncio.iscoroutine(result): return diff --git a/tests/test_litellm/integrations/otel/test_db_endpoint.py b/tests/test_litellm/integrations/otel/test_db_endpoint.py new file mode 100644 index 00000000000..5ab0a927b52 --- /dev/null +++ b/tests/test_litellm/integrations/otel/test_db_endpoint.py @@ -0,0 +1,312 @@ +"""Tests for litellm/integrations/otel/model/db_endpoint.py + +Prisma talks to PostgreSQL through a loopback query engine, so a DB span with no +``server.address`` gets attributed to ``localhost`` by the backend. These cover +the endpoint derivation that names the real server, for the local engine and for +remote and read-replica deployments, and pin the rule that no credential is ever +exported. +""" + +import os +from unittest.mock import patch + +import pytest + +from litellm.integrations.otel.model.db_endpoint import ( + DatabaseEndpoint, + db_span_attributes, + parse_database_endpoint, + postgres_endpoint, +) + +LOCAL_DSN = "postgresql://llmproxy:dbpassword9090@localhost:5432/litellm" +REMOTE_DSN = "postgresql://llmproxy:s3cr3t@litellm-prod.abc123.us-east-1.rds.amazonaws.com:6432/litellm?schema=reporting&sslmode=require" +REPLICA_DSN = "postgresql://reader:r3ad0nly@litellm-prod-ro.abc123.us-east-1.rds.amazonaws.com/litellm_replica" + + +def _resolve(service, call_type=None, database_url=None, read_replica_url=None): + """Resolve attributes with the two DB env vars set, as the proxy sets them.""" + env = {k: v for k, v in (("DATABASE_URL", database_url), ("DATABASE_URL_READ_REPLICA", read_replica_url)) if v} + with patch.dict(os.environ, env, clear=False): + for absent in {"DATABASE_URL", "DATABASE_URL_READ_REPLICA"} - set(env): + os.environ.pop(absent, None) + return dict(db_span_attributes(service, call_type)) + + +def test_local_prisma_engine_endpoint_is_the_postgres_server_not_the_engine(): + assert parse_database_endpoint(LOCAL_DSN) == DatabaseEndpoint( + address="localhost", port=5432, namespace="litellm" + ) + + +def test_remote_endpoint_keeps_host_port_and_schema_qualified_namespace(): + assert parse_database_endpoint(REMOTE_DSN) == DatabaseEndpoint( + address="litellm-prod.abc123.us-east-1.rds.amazonaws.com", + port=6432, + namespace="litellm|reporting", + ) + + +def test_read_replica_dsn_parses_to_the_replica_host_and_database(): + assert parse_database_endpoint(REPLICA_DSN) == DatabaseEndpoint( + address="litellm-prod-ro.abc123.us-east-1.rds.amazonaws.com", + port=5432, + namespace="litellm_replica", + ) + + +def test_default_schema_is_not_spelled_out_in_the_namespace(): + """``?schema=public`` and no schema at all are the same deployment, so they + must not split a group-by on db.namespace.""" + assert parse_database_endpoint("postgresql://u:p@db.internal/litellm?schema=public") == parse_database_endpoint( + "postgresql://u:p@db.internal/litellm" + ) + + +def test_unix_socket_host_parameter_wins_over_the_netloc(): + """libpq and the Cloud SQL connector both put the real target in ``host=`` + behind a localhost netloc, which is the attribution this module removes.""" + assert parse_database_endpoint( + "postgresql://u:p@localhost:5432/litellm?host=/cloudsql/proj:us-east1:inst" + ) == DatabaseEndpoint(address="/cloudsql/proj:us-east1:inst", port=5432, namespace="litellm") + + +def test_socket_only_dsn_without_a_netloc_host_still_resolves(): + assert parse_database_endpoint("postgresql:///litellm?host=/var/run/postgresql") == DatabaseEndpoint( + address="/var/run/postgresql", port=5432, namespace="litellm" + ) + + +def test_percent_encoded_database_name_is_decoded(): + endpoint = parse_database_endpoint("postgresql://u:p@db.internal/litellm%20prod") + assert endpoint is not None and endpoint.namespace == "litellm prod" + + +MISPARSED_AUTHORITY_DSNS = ( + ("postgresql://litellm:/kJ8xQz+9wT@db.internal:5432/litellm", "kJ8xQz+9wT"), + ("postgresql://litellm:12345/aBcD@db.internal:5432/litellm", "aBcD"), + # '#' sends the tail to the fragment and '?' to the query, so the path is + # empty and only the stranded userinfo '@' reveals the mis-split. + ("postgresql://litellm:12345#aBcD@db.internal/litellm", "aBcD"), + ("postgresql://litellm:12345?aBcD@db.internal/litellm", "aBcD"), + # A '?'-stranded tail that happens to parse as parameters, including one + # that hijacks the host= parameter into server.address. + ("postgresql://litellm:12345?a=aBcD@db.internal/litellm", "aBcD"), + ("postgresql://litellm:12345?host=aBcD@db.internal/litellm", "aBcD"), + # Both '/' and '?key=value' together: the slash leaves a clean path holding + # the password remainder and the query still parses, so only the stranded + # at-sign gives it away. + ("postgresql://litellm:12345/aBcD?x=1@db.internal/litellm", "aBcD"), +) + + +@pytest.mark.parametrize(("dsn", "secret"), MISPARSED_AUTHORITY_DSNS) +def test_unencoded_slash_in_password_never_yields_an_endpoint(dsn, secret): + """An unencoded '/' truncates the authority, so urlparse reports the username + as the host and the password tail as the database. Postgres drivers reject + such a DSN outright, so the only safe reading is no endpoint at all.""" + assert parse_database_endpoint(dsn) is None + + +@pytest.mark.parametrize(("dsn", "secret"), MISPARSED_AUTHORITY_DSNS) +def test_unencoded_slash_in_password_never_reaches_a_span(dsn, secret): + attrs = _resolve("postgres", "get_data", database_url=dsn) + exported = " ".join(str(value) for value in attrs.values()) + assert secret not in exported + assert "db.namespace" not in attrs + assert "server.address" not in attrs + + +def test_extra_path_segment_yields_no_endpoint(): + """A database name cannot hold an unencoded '/', so a second path segment + means the authority was mis-split even when no '@' survived into the path.""" + assert parse_database_endpoint("postgresql://db.internal:5432/litellm/extra") is None + + +@pytest.mark.parametrize("dsn", [d for d, _ in MISPARSED_AUTHORITY_DSNS]) +def test_a_mis_split_authority_never_exports_the_database_username(dsn): + """The username lands in ``parsed.hostname`` when the authority truncates, so + a span would name the DB user as the server.""" + attrs = _resolve("postgres", "get_data", database_url=dsn) + assert "server.address" not in attrs + assert "litellm" not in " ".join(str(v) for v in attrs.values()) + + +@pytest.mark.parametrize( + "dsn", + [ + "postgresql://db.internal:5432/litellm?application_name=svc@prod", + "postgresql://db.internal:5432/litellm?user=admin@company.com", + ], +) +def test_an_unencoded_at_sign_in_a_query_forfeits_the_endpoint(dsn): + """This shape is byte-for-byte indistinguishable from a mis-split password, + so it resolves to no endpoint rather than risking a credential fragment. + Percent-encoding the at-sign restores the attributes.""" + assert parse_database_endpoint(dsn) is None + assert parse_database_endpoint(dsn.replace("@", "%40")) is not None + + +def test_host_and_port_query_parameters_are_honoured_together(): + assert parse_database_endpoint("postgresql://ignored/litellm?host=real.internal&port=6543") == DatabaseEndpoint( + address="real.internal", port=6543, namespace="litellm" + ) + + +def test_percent_encoded_password_still_resolves_the_endpoint(): + """The encoded spelling is the one a driver accepts, so it must keep working.""" + assert parse_database_endpoint("postgresql://litellm:pa%2Fssw0rd@db.internal:5432/litellm") == DatabaseEndpoint( + address="db.internal", port=5432, namespace="litellm" + ) + + +def test_hostless_socket_dsn_still_names_the_database(): + """``postgresql:///litellm`` is a valid local-socket DSN that Prisma accepts, + so the database is knowable even though no server address is.""" + assert parse_database_endpoint("postgresql:///litellm") == DatabaseEndpoint( + address=None, port=None, namespace="litellm" + ) + + +def test_hostless_socket_dsn_emits_namespace_without_a_server(): + attrs = _resolve("postgres", "get_data", database_url="postgresql:///litellm") + assert attrs["db.namespace"] == "litellm" + assert "server.address" not in attrs + assert "server.port" not in attrs + + +def test_dsn_with_neither_host_nor_database_yields_no_endpoint(): + assert parse_database_endpoint("postgresql://") is None + + +def test_prisma_default_schema_is_left_implicit(): + endpoint = parse_database_endpoint("postgresql://u:p@db.internal/litellm?schema=public") + assert endpoint is not None and endpoint.namespace == "litellm" + + +@pytest.mark.parametrize("spelling", ["PUBLIC", "Public", "reporting"]) +def test_a_non_default_schema_stays_in_the_namespace(spelling): + """Prisma quotes the schema name, so ``?schema=PUBLIC`` provisions a second + schema alongside ``public`` with its own tables. Case-folding them into one + namespace would report two different schemas as the same database.""" + endpoint = parse_database_endpoint(f"postgresql://u:p@db.internal/litellm?schema={spelling}") + assert endpoint is not None and endpoint.namespace == f"litellm|{spelling}" + + +def test_postgres_scheme_alias_is_accepted(): + assert parse_database_endpoint("postgres://u:p@db.internal/litellm") == DatabaseEndpoint( + address="db.internal", port=5432, namespace="litellm" + ) + + +@pytest.mark.parametrize( + "dsn", + [ + None, + "", + "mysql://u:p@db.internal:3306/litellm", + "postgresql://u:p@db.internal:not-a-port/litellm", + "not a url at all", + ], +) +def test_unusable_dsn_degrades_to_no_endpoint(dsn): + assert parse_database_endpoint(dsn) is None + + +def test_database_without_name_or_schema_has_no_namespace(): + assert parse_database_endpoint("postgresql://u:p@db.internal:5432/") == DatabaseEndpoint( + address="db.internal", port=5432, namespace=None + ) + + +def test_postgres_service_span_carries_system_operation_and_endpoint(): + assert _resolve("postgres", "get_data", database_url=REMOTE_DSN) == { + "db.system.name": "postgresql", + "db.system": "postgresql", + "db.operation.name": "get_data", + "server.address": "litellm-prod.abc123.us-east-1.rds.amazonaws.com", + "server.port": 6432, + "db.namespace": "litellm|reporting", + } + + +def test_legacy_db_system_is_dual_emitted_for_datadog(): + """Datadog's OTLP intake infers the database span type from ``db.system``, + not from the semconv-current ``db.system.name``.""" + assert _resolve("postgres", "get_data", database_url=LOCAL_DSN)["db.system"] == "postgresql" + assert _resolve("redis", "set")["db.system"] == "redis" + + +def test_batch_write_service_is_also_attributed_to_postgres(): + attrs = _resolve("batch_write_to_db", "_PROXY_track_cost_callback", database_url=REMOTE_DSN) + assert attrs["db.system.name"] == "postgresql" + assert attrs["server.address"] == "litellm-prod.abc123.us-east-1.rds.amazonaws.com" + + +def test_redis_service_never_borrows_the_postgres_endpoint(): + assert _resolve("redis", "set", database_url=REMOTE_DSN) == { + "db.system.name": "redis", + "db.system": "redis", + "db.operation.name": "set", + } + + +def test_non_datastore_service_gets_no_db_attributes(): + assert _resolve("reset_budget_job", "reset_budget", database_url=REMOTE_DSN) == {} + + +def test_configured_read_replica_suppresses_the_endpoint_rather_than_naming_the_primary(): + """Reads are routed to the replica per Prisma call, underneath the span, so + naming the writer would pin replica latency onto the primary.""" + attrs = _resolve("postgres", "get_data", database_url=REMOTE_DSN, read_replica_url=REPLICA_DSN) + assert attrs == { + "db.system.name": "postgresql", + "db.system": "postgresql", + "db.operation.name": "get_data", + } + + +def test_endpoint_attributes_are_omitted_when_database_url_is_unset(): + assert _resolve("postgres", "get_data") == { + "db.system.name": "postgresql", + "db.system": "postgresql", + "db.operation.name": "get_data", + } + + +def test_blank_call_type_does_not_emit_an_empty_operation_attribute(): + assert "db.operation.name" not in _resolve("postgres", "") + assert "db.operation.name" not in _resolve("postgres", None) + + +@pytest.mark.parametrize( + ("dsn", "secrets"), + [ + (LOCAL_DSN, ("dbpassword9090", "llmproxy")), + (REMOTE_DSN, ("s3cr3t", "llmproxy", "sslmode")), + (REPLICA_DSN, ("r3ad0nly", "reader")), + ], +) +def test_no_credential_reaches_any_exported_attribute(dsn, secrets): + attrs = _resolve("postgres", "get_data", database_url=dsn) + assert attrs["server.address"] + exported = " ".join(str(value) for value in attrs.values()) + for secret in secrets: + assert secret not in exported + + +def test_a_runtime_endpoint_change_is_reflected_on_the_next_span(): + """The RDS IAM refresh, the reconnect path and the DB-backed + environment_variables overlay can all rewrite DATABASE_URL after startup, so + a value cached for the process lifetime would report a server the process no + longer talks to.""" + first = _resolve("postgres", "get_data", database_url=LOCAL_DSN) + assert first["server.address"] == "localhost" + moved = _resolve("postgres", "get_data", database_url=REMOTE_DSN) + assert moved["server.address"] == "litellm-prod.abc123.us-east-1.rds.amazonaws.com" + + +def test_a_replica_configured_after_the_first_span_suppresses_the_endpoint(): + assert _resolve("postgres", "get_data", database_url=REMOTE_DSN)["server.address"] + later = _resolve("postgres", "get_data", database_url=REMOTE_DSN, read_replica_url=REPLICA_DSN) + assert "server.address" not in later diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py index 82b074220fa..bb2d970e9c7 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py @@ -8,6 +8,8 @@ hooks, proxy SERVER span lifecycle (start + setters), parent-context resolution import asyncio import contextlib +import os +from unittest.mock import patch from datetime import datetime, timedelta, timezone import pytest @@ -1521,6 +1523,36 @@ def test_async_service_success_hook_emits_service_span(): assert span.status.status_code is StatusCode.UNSET +def test_postgres_db_span_names_the_database_server_not_the_prisma_engine(): + """Prisma reaches Postgres over loopback, so without server.address the + backend attributes the wait to localhost.""" + dsn = "postgresql://llmproxy:dbpassword9090@litellm-prod.abc123.us-east-1.rds.amazonaws.com:6432/litellm?schema=reporting" + logger, exporter = _logger() + parent = _service_parent(logger) + try: + with patch.dict(os.environ, {"DATABASE_URL": dsn}, clear=False): + os.environ.pop("DATABASE_URL_READ_REPLICA", None) + asyncio.run( + logger.async_service_success_hook( + payload=_ServicePayload("postgres", "get_data"), + parent_otel_span=parent, + ) + ) + finally: + parent.end() + span = {s.name: s for s in exporter.get_finished_spans()}["postgres get_data"] + assert span.kind is SpanKind.CLIENT + assert span.attributes["db.system.name"] == "postgresql" + assert span.attributes["db.operation.name"] == "get_data" + assert span.attributes["server.address"] == "litellm-prod.abc123.us-east-1.rds.amazonaws.com" + assert span.attributes["server.port"] == 6432 + assert span.attributes["db.namespace"] == "litellm|reporting" + assert span.attributes["db.system"] == "postgresql" + exported = " ".join(str(value) for value in span.attributes.values()) + assert "dbpassword9090" not in exported + assert "llmproxy" not in exported + + def test_async_service_failure_hook_marks_error_status(): logger, exporter = _logger() parent = _service_parent(logger) diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index fa1c9fa8a79..28d2cb22e7c 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -34,6 +34,7 @@ from litellm.integrations.opentelemetry import ( _normalize_team_metadata_keys, ) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.types.services import ServiceLoggerPayload, ServiceTypes class TestOpenTelemetryGuardrails(unittest.TestCase): @@ -6212,3 +6213,110 @@ class TestDynamicTracerProviderCache(unittest.TestCase): self.assertTrue(entry.owns_exporter) self.assertIsNotNone(entry.provider._atexit_handler) +class TestOpenTelemetryDatabaseSemconvAttributes(unittest.TestCase): + """A Postgres service span must name the PostgreSQL server it reached. + + Without ``db.system`` and ``server.address``, the only host in the trace is + the loopback address of Prisma's local query engine, so the backend + attributes the wait to ``localhost`` and it cannot be correlated with the + database's own metrics. + """ + + DSN = "postgresql://llmproxy:dbpassword9090@litellm-prod.abc123.us-east-1.rds.amazonaws.com:6432/litellm?schema=reporting" + REPLICA_DSN = "postgresql://reader:r3ad0nly@litellm-prod-ro.abc123.us-east-1.rds.amazonaws.com/litellm" + + def _service_span(self, service, call_type, dsn, error=None, replica_dsn=None): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + otel = OpenTelemetry() + otel.tracer = provider.get_tracer(__name__) + parent = otel.tracer.start_span("Received Proxy Server Request") + payload = ServiceLoggerPayload( + is_error=error is not None, + error=error, + service=service, + duration=0.25, + call_type=call_type, + event_metadata=None, + ) + hook = otel.async_service_failure_hook if error else otel.async_service_success_hook + kwargs = {"error": error} if error else {} + env = {k: v for k, v in (("DATABASE_URL", dsn), ("DATABASE_URL_READ_REPLICA", replica_dsn)) if v} + with patch.dict(os.environ, env, clear=False): + for absent in {"DATABASE_URL", "DATABASE_URL_READ_REPLICA"} - set(env): + os.environ.pop(absent, None) + asyncio.run( + hook( + payload=payload, + parent_otel_span=parent, + start_time=datetime.now(), + end_time=datetime.now(), + **kwargs, + ) + ) + parent.end() + return next(s for s in exporter.get_finished_spans() if s.name == service.value) + + def test_postgres_span_names_the_database_server(self): + span = self._service_span(ServiceTypes.DB, "get_data", self.DSN) + self.assertEqual(span.attributes["db.system.name"], "postgresql") + self.assertEqual(span.attributes["db.operation.name"], "get_data") + self.assertEqual( + span.attributes["server.address"], + "litellm-prod.abc123.us-east-1.rds.amazonaws.com", + ) + self.assertEqual(span.attributes["server.port"], 6432) + self.assertEqual(span.attributes["db.namespace"], "litellm|reporting") + + def test_datastore_span_is_a_client_span_carrying_the_legacy_db_system(self): + """Datadog types a span as a database call from CLIENT kind plus + ``db.system``; an INTERNAL span is classified as custom work.""" + span = self._service_span(ServiceTypes.DB, "get_data", self.DSN) + self.assertEqual(span.kind, trace.SpanKind.CLIENT) + self.assertEqual(span.attributes["db.system"], "postgresql") + + def test_internal_service_span_stays_internal(self): + span = self._service_span(ServiceTypes.RESET_BUDGET_JOB, "reset_budget", self.DSN) + self.assertEqual(span.kind, trace.SpanKind.INTERNAL) + self.assertNotIn("db.system.name", span.attributes) + self.assertNotIn("server.address", span.attributes) + + def test_existing_service_and_call_type_attributes_are_unchanged(self): + span = self._service_span(ServiceTypes.DB, "get_data", self.DSN) + self.assertEqual(span.attributes["service"], "postgres") + self.assertEqual(span.attributes["call_type"], "get_data") + + def test_failed_postgres_span_also_names_the_database_server(self): + span = self._service_span(ServiceTypes.DB, "get_data", self.DSN, error="connection refused") + self.assertEqual(span.attributes["db.system.name"], "postgresql") + self.assertEqual(span.kind, trace.SpanKind.CLIENT) + self.assertEqual( + span.attributes["server.address"], + "litellm-prod.abc123.us-east-1.rds.amazonaws.com", + ) + self.assertEqual(span.attributes["error"], "connection refused") + + def test_no_credential_from_the_dsn_lands_on_the_span(self): + span = self._service_span(ServiceTypes.DB, "get_data", self.DSN) + exported = " ".join(str(value) for value in span.attributes.values()) + self.assertIn("litellm-prod.abc123.us-east-1.rds.amazonaws.com", exported) + self.assertNotIn("dbpassword9090", exported) + self.assertNotIn("llmproxy", exported) + + def test_redis_span_does_not_borrow_the_postgres_endpoint(self): + span = self._service_span(ServiceTypes.REDIS, "async_set_cache", self.DSN) + self.assertEqual(span.attributes["db.system.name"], "redis") + self.assertEqual(span.kind, trace.SpanKind.CLIENT) + self.assertNotIn("server.address", span.attributes) + + def test_configured_read_replica_suppresses_the_endpoint(self): + span = self._service_span(ServiceTypes.DB, "get_data", self.DSN, replica_dsn=self.REPLICA_DSN) + self.assertEqual(span.attributes["db.system.name"], "postgresql") + self.assertNotIn("server.address", span.attributes) + self.assertNotIn("db.namespace", span.attributes) + + def test_unset_database_url_leaves_the_span_without_endpoint_attributes(self): + span = self._service_span(ServiceTypes.DB, "get_data", None) + self.assertEqual(span.attributes["db.system.name"], "postgresql") + self.assertNotIn("server.address", span.attributes) diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index af40245ebfa..f9311497729 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -241,10 +241,32 @@ def test_split_concatenated_json_non_dict_value(): assert result == [{}] -def test_split_concatenated_json_invalid_raises(): - """Completely invalid JSON raises JSONDecodeError.""" - with pytest.raises(json.JSONDecodeError): - split_concatenated_json_objects("not json at all") +def test_split_concatenated_json_wholly_invalid_returns_empty(): + """ + Wholly unparseable JSON degrades to an empty list instead of raising. + + Regression for https://github.com/BerriAI/litellm/issues/18667: a raise + here propagated out of `_convert_to_bedrock_tool_call_invoke` and turned + every replayed conversation into a 500. + """ + assert split_concatenated_json_objects("not json at all") == [] + + +def test_split_concatenated_json_malformed_object_returns_empty(): + """ + A single malformed object (missing comma between keys) degrades to an + empty list rather than raising `Expecting ',' delimiter`. + """ + assert split_concatenated_json_objects('{"location": "Boston" "unit": "celsius"}') == [] + + +def test_split_concatenated_json_salvages_prefix_before_truncated_tail(): + """ + Complete objects parsed before an unparseable/truncated tail are kept; + only the bad tail is discarded. + """ + result = split_concatenated_json_objects('{"a": 1}{"b": 2}{"c":') + assert result == [{"a": 1}, {"b": 2}] # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index de5d0a180c6..fffbc884782 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -2287,6 +2287,116 @@ def test_bedrock_tool_call_invoke_non_dict_arguments(): assert result[0]["toolUse"]["input"] == {} +def test_bedrock_tool_call_invoke_malformed_json_does_not_raise(): + """ + Regression for https://github.com/BerriAI/litellm/issues/18667. + + When the model emits malformed JSON in tool-call arguments (here a + missing comma between keys), replaying that history must NOT raise + `Unable to convert openai tool calls ... Expecting ',' delimiter`. + It degrades to an empty-object input so the conversation can continue. + """ + tool_calls = [ + { + "id": "toolu_abc123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location": "Boston" "unit": "celsius"}', + }, + } + ] + result = _convert_to_bedrock_tool_call_invoke(tool_calls) + assert len(result) == 1 + assert result[0]["toolUse"]["toolUseId"] == "toolu_abc123" + assert result[0]["toolUse"]["name"] == "get_weather" + assert result[0]["toolUse"]["input"] == {} + + +def test_bedrock_tool_call_invoke_salvages_valid_prefix_before_truncated_tail(): + """ + A valid leading object followed by a truncated tail keeps the valid + object rather than dropping everything or raising. + """ + tool_calls = [ + { + "id": "call_partial", + "type": "function", + "function": {"name": "shell", "arguments": '{"cmd": "ls"}{"cmd":'}, + } + ] + result = _convert_to_bedrock_tool_call_invoke(tool_calls) + assert len(result) == 1 + assert result[0]["toolUse"]["input"] == {"cmd": "ls"} + + +def test_bedrock_tool_call_invoke_mixed_turn_survives_one_malformed_call(): + """ + Regression for LIT-4574: an assistant turn with several tool calls where only one + has malformed/truncated arguments must keep the valid calls intact and degrade just + the bad one to empty input, instead of killing the entire turn. + """ + tool_calls = [ + { + "id": "t_good", + "type": "function", + "function": { + "name": "good_tool", + "arguments": '{"item_type": "email", "item_id": "AAMkAD=="}', + }, + }, + { + "id": "t_bad", + "type": "function", + "function": {"name": "bad_tool", "arguments": '{"item_type": "email"'}, + }, + ] + result = _convert_to_bedrock_tool_call_invoke(tool_calls) + tool_uses = [block["toolUse"] for block in result if "toolUse" in block] + assert len(tool_uses) == 2 + by_name = {tool_use["name"]: tool_use for tool_use in tool_uses} + assert by_name["good_tool"]["input"] == {"item_type": "email", "item_id": "AAMkAD=="} + assert by_name["bad_tool"]["input"] == {} + + +def test_bedrock_tool_call_invoke_truncated_json_arguments(): + """ + Truncated tool call arguments (issue #35303) must not raise. A client replaying a + partially streamed tool call would otherwise trigger a pre-network exception that the + router maps to a retryable APIConnectionError and retries through the fallback graph. + """ + tool_calls = [ + { + "id": "tooluse_MAh2QLVjBRkvi5QJkLQ08V", + "type": "function", + "function": { + "name": "replace_note_content", + "arguments": '{"note_id": "999af35c-4061-4ece-8581-7d43fc988ba4", "title": "WG"', + }, + } + ] + result = _convert_to_bedrock_tool_call_invoke(tool_calls) + assert len(result) == 1 + assert result[0]["toolUse"]["toolUseId"] == "tooluse_MAh2QLVjBRkvi5QJkLQ08V" + assert result[0]["toolUse"]["input"] == {} + + +def test_bedrock_tool_call_invoke_unconvertible_raises_non_retryable_bad_request(): + """ + Conversion failures are client input errors, so they must surface as a non-retryable + BadRequestError instead of a bare Exception that maps to APIConnectionError, and the + message must not embed the tool call payload (issue #35303). + """ + tool_calls = [{"id": "call_bad", "type": "function", "function": None}] + + with pytest.raises(litellm.BadRequestError) as exc_info: + _convert_to_bedrock_tool_call_invoke(tool_calls) + + assert exc_info.value.status_code == 400 + assert "call_bad" in str(exc_info.value) + assert "function" not in str(exc_info.value).split("Received error=")[0] + + def test_make_valid_bedrock_tool_name_preserves_hyphens(): assert make_valid_bedrock_tool_name("my-tool") == "my-tool" assert ( diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index e46057fe11a..99781a2f211 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -4918,3 +4918,51 @@ def test_payload_without_guardrail_cost_is_unchanged(logging_obj): assert payload is not None assert payload["response_cost"] == pytest.approx(0.0000429) assert payload["cost_breakdown"] is None + + +_AWS_SECRET = "wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY" +_GEMINI_KEY = "AIzaSyC0000000000000000000000000000000" + + +def test_empty_api_base_does_not_dump_call_state(logging_obj): + """Direct (non-HTTP) providers pass api_base='', which used to echo model_call_details.""" + logging_obj.model_call_details["litellm_params"] = { + "api_key": "sk-proj-hunter2hunter2hunter2hunter2", + "aws_secret_access_key": _AWS_SECRET, + } + + curl_command = logging_obj._get_request_curl_command( + api_base="", + headers={}, + additional_args={}, + data={"model": "some-model"}, + ) + + assert "litellm_call_id" not in curl_command + assert _AWS_SECRET not in curl_command + assert "hunter2" not in curl_command + + +def test_pre_call_redacts_and_masks_raw_request(logging_obj): + """log_raw_request_response echoes the request body and api_base back to loggers/UI.""" + metadata = {"user_api_key_alias": "qa-key"} + logging_obj.model_call_details["litellm_params"] = {"metadata": metadata} + logging_obj.log_raw_request_response = True + + logging_obj.pre_call( + input="hi", + api_key="", + additional_args={ + "api_base": f"https://generativelanguage.googleapis.com/v1beta/models/x:generateContent?key={_GEMINI_KEY}", + "headers": {}, + "complete_input_dict": {"aws_secret_access_key": _AWS_SECRET}, + }, + ) + + raw_request = metadata["raw_request"] + assert _AWS_SECRET not in raw_request + assert "REDACTED" in raw_request + + raw_api_base = logging_obj.model_call_details["raw_request_typed_dict"]["raw_request_api_base"] + assert _GEMINI_KEY not in raw_api_base + assert "key=*****" in raw_api_base diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 2509f6480d5..298360789eb 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -6043,3 +6043,12 @@ def test_streaming_usage_chunk_is_transformed(): assert chunk.usage.prompt_tokens == 11 assert chunk.usage.completion_tokens == 4 assert chunk.usage.total_tokens == 15 + + +def test_update_optional_params_with_thinking_tokens_bool_thinking_does_not_crash(): + config = AmazonConverseConfig() + optional_params = {"thinking": True} + config.update_optional_params_with_thinking_tokens( + non_default_params={"thinking": True}, optional_params=optional_params + ) + assert "maxTokens" not in optional_params diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index 2dda8bf722a..69f2312f203 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -1,7 +1,9 @@ import asyncio import json +import logging import os import sys +import time from unittest.mock import AsyncMock, Mock, patch import httpx @@ -9,6 +11,7 @@ import pytest sys.path.insert(0, os.path.abspath("../../../..")) # Adds the parent directory to the system path import litellm +from litellm._logging import verbose_logger from litellm.integrations.code_interpreter_interception.handler import ( CodeInterpreterInterceptionLogger, LITELLM_CODE_EXECUTION_TOOL_NAME, @@ -2158,3 +2161,68 @@ async def test_vector_store_search_handler_direct_config_async_skips_http(): pre_call_args = logging_obj.pre_call.call_args.kwargs["additional_args"] assert pre_call_args["query"] == ["q1", "q2"] assert pre_call_args["vector_store_id"] == "vs_direct" + + +def _direct_vector_store_debug_logging_obj(): + from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging + + logging_obj = LitellmLogging( + model="valkey", + messages=[{"role": "user", "content": "q"}], + stream=False, + call_type="vector_store_search", + start_time=time.time(), + litellm_call_id="vs-debug-call-id", + function_id="vs-debug-function-id", + log_raw_request_response=True, + ) + logging_obj.update_environment_variables( + model="valkey", + optional_params={"vector_store_id": "vs_direct", "query": "q"}, + litellm_params={ + "litellm_call_id": "vs-debug-call-id", + "vector_store_id": "vs_direct", + "litellm_request_debug": True, + "metadata": {"user_api_key_alias": "vs-test-key"}, + "valkey_host": "valkey.internal", + "valkey_password": "sup3r-s3cret-valkey-pw", + "litellm_embedding_config": {"api_key": "sk-embedding-s3cret"}, + }, + ) + return logging_obj + + +@pytest.mark.parametrize("is_async", [False, True]) +def test_direct_vector_store_search_debug_log_omits_stored_credentials(caplog, is_async): + """Regression: an empty api_base made pre_call dump the whole model_call_details, so every + search shipped the stored valkey_password / embedding api_key into the raw_request metadata.""" + handler = BaseLLMHTTPHandler() + stub_response = {"object": "vector_store.search_results.page", "search_query": "q", "data": []} + config = _make_stub_direct_vector_store_config(stub_response) + logging_obj = _direct_vector_store_debug_logging_obj() + + with caplog.at_level(logging.DEBUG, logger=verbose_logger.name): + result = handler.vector_store_search_handler( + vector_store_id="vs_direct", + query="q", + vector_store_search_optional_params={"max_num_results": 4}, + vector_store_provider_config=config, + custom_llm_provider="valkey", + litellm_params=GenericLiteLLMParams( + valkey_host="valkey.internal", + valkey_password="sup3r-s3cret-valkey-pw", + ), + logging_obj=logging_obj, + _is_async=is_async, + ) + if is_async: + result = asyncio.run(result) + + assert result is stub_response + raw_request = logging_obj.model_call_details["litellm_params"]["metadata"]["raw_request"] + assert "sup3r-s3cret-valkey-pw" not in raw_request + assert "sk-embedding-s3cret" not in raw_request + assert "valkey://vs_direct" in raw_request + logged = "\n".join(record.getMessage() for record in caplog.records) + assert "sup3r-s3cret-valkey-pw" not in logged + assert "sk-embedding-s3cret" not in logged diff --git a/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py b/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py index 6f5aaabae06..510776ddfdf 100644 --- a/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py +++ b/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py @@ -271,6 +271,47 @@ class TestDashscopeCostCalculator: assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10) + def test_dashscope_nested_cache_creation_input_tokens_bill_at_cache_write_rate(self): + """ + Regression (LIT-5757): DashScope nests cache_creation_input_tokens inside + prompt_tokens_details; those tokens must bill at the tier's cache-creation + rate instead of being folded into text tokens at the input rate. + """ + self._register_tiered_model( + "dashscope/qwen-nested-cache-write-test", + [ + { + "range": [0, 128000], + "input_cost_per_token": 4e-07, + "cache_read_input_token_cost": 1.6e-07, + "cache_creation_input_token_cost": 5e-07, + "output_cost_per_token": 1.6e-06, + } + ], + ) + + usage = Usage( + prompt_tokens=2059, + completion_tokens=201, + total_tokens=2260, + prompt_tokens_details={ + "cached_tokens": 0, + "text_tokens": 2059, + "cache_type": "ephemeral", + "cache_creation_input_tokens": 2048, + "cache_creation": {"ephemeral_5m_input_tokens": 2048}, + }, + completion_tokens_details={"reasoning_tokens": 170}, + ) + + prompt_cost, _ = dashscope_cost_per_token( + model="qwen-nested-cache-write-test", usage=usage + ) + + assert math.isclose( + prompt_cost, (2048 * 5e-07) + (11 * 4e-07), rel_tol=1e-10 + ) + def test_dashscope_tiered_cache_creation_falls_back_to_tier_input_rate(self): """ Tiers without a cache_creation_input_token_cost bill cache-creation tokens at diff --git a/tests/test_litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py b/tests/test_litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py index ec51e5d303d..d5783e3567f 100644 --- a/tests/test_litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py +++ b/tests/test_litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py @@ -101,3 +101,8 @@ async def test_async_transform_request_strips_unsupported_tools_from_body(): assert [tool["type"] for tool in body["tools"]] == ["function"] assert body["tools"][0]["function"]["name"] == "shell" + + +def test_thinking_mode_active_bool_thinking_returns_false_without_crashing(): + config = DeepSeekChatConfig() + assert config._thinking_mode_active(model="deepseek-reasoner", optional_params={"thinking": True}) is False diff --git a/tests/test_litellm/llms/tinyfish/test_tinyfish_search.py b/tests/test_litellm/llms/tinyfish/test_tinyfish_search.py index 58363e3baea..2dcccb8ea7e 100644 --- a/tests/test_litellm/llms/tinyfish/test_tinyfish_search.py +++ b/tests/test_litellm/llms/tinyfish/test_tinyfish_search.py @@ -47,7 +47,9 @@ def _make_mock_response( mock = MagicMock() mock.status_code = status_code - mock.headers = headers or {} + # httpx.Headers normalizes keys to lowercase — mirror production so tests + # assert what callers actually see. + mock.headers = httpx.Headers(headers or {}) if json_data is not None: mock.json.return_value = json_data mock.text = text if text is not None else _json.dumps(json_data) @@ -222,7 +224,7 @@ class TestTransformSearchRequest: assert param not in result["_tinyfish_params"] def test_arbitrary_param_passed_through(self): - # `fetch` is a TinyFish-specific param (JSON-encoded tf-fetch config). + # `fetch` is a TinyFish-specific param (JSON-encoded fetch config). # The passthrough loop should forward it verbatim without LiteLLM needing # to know about it. config = TinyfishSearchConfig() @@ -237,26 +239,49 @@ class TestTransformSearchRequest: config = TinyfishSearchConfig() result = config.transform_search_request( query="test", - optional_params={"fetch": {"format": "html", "fetch_path": "fast"}}, - ) - assert ( - result["_tinyfish_params"]["fetch"] - == '{"format":"html","fetch_path":"fast"}' + optional_params={"fetch": {"format": "html"}}, ) + assert result["_tinyfish_params"]["fetch"] == '{"format":"html"}' def test_bool_param_serialized_as_lowercase(self): - # urlencode renders Python bool as capitalized "True"/"False"; ux-labs - # rejects those (e.g. include_thumbnail must be literal "true"/"false"). - # Normalize before passing through. + # urlencode renders Python bool as capitalized "True"/"False"; TinyFish + # Search's bool params require lowercase "true"/"false" strings on the + # wire. Normalize before passing through. config = TinyfishSearchConfig() true_result = config.transform_search_request( - query="test", optional_params={"include_thumbnail": True} + query="test", optional_params={"some_bool_param": True} ) false_result = config.transform_search_request( - query="test", optional_params={"include_thumbnail": False} + query="test", optional_params={"some_bool_param": False} ) - assert true_result["_tinyfish_params"]["include_thumbnail"] == "true" - assert false_result["_tinyfish_params"]["include_thumbnail"] == "false" + assert true_result["_tinyfish_params"]["some_bool_param"] == "true" + assert false_result["_tinyfish_params"]["some_bool_param"] == "false" + + def test_float_param_passes_through(self): + # Float values pass the urlencode adapter and land on the wire as + # their decimal string form. If TinyFish's server rejects a float + # for a param it expects as int, the server's 400 response is + # attributed via _wrap_error (`TinyFish Search: ...`) — better than + # a client-side pydantic ValidationError with no context. + config = TinyfishSearchConfig() + result = config.transform_search_request( + query="test", + optional_params={"some_float_param": 0.5}, + ) + assert result["_tinyfish_params"]["some_float_param"] == 0.5 + + def test_list_param_auto_json_encoded(self): + # TinyFish Search's JSON-array params arrive on the wire as JSON- + # encoded strings. Accept the natural Python list form and serialize + # so the caller doesn't have to pre-stringify. Params whose wire + # format is a plain comma-separated string are the caller's + # responsibility to pass as a Python str. + config = TinyfishSearchConfig() + result = config.transform_search_request( + query="test", + optional_params={"some_list_param": ["a.example", "b.example"]}, + ) + assert result["_tinyfish_params"]["some_list_param"] == '["a.example","b.example"]' def test_pre_stringified_param_passed_unchanged(self): # If the caller already JSON-encoded, don't re-encode. @@ -422,10 +447,104 @@ class TestTransformSearchResponse: assert getattr(first, "position", None) == 1 assert getattr(first, "site_name", None) == "tinyfish.ai" + def test_top_level_extras_flow_through(self): + # TinyFish returns `query`, `total_results`, `page` at the envelope + # level. These must ride through to the caller via SearchResponse's + # extra="allow" so pagination logic, echo checks, etc. work. + config = TinyfishSearchConfig() + mock_response = _make_mock_response(MOCK_TINYFISH_RESPONSE) + result = config.transform_search_response( + raw_response=mock_response, logging_obj=None + ) + assert getattr(result, "query", None) == "web automation tools" + assert getattr(result, "total_results", None) == 2 + assert getattr(result, "page", None) == 0 + + def test_top_level_future_extras_flow_through(self): + # Any future TinyFish top-level field must ride through unchanged + # (design contract: no LiteLLM code change needed for new fields). + config = TinyfishSearchConfig() + body = { + "results": [ + {"title": "x", "url": "https://x", "snippet": "x"}, + ], + "query": "test", + "example_int_extra": 123, # hypothetical future field + "example_str_extra": "value", # hypothetical future field + "example_id_extra": "abc-def", # hypothetical future field + } + result = config.transform_search_response( + raw_response=_make_mock_response(body), logging_obj=None + ) + assert getattr(result, "example_int_extra", None) == 123 + assert getattr(result, "example_str_extra", None) == "value" + assert getattr(result, "example_id_extra", None) == "abc-def" + + def test_response_headers_stashed_on_hidden_params(self): + # TinyFish Search sets X-Request-ID on every success response. Confirm it + # lands on both `_hidden_params["headers"]` (raw) and + # `_hidden_params["additional_headers"]` (sanitized/prefixed). + # httpx.Headers lowercases every key, so assertions use lowercase. + config = TinyfishSearchConfig() + mock_response = _make_mock_response( + MOCK_TINYFISH_RESPONSE, + headers={"X-Request-ID": "req-abc-123", "Content-Type": "application/json"}, + ) + result = config.transform_search_response( + raw_response=mock_response, logging_obj=None + ) + # Raw copy — httpx has normalized keys to lowercase. + assert result._hidden_params["headers"]["x-request-id"] == "req-abc-123" + # process_response_headers prefixes non-OpenAI-standard keys with "llm_provider-". + assert result._hidden_params["additional_headers"]["llm_provider-x-request-id"] == "req-abc-123" + + def test_response_headers_future_headers_flow_through(self): + # "Accept extra": any header TinyFish Search adds later must ride + # through without a LiteLLM code change. + config = TinyfishSearchConfig() + mock_response = _make_mock_response( + MOCK_TINYFISH_RESPONSE, + headers={ + "X-Request-ID": "req-1", + "X-Example-Header-A": "value-a", # hypothetical future header + "X-Example-Header-B": "value-b", # hypothetical future header + }, + ) + result = config.transform_search_response( + raw_response=mock_response, logging_obj=None + ) + raw = result._hidden_params["headers"] + # httpx lowercases header names on read. + assert raw["x-example-header-a"] == "value-a" + assert raw["x-example-header-b"] == "value-b" + + def test_response_headers_strips_x_litellm_spoof(self): + # A provider setting `x-litellm-*` in its response must not be able to + # spoof LiteLLM-internal markers via _hidden_params["additional_headers"]. + # The raw copy preserves the header (opt-in debug view); the sanitized + # copy prefixes it with `llm_provider-` so bare `x-litellm-*` markers + # can't be spoofed (values still survive under the prefixed key for + # observability). + config = TinyfishSearchConfig() + mock_response = _make_mock_response( + MOCK_TINYFISH_RESPONSE, + headers={"x-litellm-attempted-fallbacks": "spoofed", "X-Request-ID": "r1"}, + ) + result = config.transform_search_response( + raw_response=mock_response, logging_obj=None + ) + # Raw view still has the spoof. + assert result._hidden_params["headers"]["x-litellm-attempted-fallbacks"] == "spoofed" + # Sanitized view: the spoof survives only under the llm_provider- prefix + # (never under the bare x-litellm-* key that LiteLLM downstream trusts). + additional = result._hidden_params["additional_headers"] + assert "x-litellm-attempted-fallbacks" not in additional + assert additional.get("llm_provider-x-litellm-attempted-fallbacks") == "spoofed" + def test_fetch_field_rides_through_to_search_result(self): - # Mirrors browser-search's per-result `fetch` nested object (see - # api/src/parser.rs SearchResult.fetch). Confirms `fetch=...` requests - # surface their content to LiteLLM callers without provider changes. + # Mirrors TinyFish Search's per-result `fetch` nested object. + # Confirms `fetch=...` requests surface their content to LiteLLM + # callers without provider changes. config = TinyfishSearchConfig() fetched = { "results": [ @@ -568,7 +687,7 @@ class TestTransformSearchResponse: class TestErrorHandling: def test_4xx_response_raises_with_attribution_and_unwrapped_message(self): - # Reproduces ux-labs' error envelope shape for an INVALID_INPUT response. + # Reproduces TinyFish Search's error envelope shape for an INVALID_INPUT response. config = TinyfishSearchConfig() body = { "error": { @@ -590,7 +709,7 @@ class TestErrorHandling: def test_429_preserves_status_code_and_headers(self): config = TinyfishSearchConfig() - body = {"error": {"code": "RATE_LIMIT_EXCEEDED", "message": "60 rpm"}} + body = {"error": {"code": "RATE_LIMIT_EXCEEDED", "message": "rate limit exceeded"}} mock_response = _make_mock_response( body, status_code=429, headers={"Retry-After": "60"} ) @@ -600,10 +719,12 @@ class TestErrorHandling: ) assert getattr(exc_info.value, "status_code", None) == 429 headers = getattr(exc_info.value, "headers", {}) or {} - assert headers.get("Retry-After") == "60" + # httpx lowercases; the exception carries the same dict shape. + assert headers.get("retry-after") == "60" - def test_5xx_with_non_ux_labs_body_falls_back_to_raw_text(self): - # Cloudflare-style JSON or any other envelope: unwrap fails, fall back to raw. + def test_5xx_with_non_tinyfish_envelope_shape_falls_back_to_raw_text(self): + # A JSON body that doesn't match TinyFish Search's error envelope shape: + # unwrap fails, fall back to the raw body text. config = TinyfishSearchConfig() body = {"errors": [{"code": "10000", "message": "Internal"}]} mock_response = _make_mock_response(body, status_code=502) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index 3392203dbab..051df30dfcb 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -6406,7 +6406,7 @@ async def test_execute_mcp_tool_rest_server_id_injects_requested_server_credenti with ( patch.dict( mcp_module.global_mcp_server_manager.tool_name_to_mcp_server_name_mapping, - {"echo": collision_server.name}, + {"echo": collision_server.name, "echo_requested-echo": requested_server.name}, ), patch.object( mcp_module.global_mcp_server_manager, @@ -8430,3 +8430,72 @@ class TestListFiltersHonorThePrefixBoundary: assert listed == callable_, f"grants={grants!r} listed={listed} callable={callable_}" assert listed is expected, f"grants={grants!r} expected={expected} got={listed}" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "auth_type", + [MCPAuth.authorization, MCPAuth.bearer_token, MCPAuth.api_key, MCPAuth.basic, MCPAuth.token], +) +async def test_list_tools_injects_byok_credential_for_non_oauth2_auth_types(auth_type): + """Regression for BYOK servers on a non-oauth2 auth_type: the stored per-user credential must be + attached when listing tools, otherwise the upstream 401 is absorbed and the server lists nothing.""" + try: + from litellm.proxy._experimental.mcp_server.server import ( + _get_tools_from_mcp_servers, + set_auth_context, + ) + except ImportError: + pytest.skip("MCP server not available") + + user_api_key_auth = UserAPIKeyAuth(api_key="test_key", user_id="byok_user") + set_auth_context(user_api_key_auth) + + server = MagicMock() + server.server_id = "byok_server" + server.name = "byok" + server.alias = "byok" + server.server_name = "byok" + server.auth_type = auth_type + server.is_byok = True + server.allowed_tools = None + server.disallowed_tools = None + server.extra_headers = None + server.tool_name_to_display_name = None + server.tool_name_to_description = None + + seen_auth_headers = [] + + async def mock_get_tools_from_server(server, mcp_auth_header=None, add_prefix=False, **kwargs): + seen_auth_headers.append(mcp_auth_header) + tool = MagicMock() + tool.name = f"{server.alias}-toolA" if add_prefix else "toolA" + tool.description = "desc" + tool.inputSchema = {} + return [tool] + + mock_manager = MagicMock() + mock_manager.get_allowed_mcp_servers = AsyncMock(return_value=[server.server_id]) + mock_manager.get_mcp_server_by_id = MagicMock(return_value=server) + mock_manager.filter_server_ids_by_ip_with_info = lambda server_ids, client_ip: (server_ids, 0) + mock_manager._get_tools_from_server = mock_get_tools_from_server + + with ( + patch( + "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + mock_manager, + ), + patch( + "litellm.proxy._experimental.mcp_server.server._get_byok_credential", + AsyncMock(return_value="personal-api-key"), + ), + ): + listing = await _get_tools_from_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=None, + mcp_servers=None, + mcp_server_auth_headers=None, + ) + + assert seen_auth_headers == ["personal-api-key"] + assert [tool.name for tool in listing.tools] == ["byok-toolA"] diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index cd1ef7320dd..9064312fd6d 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -42,6 +42,7 @@ from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( _deserialize_json_list, _normalize_mcp_server_cost_info, _obo_retry_applies, + _resolve_openapi_tool_auth, _should_strip_caller_authorization, _without_authorization, ) @@ -4833,6 +4834,74 @@ class TestMCPServerManager: with pytest.raises(ValueError, match="Tool missing_tool not found"): manager._resolve_mcp_server_for_tool_call("github", "missing_tool") + @staticmethod + def _manager_with_deepwiki_and_huggingface() -> MCPServerManager: + manager = MCPServerManager() + deepwiki = MCPServer(server_id="deepwiki-id", name="deepwiki", server_name="deepwiki", transport=MCPTransport.http) + huggingface = MCPServer( + server_id="huggingface-id", name="huggingface", server_name="huggingface", transport=MCPTransport.http + ) + manager.registry = {"deepwiki-id": deepwiki, "huggingface-id": huggingface} + manager.tool_name_to_mcp_server_name_mapping = { + "read_wiki_structure": "deepwiki", + "deepwiki-read_wiki_structure": "deepwiki", + "hub_repo_search": "huggingface", + "huggingface-hub_repo_search": "huggingface", + } + return manager + + def test_resolve_mcp_server_for_tool_call_rejects_tool_exposed_only_by_another_server(self): + manager = self._manager_with_deepwiki_and_huggingface() + + with pytest.raises(ValueError, match="Tool read_wiki_structure not found"): + manager._resolve_mcp_server_for_tool_call("huggingface", "read_wiki_structure") + with pytest.raises(ValueError, match="Tool hub_repo_search not found"): + manager._resolve_mcp_server_for_tool_call("deepwiki", "hub_repo_search") + + assert manager._resolve_mcp_server_for_tool_call("deepwiki", "read_wiki_structure") is manager.registry["deepwiki-id"] + assert manager._resolve_mcp_server_for_tool_call("huggingface", "hub_repo_search") is manager.registry["huggingface-id"] + + def test_get_mcp_server_from_tool_name_rejects_other_servers_prefix(self): + manager = self._manager_with_deepwiki_and_huggingface() + + assert manager._get_mcp_server_from_tool_name("huggingface-read_wiki_structure") is None + assert manager._get_mcp_server_from_tool_name("deepwiki-hub_repo_search") is None + assert manager._get_mcp_server_from_tool_name("deepwiki-read_wiki_structure") is manager.registry["deepwiki-id"] + assert manager._get_mcp_server_from_tool_name("huggingface-hub_repo_search") is manager.registry["huggingface-id"] + + def test_resolve_mcp_server_for_tool_call_shared_bare_name_resolves_via_own_prefixed_spelling(self): + manager = MCPServerManager() + zapier = MCPServer(server_id="zapier-id", name="zapier", alias="zapier-alias", transport=MCPTransport.http) + other = MCPServer(server_id="other-id", name="other", server_name="other", transport=MCPTransport.http) + manager.registry = {"zapier-id": zapier, "other-id": other} + manager.tool_name_to_mcp_server_name_mapping = { + "create_zap": "other", + "other-create_zap": "other", + "zapier-alias-create_zap": "zapier-alias", + } + + assert manager._resolve_mcp_server_for_tool_call("zapier", "create_zap") is zapier + assert manager._resolve_mcp_server_for_tool_call("other", "create_zap") is other + + def test_remove_server_drops_only_its_own_tool_mapping_rows(self): + manager = self._manager_with_deepwiki_and_huggingface() + + manager.remove_server( + LiteLLM_MCPServerTable( + server_id="huggingface-id", + alias="huggingface", + url="https://huggingface.co/mcp", + transport=MCPTransport.http, + created_at=datetime.now(), + updated_at=datetime.now(), + ) + ) + + assert manager.tool_name_to_mcp_server_name_mapping == { + "read_wiki_structure": "deepwiki", + "deepwiki-read_wiki_structure": "deepwiki", + } + @pytest.mark.asyncio async def test_resolve_oauth2_headers_skipped_when_not_user_oauth(self): """Returns input headers unchanged when server does not need user OAuth.""" @@ -10457,3 +10526,242 @@ class TestSessionResourceScopeIntersect: ): fallback = await manager.get_allowed_mcp_servers(auth) assert fallback == ["granted-id"] + + +class TestClientForwardedDiscoveryFailureIsNotFatal: + """A failed OAuth metadata discovery may only brick the flows the gateway runs itself. + + ``true_passthrough`` / ``oauth_delegate`` forward the caller's own bearer and mint nothing, so an + upstream that publishes no RFC 9728 metadata (an internal API, or any IdP unreachable from the + pod) must still serve sessions instead of 503-ing before the upstream is ever contacted. + """ + + @staticmethod + def _config(auth_type: MCPAuthType, dcr_bridge: bool | None) -> dict[str, dict[str, object]]: + entry: Final[dict[str, object]] = { + "url": "https://up.example.com/mcp", + "transport": MCPTransport.http, + "auth_type": auth_type, + **({"oauth2_flow": "authorization_code"} if auth_type == MCPAuth.oauth2 else {}), + **({"dcr_bridge": dcr_bridge} if dcr_bridge is not None else {}), + } + return {"upstream": entry} + + async def _registered(self, manager: MCPServerManager, auth_type: MCPAuthType, dcr_bridge: bool | None): + with ( + patch.object(manager, "_discover_oauth_metadata_for_server", new=AsyncMock(return_value=None)), + patch.object(manager, "initialize_tool_name_to_mcp_server_name_mapping"), + ): + await manager.load_servers_from_config(self._config(auth_type, dcr_bridge)) + return next(iter(manager.config_mcp_servers.values())) + + @pytest.mark.parametrize( + "auth_type, dcr_bridge, serves_without_endpoints", + [ + (MCPAuth.true_passthrough, None, True), + (MCPAuth.true_passthrough, True, True), + (MCPAuth.oauth_delegate, None, True), + (MCPAuth.oauth_delegate, True, True), + (MCPAuth.oauth2, None, False), + (MCPAuth.oauth2_token_exchange, None, False), + ], + ) + @pytest.mark.parametrize("failure", ["incomplete", "timed_out"]) + @pytest.mark.asyncio + async def test_discovery_failure_blocks_only_gateway_run_flows( + self, + auth_type: MCPAuthType, + dcr_bridge: bool | None, + serves_without_endpoints: bool, + failure: str, + ): + manager = MCPServerManager() + server = await self._registered(manager, auth_type, dcr_bridge) + manager._set_oauth_discovery_deferred(server.server_id, True) + + async def never_returns(_server): + await asyncio.Future() + + discovery_patch: Final = ( + {"new": AsyncMock(return_value=None)} if failure == "incomplete" else {"side_effect": never_returns} + ) + with ( + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.MCP_METADATA_TIMEOUT", 0.01), + patch.object(manager, "_discover_oauth_metadata_for_server", **discovery_patch), + ): + if not serves_without_endpoints: + with pytest.raises(HTTPException) as exc: + await manager.ensure_oauth_metadata_discovered(server) + assert exc.value.status_code == 503 + return + + resolved = await manager.ensure_oauth_metadata_discovered(server) + + assert resolved is manager.config_mcp_servers[server.server_id] + assert resolved.authorization_url is None + assert resolved.token_url is None + assert manager._oauth_discovery_slot(server.server_id) is not None + + @pytest.mark.parametrize( + "auth_type, serves_the_listing", + [(MCPAuth.true_passthrough, True), (MCPAuth.oauth_delegate, True), (MCPAuth.oauth2, False)], + ) + @pytest.mark.asyncio + async def test_listing_leg_serves_a_forwarding_server_whose_discovery_failed( + self, auth_type: MCPAuthType, serves_the_listing: bool + ): + """The listing leg is where the 503 became an empty tool list, so pin the fix there too. + + ``_get_tools_from_server`` is the per-server leg the aggregate absorbs: a failure here is what + the fan-out turns into HTTP 200 with ``tools: []``, which is why the outage carried no + diagnostic. A forwarding server must now reach its upstream, and a gateway-run flow must still + surface the fault rather than be silently listed as empty. + """ + manager = MCPServerManager() + server = await self._registered(manager, auth_type, None) + manager._set_oauth_discovery_deferred(server.server_id, True) + manager._fetch_tools_with_timeout = AsyncMock( + return_value=[MCPTool(name="list_reports", description="d", inputSchema={"type": "object"})] + ) + + with patch.object(manager, "_discover_oauth_metadata_for_server", new=AsyncMock(return_value=None)): + if not serves_the_listing: + with pytest.raises(MCPServerListError): + await manager._get_tools_from_server(server=server) + return + tools = await manager._get_tools_from_server(server=server) + + assert [tool.name for tool in tools] == ["upstream-list_reports"] + manager._fetch_tools_with_timeout.assert_awaited_once() + + @pytest.mark.parametrize("auth_type", [MCPAuth.true_passthrough, MCPAuth.oauth_delegate]) + @pytest.mark.asyncio + async def test_client_forwarded_servers_keep_discovering_their_front_door_endpoints( + self, auth_type: MCPAuthType + ): + """Exempting these modes from the FAILURE must not exempt them from discovery itself. + + ``/authorize``, ``/token`` and ``/register`` read the discovered endpoints for these servers + (``_resolve_ephemeral_dcr_client`` mints for ``true_passthrough`` whatever ``dcr_bridge`` + says), so an exemption written into the unresolved-endpoints predicate would disarm the slot + and silently drop a working front door. + """ + manager = MCPServerManager() + metadata: Final = MCPOAuthMetadata( + authorization_url="https://idp.example.com/authorize", + token_url="https://idp.example.com/token", + registration_url="https://idp.example.com/register", + ) + with ( + patch.object(manager, "_discover_oauth_metadata_for_server", new=AsyncMock(return_value=metadata)), + patch.object(manager, "initialize_tool_name_to_mcp_server_name_mapping"), + ): + await manager.load_servers_from_config(self._config(auth_type, None)) + server = next(iter(manager.config_mcp_servers.values())) + resolved = await manager.ensure_oauth_metadata_discovered(server) + + assert resolved.authorization_url == "https://idp.example.com/authorize" + assert resolved.token_url == "https://idp.example.com/token" + assert resolved.registration_url == "https://idp.example.com/register" + assert manager.config_mcp_servers[server.server_id].authorization_url == "https://idp.example.com/authorize" + assert manager._oauth_discovery_slot(server.server_id) is None + + +class TestResolveOpenapiToolAuth: + """The credential matrix for a ``spec_path`` server's two OpenAPI dispatch arms. + + A per-server ``x-mcp-{alias}-authorization`` is already a complete header value and is forwarded + verbatim; a BYOK credential is a raw secret that takes the auth-type prefix. Conflating them + ships ``Bearer Bearer ``, so every cell pins which of the two a value came from. + """ + + @staticmethod + def _server(auth_type: MCPAuthType = MCPAuth.oauth_delegate) -> MCPServer: + return MCPServer( + server_id="srv-openapi", + name="report_api", + server_name="report_api", + alias="report_api", + url="https://api.internal.example.com", + transport=MCPTransport.http, + auth_type=auth_type, + spec_path="https://api.internal.example.com/openapi.json", + extra_headers=["X-Tenant"], + ) + + @pytest.mark.parametrize( + "per_server, byok, expected_auth, expected_extra_keys, expected_credential", + [ + ( + {"report_api": "Bearer caller-token"}, + None, + "Bearer caller-token", + {"X-Tenant"}, + "Bearer caller-token", + ), + ( + {"report_api": {"Authorization": "Bearer caller-token", "X-Trace": "abc"}}, + None, + "Bearer caller-token", + {"X-Tenant", "X-Trace"}, + {"Authorization": "Bearer caller-token", "X-Trace": "abc"}, + ), + ( + {"report_api": {"X-Api-Key": "k1"}}, + "byok-secret", + "Bearer byok-secret", + {"X-Tenant", "X-Api-Key"}, + "byok-secret", + ), + ({"report_api": {"X-Api-Key": "k1"}}, None, None, {"X-Tenant", "X-Api-Key"}, None), + (None, "byok-secret", "Bearer byok-secret", {"X-Tenant"}, "byok-secret"), + (None, None, None, {"X-Tenant"}, None), + ({"other_server": "Bearer wrong"}, None, None, {"X-Tenant"}, None), + ], + ) + def test_credential_matrix( + self, + per_server: dict | None, + byok: str | None, + expected_auth: str | None, + expected_extra_keys: set, + expected_credential: object, + ): + auth_value, forwarded, credential = _resolve_openapi_tool_auth( + mcp_server=self._server(), + mcp_auth_header=byok, + mcp_server_auth_headers=per_server, + raw_headers={"x-tenant": "acme", "authorization": "Bearer admission-key"}, + user_api_key_auth=None, + ) + + assert auth_value == expected_auth + assert set(forwarded or {}) == expected_extra_keys + assert credential == expected_credential + + def test_per_server_value_is_never_re_prefixed(self): + """The regression that a naive wiring produces: the caller already sent ``Bearer ``.""" + auth_value, _, credential = _resolve_openapi_tool_auth( + mcp_server=self._server(auth_type=MCPAuth.api_key), + mcp_auth_header="byok-secret", + mcp_server_auth_headers={"report_api": "Bearer caller-token"}, + raw_headers=None, + user_api_key_auth=None, + ) + + assert auth_value == "Bearer caller-token" + assert credential == "Bearer caller-token" + assert not auth_value.startswith("ApiKey ") + + def test_per_server_authorization_is_not_also_left_in_forwarded_headers(self): + """``resolve_openapi_upstream_auth`` pops Authorization out of the forwarded headers, so a + second copy there would give the passthrough arm two sources to reconcile.""" + _, forwarded, _ = _resolve_openapi_tool_auth( + mcp_server=self._server(), + mcp_auth_header=None, + mcp_server_auth_headers={"report_api": {"Authorization": "Bearer caller-token"}}, + raw_headers={"x-tenant": "acme"}, + user_api_key_auth=None, + ) + + assert "Authorization" not in (forwarded or {}) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py index 63473bf3cf5..7bd846aeda4 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py @@ -189,9 +189,11 @@ async def test_openapi_local_tool_denied_when_server_not_resolvable(): pre_call = AsyncMock(return_value={}) handle_local = AsyncMock(return_value=[]) + resolve_auth = MagicMock() # `_get_mcp_server_from_tool_name` returns None — no server context. with ( + patch.object(mcp_module, "_resolve_openapi_tool_auth", new=resolve_auth), patch.object( mcp_module.global_mcp_server_manager, "_get_mcp_server_from_tool_name", @@ -228,6 +230,9 @@ async def test_openapi_local_tool_denied_when_server_not_resolvable(): assert exc.value.status_code == 503 pre_call.assert_not_awaited() handle_local.assert_not_awaited() + # The credential resolver takes a non-optional server, so the 503 guard above it is what keeps + # a missing server from ever reaching it. Pinned here so moving the guard reds this test. + resolve_auth.assert_not_called() @pytest.mark.asyncio @@ -549,3 +554,101 @@ async def test_unknown_tool_name_still_reports_not_found(): assert exc.value.status_code == 404 assert "not found" in str(exc.value.detail) + + +OPENAPI_PER_SERVER_TOKEN = "Bearer per-server-upstream-token" + + +def _spec_path_server() -> MCPServer: + return MCPServer( + server_id="srv-reports", + name="report_api", + server_name="report_api", + alias="report_api", + url="https://api.internal.example.com", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth_delegate, + spec_path="https://api.internal.example.com/openapi.json", + ) + + +@pytest.mark.parametrize("dispatch_arm", ["local_registry", "call_tool"]) +@pytest.mark.asyncio +async def test_per_server_auth_header_reaches_both_openapi_dispatch_arms(dispatch_arm: str): + """`x-mcp-{alias}-authorization` must survive on BOTH OpenAPI dispatch arms. + + OpenAPI tools live in the local tool registry, so `execute_mcp_tool` serves MCP-protocol and REST + tool calls while `MCPServerManager.call_tool` serves the responses-API handler. Both arms sourced + the upstream credential only from the deprecated global / BYOK `mcp_auth_header`, so the + per-server header was dropped and the upstream API saw no Authorization at all. + + Asserting on the resolver kwarg as well as the ContextVar is deliberate: for the client-forwarded + modes the credential has to reach `resolve_openapi_upstream_auth`, whose passthrough arm outranks + the ContextVar when it materializes a header. + """ + from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + _request_auth_header, + ) + + server = _spec_path_server() + auth_headers = {"report_api": {"Authorization": OPENAPI_PER_SERVER_TOKEN}} + user = UserAPIKeyAuth(api_key="sk-user", user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER.value) + captured: dict = {} + + async def fake_resolver(**kwargs): + captured["resolver_credential"] = kwargs["mcp_auth_header"] + return None, kwargs["forwarded_headers"] + + async def capture_local(_name, _arguments): + captured["injected"] = _request_auth_header.get() + return [] + + async def capture_openapi_handler(_server, _name, _arguments): + captured["injected"] = _request_auth_header.get() + return [] + + manager = mcp_module.global_mcp_server_manager + with ( + patch.object(manager, "resolve_openapi_upstream_auth", new=fake_resolver), + patch.object(manager, "pre_call_tool_check", new=AsyncMock(return_value={})), + ): + if dispatch_arm == "local_registry": + fake_tool = MagicMock() + fake_tool.name = "list_reports" + with ( + patch.object(manager, "_get_mcp_server_from_tool_name", return_value=server), + patch.object(mcp_module.global_mcp_tool_registry, "get_tool", return_value=fake_tool), + patch( + "litellm.proxy._experimental.mcp_server.server._handle_local_mcp_tool", + new=capture_local, + ), + patch( + "litellm.proxy._experimental.mcp_server.server.MCPRequestHandler.is_tool_allowed", + return_value=True, + ), + ): + await mcp_module.execute_mcp_tool( + name="list_reports", + arguments={}, + allowed_mcp_servers=[server], + start_time=datetime.now(timezone.utc), + user_api_key_auth=user, + mcp_server_auth_headers=auth_headers, + ) + else: + with ( + patch.object(manager, "_resolve_mcp_server_for_tool_call", return_value=server), + patch.object(manager, "_call_openapi_tool_handler", new=capture_openapi_handler), + ): + await manager.call_tool( + server_name="report_api", + name="list_reports", + arguments={}, + user_api_key_auth=user, + mcp_server_auth_headers=auth_headers, + ) + + assert captured["resolver_credential"] == {"Authorization": OPENAPI_PER_SERVER_TOKEN} + assert captured["injected"] == OPENAPI_PER_SERVER_TOKEN + assert _request_auth_header.get() is None diff --git a/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py b/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py index 0a427df0cb7..9a90daeccb7 100644 --- a/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py @@ -164,6 +164,41 @@ class TestProxyExceptionPassthrough: mock_logging.post_call_failure_hook.assert_awaited_once() +class TestFailureHookRequestData: + @pytest.mark.asyncio + async def test_failure_hook_gets_post_setup_data_with_logging_obj(self): + """Request setup replaces the processor's data dict (adding the logging + object the failure hook needs to lift token usage from); the exception + handler must pass that replaced dict, not the raw request body dict.""" + import litellm.proxy.anthropic_endpoints.endpoints as ep + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy._types import ProxyException, UserAPIKeyAuth + + captured = {} + + async def fake_process(self, **kwargs): + self.data = {**self.data, "litellm_logging_obj": "logging-obj-sentinel"} + captured["processor_data"] = self.data + raise RuntimeError("provider timeout") + + with ( + patch.object(ep, "_read_request_body", new=AsyncMock(return_value={"model": "claude-sonnet"})), + patch.object(ep.ProxyBaseLLMRequestProcessing, "base_process_llm_request", new=fake_process), + patch.object(proxy_server, "proxy_logging_obj") as mock_logging, + ): + mock_logging.post_call_failure_hook = AsyncMock() + with pytest.raises(ProxyException): + await ep.anthropic_response( + fastapi_response=MagicMock(), + request=MagicMock(), + user_api_key_dict=UserAPIKeyAuth(), + ) + + hook_request_data = mock_logging.post_call_failure_hook.await_args.kwargs["request_data"] + assert hook_request_data is captured["processor_data"] + assert hook_request_data["litellm_logging_obj"] == "logging-obj-sentinel" + + class TestEventLoggingBatchEndpoint: """Test the stubbed event logging batch endpoint""" diff --git a/tests/test_litellm/proxy/common_utils/test_registry_read_through.py b/tests/test_litellm/proxy/common_utils/test_registry_read_through.py new file mode 100644 index 00000000000..f0fbdea4e85 --- /dev/null +++ b/tests/test_litellm/proxy/common_utils/test_registry_read_through.py @@ -0,0 +1,476 @@ +import asyncio +from typing import Final + +import pytest + +from litellm.proxy.common_utils.registry_read_through import RegistryReadThrough + + +class ResyncSpy: + def __init__(self, found: bool = True, error: Exception | None = None) -> None: + self.found = found + self.error = error + self.calls: list[str] = [] + + async def __call__(self, key: str) -> bool: + self.calls.append(key) + if self.error is not None: + raise self.error + return self.found + + +@pytest.mark.asyncio +async def test_attempt_returns_true_when_resync_finds_object(): + spy: Final = ResyncSpy(found=True) + read_through: Final = RegistryReadThrough(resync=spy) + + assert await read_through.attempt("new-model") is True + assert spy.calls == ["new-model"] + + +@pytest.mark.asyncio +async def test_attempt_found_key_is_not_negative_cached(): + spy: Final = ResyncSpy(found=True) + read_through: Final = RegistryReadThrough(resync=spy) + + assert await read_through.attempt("new-model") is True + assert await read_through.attempt("new-model") is True + assert spy.calls == ["new-model", "new-model"] + + +@pytest.mark.asyncio +async def test_missing_key_is_negative_cached_within_ttl(): + spy: Final = ResyncSpy(found=False) + read_through: Final = RegistryReadThrough(resync=spy, miss_ttl_seconds=60.0) + + assert await read_through.attempt("ghost-model") is False + assert await read_through.attempt("ghost-model") is False + assert spy.calls == ["ghost-model"] + + +@pytest.mark.asyncio +async def test_negative_cache_expires_and_resync_runs_again(): + spy: Final = ResyncSpy(found=False) + read_through: Final = RegistryReadThrough(resync=spy, miss_ttl_seconds=0.05) + + assert await read_through.attempt("ghost-model") is False + await asyncio.sleep(0.1) + assert await read_through.attempt("ghost-model") is False + assert spy.calls == ["ghost-model", "ghost-model"] + + +@pytest.mark.asyncio +async def test_resync_exception_returns_false_without_negative_caching(): + spy: Final = ResyncSpy(error=RuntimeError("db down")) + read_through: Final = RegistryReadThrough(resync=spy) + + assert await read_through.attempt("new-model") is False + assert await read_through.attempt("new-model") is False + assert spy.calls == ["new-model", "new-model"] + + +@pytest.mark.asyncio +async def test_concurrent_attempts_for_missing_key_resync_once(): + class SlowResyncSpy(ResyncSpy): + async def __call__(self, key: str) -> bool: + await asyncio.sleep(0.05) + return await super().__call__(key) + + spy: Final = SlowResyncSpy(found=False) + read_through: Final = RegistryReadThrough(resync=spy, miss_ttl_seconds=60.0) + + results: Final = await asyncio.gather(*(read_through.attempt("ghost-model") for _ in range(5))) + assert results == [False] * 5 + assert spy.calls == ["ghost-model"] + + +@pytest.mark.asyncio +async def test_distinct_keys_do_not_share_negative_cache(): + spy: Final = ResyncSpy(found=False) + read_through: Final = RegistryReadThrough(resync=spy, miss_ttl_seconds=60.0) + + assert await read_through.attempt("ghost-a") is False + assert await read_through.attempt("ghost-b") is False + assert spy.calls == ["ghost-a", "ghost-b"] + + +@pytest.mark.asyncio +async def test_resync_budget_exhausted_blocks_resync_without_negative_caching(): + spy: Final = ResyncSpy(found=False) + read_through: Final = RegistryReadThrough( + resync=spy, miss_ttl_seconds=60.0, max_resyncs_per_window=2, resync_window_seconds=60.0 + ) + + assert await read_through.attempt("ghost-a") is False + assert await read_through.attempt("ghost-b") is False + assert await read_through.attempt("ghost-c") is False + assert spy.calls == ["ghost-a", "ghost-b"] + assert read_through._recent_misses.get_cache("ghost-c") is None + + +@pytest.mark.asyncio +async def test_resync_budget_replenishes_after_window(): + spy: Final = ResyncSpy(found=True) + read_through: Final = RegistryReadThrough(resync=spy, max_resyncs_per_window=1, resync_window_seconds=0.05) + + assert await read_through.attempt("model-a") is True + assert await read_through.attempt("model-b") is False + await asyncio.sleep(0.1) + assert await read_through.attempt("model-b") is True + assert spy.calls == ["model-a", "model-b"] + + +class FakeAgentRow: + def __init__(self, agent_id: str, agent_name: str) -> None: + self.agent_id = agent_id + self.agent_name = agent_name + self.object_permission = None + self.spend = 0.0 + + def model_dump(self): + return { + "agent_id": self.agent_id, + "agent_name": self.agent_name, + "agent_card_params": {"name": self.agent_name, "url": "http://db-agent"}, + "litellm_params": {}, + "object_permission": None, + "spend": self.spend, + } + + +@pytest.fixture +def clean_agent_registry(): + from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry + + original_agents: Final = list(global_agent_registry.agent_list) + original_config_agents: Final = getattr(global_agent_registry, "config_agents", ()) + global_agent_registry.agent_list = [] + global_agent_registry.config_agents = () + try: + yield global_agent_registry + finally: + global_agent_registry.agent_list = original_agents + global_agent_registry.config_agents = original_config_agents + + +@pytest.mark.asyncio +async def test_get_agent_with_read_through_recovers_agent_created_on_sibling_replica( + clean_agent_registry, monkeypatch +): + from unittest.mock import AsyncMock, MagicMock + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import get_agent_with_read_through + + agent_id: Final = "read-through-db-agent-id" + prisma_client: Final = MagicMock() + prisma_client.db.litellm_agentstable.find_unique = AsyncMock( + return_value=FakeAgentRow(agent_id, "read-through-db-agent") + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + + assert clean_agent_registry.get_agent_by_id(agent_id=agent_id) is None + agent: Final = await get_agent_with_read_through(agent_id) + + assert agent is not None + assert agent.agent_id == agent_id + prisma_client.db.litellm_agentstable.find_unique.assert_awaited_once_with( + where={"agent_id": agent_id}, + include={"object_permission": True}, + ) + + +@pytest.mark.asyncio +async def test_get_agent_with_read_through_recovers_agent_by_name(clean_agent_registry, monkeypatch): + from unittest.mock import AsyncMock, MagicMock + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import get_agent_with_read_through + + agent_name: Final = "read-through-db-agent-by-name" + prisma_client: Final = MagicMock() + prisma_client.db.litellm_agentstable.find_unique = AsyncMock( + side_effect=[None, FakeAgentRow("read-through-name-lookup-id", agent_name)] + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + + agent: Final = await get_agent_with_read_through(agent_name) + + assert agent is not None + assert agent.agent_name == agent_name + prisma_client.db.litellm_agentstable.find_unique.assert_awaited_with( + where={"agent_name": agent_name}, + include={"object_permission": True}, + ) + + +@pytest.mark.asyncio +async def test_get_agent_with_read_through_returns_none_for_unknown_agent(clean_agent_registry, monkeypatch): + from unittest.mock import AsyncMock, MagicMock + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import get_agent_with_read_through + + prisma_client: Final = MagicMock() + prisma_client.db.litellm_agentstable.find_unique = AsyncMock(return_value=None) + monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + + assert await get_agent_with_read_through("agent-nobody-created") is None + assert prisma_client.db.litellm_agentstable.find_unique.await_count == 2 + + +@pytest.mark.asyncio +async def test_resync_agents_already_registered_skips_db(clean_agent_registry, monkeypatch): + from unittest.mock import AsyncMock, MagicMock + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import _resync_agents + + agent_id: Final = "read-through-dedup-agent-id" + prisma_client: Final = MagicMock() + prisma_client.db.litellm_agentstable.find_unique = AsyncMock( + return_value=FakeAgentRow(agent_id, "read-through-dedup-agent") + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + + assert await _resync_agents(agent_id) is True + assert await _resync_agents(agent_id) is True + assert prisma_client.db.litellm_agentstable.find_unique.await_count == 1 + assert len(clean_agent_registry.agent_list) == 1 + + +class FakeGuardrailRow: + def __init__(self, guardrail_id: str, guardrail_name: str) -> None: + self.guardrail_id = guardrail_id + self.guardrail_name = guardrail_name + + def __iter__(self): + return iter( + { + "guardrail_id": self.guardrail_id, + "guardrail_name": self.guardrail_name, + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_call", + "default_on": True, + "blocked_words": [{"keyword": "secret", "action": "BLOCK"}], + }, + "guardrail_info": {}, + "status": "active", + }.items() + ) + + +@pytest.mark.asyncio +async def test_get_guardrail_with_read_through_recovers_guardrail_created_on_sibling_replica(monkeypatch): + from unittest.mock import AsyncMock, MagicMock + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import ( + get_initialized_guardrail_with_read_through, + ) + from litellm.proxy.guardrails.guardrail_registry import IN_MEMORY_GUARDRAIL_HANDLER + + guardrail_id: Final = "read-through-db-guardrail-id" + guardrail_name: Final = "read-through-db-guardrail" + prisma_client: Final = MagicMock() + prisma_client.db.litellm_guardrailstable.find_first = AsyncMock( + return_value=FakeGuardrailRow(guardrail_id, guardrail_name) + ) + prisma_client.db.litellm_guardrailstable.find_many = AsyncMock( + side_effect=AssertionError("full-table guardrail scan on read-through miss") + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + + try: + guardrail: Final = await get_initialized_guardrail_with_read_through(guardrail_name=guardrail_name) + assert guardrail is not None + assert guardrail.guardrail_name == guardrail_name + prisma_client.db.litellm_guardrailstable.find_first.assert_awaited_once_with( + where={"guardrail_name": guardrail_name, "status": "active"} + ) + finally: + IN_MEMORY_GUARDRAIL_HANDLER.delete_in_memory_guardrail(guardrail_id) + + +@pytest.mark.asyncio +async def test_get_guardrail_with_read_through_returns_none_for_unknown_guardrail(monkeypatch): + from unittest.mock import AsyncMock, MagicMock + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import ( + get_initialized_guardrail_with_read_through, + ) + + prisma_client: Final = MagicMock() + prisma_client.db.litellm_guardrailstable.find_first = AsyncMock(return_value=None) + monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + + assert await get_initialized_guardrail_with_read_through(guardrail_name="guardrail-nobody-created") is None + + +@pytest.mark.asyncio +async def test_resync_guardrails_never_loads_non_active_rows(monkeypatch): + from unittest.mock import AsyncMock, MagicMock + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import _resync_guardrails + + pending_name: Final = "pending-review-guardrail" + prisma_client: Final = MagicMock() + prisma_client.db.litellm_guardrailstable.find_first = AsyncMock(return_value=None) + monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + + assert await _resync_guardrails(pending_name) is False + prisma_client.db.litellm_guardrailstable.find_first.assert_awaited_once_with( + where={"guardrail_name": pending_name, "status": "active"} + ) + + +@pytest.mark.asyncio +async def test_resync_guardrails_syncs_under_guardrail_reconcile_lock(monkeypatch): + from unittest.mock import AsyncMock, MagicMock + + import litellm.proxy.common_utils.registry_read_through as read_through_module + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import _resync_guardrails + from litellm.proxy.guardrails.guardrail_registry import ( + GUARDRAIL_RECONCILE_LOCK, + IN_MEMORY_GUARDRAIL_HANDLER, + ) + + guardrail_name: Final = "lock-scope-guardrail" + prisma_client: Final = MagicMock() + prisma_client.db.litellm_guardrailstable.find_first = AsyncMock( + return_value=FakeGuardrailRow("lock-scope-guardrail-id", guardrail_name) + ) + lock_states: list[bool] = [] + + def record_sync(guardrail) -> None: + lock_states.append(GUARDRAIL_RECONCILE_LOCK.locked()) + + monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + monkeypatch.setattr(IN_MEMORY_GUARDRAIL_HANDLER, "sync_guardrail_from_db", record_sync) + monkeypatch.setattr(read_through_module, "_initialized_guardrail", lambda guardrail_name: MagicMock()) + + assert await _resync_guardrails(guardrail_name) is True + assert lock_states == [True] + assert not GUARDRAIL_RECONCILE_LOCK.locked() + + +@pytest.mark.asyncio +async def test_resync_model_deployments_mutates_router_under_model_reconcile_lock(monkeypatch): + from unittest.mock import AsyncMock, MagicMock + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import _resync_model_deployments + + prisma_client: Final = MagicMock() + prisma_client.db.litellm_proxymodeltable.find_many = AsyncMock(return_value=[MagicMock()]) + router: Final = MagicMock() + router.get_model_list.return_value = [] + lock_states: list[bool] = [] + + def record_add_deployment(db_models) -> None: + lock_states.append(proxy_server.MODEL_RECONCILE_LOCK.locked()) + + monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", None) + monkeypatch.setattr(proxy_server.proxy_config, "_add_deployment", record_add_deployment) + + assert await _resync_model_deployments("lock-scope-model") is True + assert lock_states == [True] + assert not proxy_server.MODEL_RECONCILE_LOCK.locked() + + +@pytest.mark.asyncio +async def test_resync_model_deployments_respects_supported_db_objects(monkeypatch): + from unittest.mock import AsyncMock, MagicMock + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import _resync_model_deployments + + prisma_client: Final = MagicMock() + prisma_client.db.litellm_proxymodeltable.find_many = AsyncMock( + side_effect=AssertionError("db hit for an object type this replica does not load") + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + monkeypatch.setattr(proxy_server, "general_settings", {"supported_db_objects": ["guardrails"]}) + + assert await _resync_model_deployments("gated-out-model") is False + + +@pytest.mark.asyncio +async def test_resync_guardrails_respects_supported_db_objects(monkeypatch): + from unittest.mock import AsyncMock, MagicMock + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import _resync_guardrails + + prisma_client: Final = MagicMock() + prisma_client.db.litellm_guardrailstable.find_unique = AsyncMock( + side_effect=AssertionError("db hit for an object type this replica does not load") + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + monkeypatch.setattr(proxy_server, "general_settings", {"supported_db_objects": ["models"]}) + + assert await _resync_guardrails("gated-out-guardrail") is False + + +@pytest.mark.asyncio +async def test_resync_agents_respects_supported_db_objects(clean_agent_registry, monkeypatch): + from unittest.mock import AsyncMock, MagicMock + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import _resync_agents + + prisma_client: Final = MagicMock() + prisma_client.db.litellm_agentstable.find_unique = AsyncMock( + side_effect=AssertionError("db hit for an object type this replica does not load") + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + monkeypatch.setattr(proxy_server, "general_settings", {"supported_db_objects": ["models"]}) + + assert await _resync_agents("gated-out-agent") is False + + +@pytest.mark.asyncio +async def test_resync_agents_waits_for_agent_reload_and_skips_duplicate_registration(clean_agent_registry, monkeypatch): + from unittest.mock import AsyncMock, MagicMock + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.agent_endpoints.agent_registry import AGENT_RECONCILE_LOCK + from litellm.proxy.common_utils.registry_read_through import _resync_agents + from litellm.types.agents import AgentResponse + + agent_id: Final = "reload-race-agent-id" + prisma_client: Final = MagicMock() + prisma_client.db.litellm_agentstable.find_unique = AsyncMock( + side_effect=AssertionError("db hit while the agent reload held the reconcile lock") + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + + async with AGENT_RECONCILE_LOCK: + resync_task: Final = asyncio.ensure_future(_resync_agents(agent_id)) + await asyncio.sleep(0.05) + assert not resync_task.done() + clean_agent_registry.register_agent( + agent_config=AgentResponse.model_validate(FakeAgentRow(agent_id, "reload-race-agent").model_dump()) + ) + + assert await resync_task is True + assert len(clean_agent_registry.agent_list) == 1 diff --git a/tests/test_litellm/proxy/hooks/test_batch_file_validation.py b/tests/test_litellm/proxy/hooks/test_batch_file_validation.py index 7ff1bc11d81..1ce1a2f3e51 100644 --- a/tests/test_litellm/proxy/hooks/test_batch_file_validation.py +++ b/tests/test_litellm/proxy/hooks/test_batch_file_validation.py @@ -1382,6 +1382,315 @@ async def test_check_and_increment_computes_descriptors_when_not_passed(): parallel_request_limiter._create_rate_limit_descriptors.assert_called_once() +@pytest.mark.asyncio +async def test_pre_call_enforces_project_otpm_limit_for_batch(): + """VERIA regression: ``_create_batch_rate_limit_descriptors`` only asked + for the generic key/user/team/model descriptors, so a project caller + could submit a batch that consumed none of its configured project OTPM + quota. The project OTPM descriptor must now be present and charged with + the batch's estimated *output* tokens, not its input tokens.""" + from litellm import DualCache + from litellm.proxy.hooks.batch_rate_limiter import _PROXY_BatchRateLimiter + from litellm.proxy.hooks.parallel_request_limiter_v3 import ( + _PROXY_MaxParallelRequestsHandler_v3, + ) + from litellm.proxy.utils import InternalUsageCache + + local_cache = DualCache() + parallel_request_limiter = _PROXY_MaxParallelRequestsHandler_v3( + internal_usage_cache=InternalUsageCache(local_cache) + ) + rate_limiter = _PROXY_BatchRateLimiter( + internal_usage_cache=InternalUsageCache(local_cache), + parallel_request_limiter=parallel_request_limiter, + ) + + user = UserAPIKeyAuth( + api_key="sk-project-batch-otpm", + models=["*"], + project_id="proj-mantle-batch", + project_metadata={"model_otpm_limit": {"gpt-4o-mini": 50}}, + ) + + # Two rows each declaring max_tokens=40: 80 output tokens total, over the + # configured 50-token project OTPM limit but negligible input tokens. + mock_content = MagicMock() + mock_content.content = ( + b'{"body": {"model": "gpt-4o-mini", "max_tokens": 40, ' + b'"messages": [{"role": "user", "content": "hi"}]}}\n' + b'{"body": {"model": "gpt-4o-mini", "max_tokens": 40, ' + b'"messages": [{"role": "user", "content": "hi"}]}}\n' + ) + + with ( + patch("litellm.proxy.proxy_server.general_settings", {}), + patch("litellm.proxy.proxy_server.llm_router", MagicMock()), + patch( + "litellm.proxy.openai_files_endpoints.common_utils.get_credentials_for_model", + return_value={"custom_llm_provider": "openai"}, + ), + patch("litellm.afile_content", new=AsyncMock(return_value=mock_content)), + ): + with pytest.raises(HTTPException) as exc: + await rate_limiter.async_pre_call_hook( + user_api_key_dict=user, + cache=local_cache, + data={"input_file_id": "file-abc123", "model": "gpt-4o-mini"}, + call_type="acreate_batch", + ) + + assert exc.value.status_code == 429 + assert "model_per_project_otpm" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_pre_call_enforces_project_itpm_limit_for_batch(): + """Companion to the OTPM regression above: a project's ITPM quota must + also apply to batch submissions.""" + from litellm import DualCache + from litellm.proxy.hooks.batch_rate_limiter import _PROXY_BatchRateLimiter + from litellm.proxy.hooks.parallel_request_limiter_v3 import ( + _PROXY_MaxParallelRequestsHandler_v3, + ) + from litellm.proxy.utils import InternalUsageCache + + local_cache = DualCache() + parallel_request_limiter = _PROXY_MaxParallelRequestsHandler_v3( + internal_usage_cache=InternalUsageCache(local_cache) + ) + rate_limiter = _PROXY_BatchRateLimiter( + internal_usage_cache=InternalUsageCache(local_cache), + parallel_request_limiter=parallel_request_limiter, + ) + + user = UserAPIKeyAuth( + api_key="sk-project-batch-itpm", + models=["*"], + project_id="proj-mantle-batch", + project_metadata={"model_itpm_limit": {"gpt-4o-mini": 1}}, + ) + + mock_content = MagicMock() + mock_content.content = ( + b'{"body": {"model": "gpt-4o-mini", "max_tokens": 1, ' + b'"messages": [{"role": "user", "content": "well over one token of input"}]}}\n' + ) + + with ( + patch("litellm.proxy.proxy_server.general_settings", {}), + patch("litellm.proxy.proxy_server.llm_router", MagicMock()), + patch( + "litellm.proxy.openai_files_endpoints.common_utils.get_credentials_for_model", + return_value={"custom_llm_provider": "openai"}, + ), + patch("litellm.afile_content", new=AsyncMock(return_value=mock_content)), + ): + with pytest.raises(HTTPException) as exc: + await rate_limiter.async_pre_call_hook( + user_api_key_dict=user, + cache=local_cache, + data={"input_file_id": "file-abc123", "model": "gpt-4o-mini"}, + call_type="acreate_batch", + ) + + assert exc.value.status_code == 429 + assert "model_per_project_itpm" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_pre_call_enforces_project_otpm_limit_for_non_routing_row_model(): + """VERIA regression: project ITPM/OTPM descriptors were built only for the + file-bound/top-level routing model, so a caller could bind the batch file + to an unlimited model while a JSONL row's own `body.model` named a + different, quota-limited model. That row's tokens must still be charged + against its own model's project OTPM quota.""" + from litellm import DualCache + from litellm.proxy.hooks.batch_rate_limiter import _PROXY_BatchRateLimiter + from litellm.proxy.hooks.parallel_request_limiter_v3 import ( + _PROXY_MaxParallelRequestsHandler_v3, + ) + from litellm.proxy.utils import InternalUsageCache + + local_cache = DualCache() + parallel_request_limiter = _PROXY_MaxParallelRequestsHandler_v3( + internal_usage_cache=InternalUsageCache(local_cache) + ) + rate_limiter = _PROXY_BatchRateLimiter( + internal_usage_cache=InternalUsageCache(local_cache), + parallel_request_limiter=parallel_request_limiter, + ) + + # The routing model ("unlimited-model") has no configured quota; only + # "quota-limited-model" -- named inside the JSONL row, not the routing + # model -- has a project OTPM limit. + user = UserAPIKeyAuth( + api_key="sk-project-batch-cross-model", + models=["*"], + project_id="proj-mantle-batch", + project_metadata={"model_otpm_limit": {"quota-limited-model": 50}}, + ) + + mock_content = MagicMock() + mock_content.content = ( + b'{"body": {"model": "quota-limited-model", "max_tokens": 80, ' + b'"messages": [{"role": "user", "content": "hi"}]}}\n' + ) + + with ( + patch("litellm.proxy.proxy_server.general_settings", {}), + patch("litellm.proxy.proxy_server.llm_router", MagicMock()), + patch( + "litellm.proxy.openai_files_endpoints.common_utils.get_credentials_for_model", + return_value={"custom_llm_provider": "openai"}, + ), + patch("litellm.afile_content", new=AsyncMock(return_value=mock_content)), + ): + with pytest.raises(HTTPException) as exc: + await rate_limiter.async_pre_call_hook( + user_api_key_dict=user, + cache=local_cache, + data={"input_file_id": "file-abc123", "model": "unlimited-model"}, + call_type="acreate_batch", + ) + + assert exc.value.status_code == 429 + assert "model_per_project_otpm" in str(exc.value.detail) + assert "quota-limited-model" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_pre_call_charges_each_row_model_against_its_own_project_quota(): + """A batch whose rows target two different project-quota-limited models + must charge each row's tokens only against its own model's quota, never + the other model's or the whole batch's combined total. The under-limit + model's request must succeed even though the over-limit model's row + would fail on its own.""" + from litellm import DualCache + from litellm.proxy.hooks.batch_rate_limiter import _PROXY_BatchRateLimiter + from litellm.proxy.hooks.parallel_request_limiter_v3 import ( + PROJECT_OTPM_DESCRIPTOR_KEY, + _PROXY_MaxParallelRequestsHandler_v3, + ) + from litellm.proxy.utils import InternalUsageCache + + local_cache = DualCache() + parallel_request_limiter = _PROXY_MaxParallelRequestsHandler_v3( + internal_usage_cache=InternalUsageCache(local_cache) + ) + rate_limiter = _PROXY_BatchRateLimiter( + internal_usage_cache=InternalUsageCache(local_cache), + parallel_request_limiter=parallel_request_limiter, + ) + + user = UserAPIKeyAuth( + api_key="sk-project-batch-two-models", + models=["*"], + project_id="proj-mantle-batch", + project_metadata={ + "model_otpm_limit": {"model-a": 1000, "model-b": 10}, + }, + ) + + # model-a stays comfortably under its 1000 OTPM limit; model-b's single + # row alone exceeds its 10 OTPM limit. If the two were combined into one + # counter (the pre-fix behavior for the routing model), model-a's ample + # headroom would mask model-b's overage. + mock_content = MagicMock() + mock_content.content = ( + b'{"body": {"model": "model-a", "max_tokens": 5, ' + b'"messages": [{"role": "user", "content": "hi"}]}}\n' + b'{"body": {"model": "model-b", "max_tokens": 40, ' + b'"messages": [{"role": "user", "content": "hi"}]}}\n' + ) + + with ( + patch("litellm.proxy.proxy_server.general_settings", {}), + patch("litellm.proxy.proxy_server.llm_router", MagicMock()), + patch( + "litellm.proxy.openai_files_endpoints.common_utils.get_credentials_for_model", + return_value={"custom_llm_provider": "openai"}, + ), + patch("litellm.afile_content", new=AsyncMock(return_value=mock_content)), + ): + with pytest.raises(HTTPException) as exc: + await rate_limiter.async_pre_call_hook( + user_api_key_dict=user, + cache=local_cache, + data={"input_file_id": "file-abc123", "model": "model-a"}, + call_type="acreate_batch", + ) + + assert exc.value.status_code == 429 + assert "model-b" in str(exc.value.detail) + + # model-a's own counter was not touched by model-b's rejection: a + # follow-up model-a-only batch well within its own limit must still pass. + model_a_status = await parallel_request_limiter.should_rate_limit( + descriptors=[ + { + "key": PROJECT_OTPM_DESCRIPTOR_KEY, + "value": "proj-mantle-batch:model-a", + "rate_limit": { + "requests_per_unit": None, + "tokens_per_unit": 1000, + "window_size": parallel_request_limiter.window_size, + }, + } + ], + read_only=True, + ) + assert model_a_status["overall_code"] == "OK" + + +def test_should_not_skip_when_project_has_io_limit_for_non_routing_model(): + """The no-limits skip must not fire just because the file-bound/top-level + routing model itself has no configured quota: a JSONL row can name a + different model that the project *does* quota, and that isn't knowable + without downloading and parsing the file.""" + rate_limiter = _make_rate_limiter() + # No key/team/model-level limits at all -- only a project OTPM limit for a + # model unrelated to the routing model below. + rate_limiter.parallel_request_limiter._create_rate_limit_descriptors.return_value = [ + {"rate_limit": {}} + ] + user = UserAPIKeyAuth( + api_key="sk", + models=["*"], + project_id="proj-mantle-batch", + project_metadata={"model_otpm_limit": {"some-other-model": 50}}, + ) + with patch("litellm.proxy.proxy_server.general_settings", {}): + should_skip, descriptors = rate_limiter._should_skip_batch_input_file_processing( + data={"model": "unlimited-model", "input_file_id": "file-abc"}, + user_api_key_dict=user, + ) + assert should_skip is False + assert descriptors is not None + + +def test_should_skip_when_project_has_no_io_limits_and_no_other_limits(): + """Sanity check for the new project-limits carve-out: a project caller + with no ITPM/OTPM configuration anywhere must still get the fast-path + skip when no other rate limits apply, exactly as before this fix.""" + rate_limiter = _make_rate_limiter() + rate_limiter.parallel_request_limiter._create_rate_limit_descriptors.return_value = [ + {"rate_limit": {}} + ] + user = UserAPIKeyAuth( + api_key="sk", + models=["*"], + project_id="proj-mantle-batch", + project_metadata={}, + ) + with patch("litellm.proxy.proxy_server.general_settings", {}): + should_skip, descriptors = rate_limiter._should_skip_batch_input_file_processing( + data={"model": "unlimited-model", "input_file_id": "file-abc"}, + user_api_key_dict=user, + ) + assert should_skip is True + assert descriptors is None + + @pytest.mark.asyncio async def test_count_input_file_usage_raises_on_non_bytes_content(): from litellm.proxy.hooks.batch_rate_limiter import _PROXY_BatchRateLimiter @@ -1671,3 +1980,117 @@ async def test_count_input_file_usage_collects_models_after_malformed_line(): ) assert exc.value.status_code == 403 + + +# --------------------------------------------------------------------------- +# VERIA-Low regression: Responses batch rows must not bypass project OTPM +# --------------------------------------------------------------------------- + + +def _output_estimator(): + """A `_PROXY_BatchRateLimiter` whose output-token floor is observable: + the no-`max_tokens` floor mock returns a distinctive sentinel so tests can + tell "floor was used" apart from "an explicit cap was read".""" + from litellm.proxy.hooks.batch_rate_limiter import _PROXY_BatchRateLimiter + from litellm.proxy.hooks.parallel_request_limiter_v3 import ( + _PROXY_MaxParallelRequestsHandler_v3, + ) + + limiter = MagicMock() + limiter.no_max_tokens_output_floor.return_value = 999 + limiter.get_output_candidate_count = _PROXY_MaxParallelRequestsHandler_v3.get_output_candidate_count + return _PROXY_BatchRateLimiter( + internal_usage_cache=MagicMock(), + parallel_request_limiter=limiter, + ) + + +def test_estimate_entry_output_tokens_zero_for_embeddings_url(): + """A real `/v1/embeddings` row reserves zero output tokens.""" + rate_limiter = _output_estimator() + entry = { + "url": "/v1/embeddings", + "body": {"model": "text-embedding-3-small", "input": "hello world"}, + } + + assert rate_limiter._estimate_entry_output_tokens(entry, None) == 0 + + +def test_estimate_entry_output_tokens_does_not_zero_responses_row_with_input(): + """Pre-fix: a `/v1/responses` row carries `body.input` with no `messages`/ + `prompt`, so the old body-shape heuristic misclassified it as embeddings + and reserved zero output tokens -- a project caller could submit large + Responses generations against a quota-limited model without consuming + OTPM. The row's own `url` (not body shape) must decide this.""" + rate_limiter = _output_estimator() + entry = { + "url": "/v1/responses", + "body": {"model": "gpt-4o", "input": "write me an essay"}, + } + + # No explicit cap on the row, so it must fall back to the no-max-tokens + # floor -- never straight to zero. + assert rate_limiter._estimate_entry_output_tokens(entry, None) == 999 + rate_limiter.parallel_request_limiter.no_max_tokens_output_floor.assert_called_once_with(None) + + +def test_estimate_entry_output_tokens_uses_max_output_tokens_for_responses(): + """`/v1/responses` caps output with `max_output_tokens`, not `max_tokens`/ + `max_completion_tokens`. Pre-fix this field was never inspected, so a + capped Responses row still fell through to the (possibly larger) floor + estimate instead of the caller's own declared cap.""" + rate_limiter = _output_estimator() + entry = { + "url": "/v1/responses", + "body": {"model": "gpt-4o", "input": "hi", "max_output_tokens": 123}, + } + + assert rate_limiter._estimate_entry_output_tokens(entry, None) == 123 + rate_limiter.parallel_request_limiter.no_max_tokens_output_floor.assert_not_called() + + +def test_estimate_entry_output_tokens_prefers_max_tokens_over_max_output_tokens(): + """When a row somehow carries both fields, the chat-style cap wins first -- + `max_output_tokens` is only consulted once the chat-style caps are absent.""" + rate_limiter = _output_estimator() + entry = { + "url": "/v1/chat/completions", + "body": { + "model": "gpt-4o", + "messages": [], + "max_tokens": 50, + "max_output_tokens": 500, + }, + } + + assert rate_limiter._estimate_entry_output_tokens(entry, None) == 50 + + +@pytest.mark.parametrize( + ("body_extra", "expected"), + [ + ({"max_tokens": 40, "n": 10}, 400), + ({"max_tokens": 40, "best_of": 5}, 200), + ({"max_tokens": 40, "n": 3, "best_of": 5}, 200), + ({"max_tokens": 40, "n": 0}, 40), + ({"max_tokens": 40, "n": -2}, 40), + ({"max_tokens": 40, "n": 5.0}, 200), + ({"max_tokens": 40, "n": "10"}, 400), + ({"max_tokens": 40, "n": "not-a-number"}, 40), + ({"max_tokens": 40, "n": 1e309}, 40), + ({"max_tokens": 1e309, "n": 3}, 2997), + ({"n": 3}, 2997), + ], +) +def test_estimate_entry_output_tokens_multiplies_candidate_count(body_extra, expected): + """A row generating n / best_of candidates consumes that many completions' + worth of output tokens, so the OTPM reservation must scale with the + effective candidate count. Pre-fix a `max_tokens: 40, n: 10` row consumed + up to 400 output tokens while reserving only 40.""" + rate_limiter = _output_estimator() + entry = { + "url": "/v1/chat/completions", + "body": {"model": "gpt-4o", "messages": [], **body_extra}, + } + + assert rate_limiter._estimate_entry_output_tokens(entry, None) == expected diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index 194b9dcb217..54366226dfb 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -3116,6 +3116,192 @@ async def test_project_model_rate_limits_not_triggered_for_other_model_v3(): ), f"model_per_project should not be added for unrelated model, got: {descriptor_keys}" +@pytest.mark.asyncio +async def test_project_model_itpm_otpm_limits_enforced_v3(): + """ + Project-level model_itpm_limit/model_otpm_limit must produce distinct + Bedrock Mantle-style input and output token descriptors. + """ + _api_key = hash_token("sk-project-io-test") + local_cache = DualCache() + parallel_request_handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + + captured_descriptors = [] + + async def mock_should_rate_limit(descriptors, **kwargs): + captured_descriptors.extend(descriptors) + return {"overall_code": "OK", "statuses": []} + + parallel_request_handler.should_rate_limit = mock_should_rate_limit + + user_api_key_dict = UserAPIKeyAuth( + api_key=_api_key, + project_id="proj-mantle", + project_metadata={ + "model_itpm_limit": {"bedrock_mantle/claude-opus": 20000000}, + "model_otpm_limit": {"bedrock_mantle/claude-opus": 4000000}, + }, + ) + + await parallel_request_handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data={"model": "bedrock_mantle/claude-opus"}, + call_type="", + ) + + descriptor_keys = [d["key"] for d in captured_descriptors] + assert "model_per_project_itpm" in descriptor_keys + assert "model_per_project_otpm" in descriptor_keys + assert "model_per_project" not in descriptor_keys + + itpm_descriptor = next( + d for d in captured_descriptors if d["key"] == "model_per_project_itpm" + ) + otpm_descriptor = next( + d for d in captured_descriptors if d["key"] == "model_per_project_otpm" + ) + assert itpm_descriptor["value"] == "proj-mantle:bedrock_mantle/claude-opus" + assert itpm_descriptor["rate_limit"]["tokens_per_unit"] == 20000000 + assert otpm_descriptor["value"] == "proj-mantle:bedrock_mantle/claude-opus" + assert otpm_descriptor["rate_limit"]["tokens_per_unit"] == 4000000 + + +@pytest.mark.asyncio +async def test_project_model_itpm_otpm_limits_not_triggered_for_other_model_v3(): + """Split project limits must not apply to an unrelated model.""" + _api_key = hash_token("sk-project-io-test-2") + local_cache = DualCache() + parallel_request_handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + + captured_descriptors = [] + + async def mock_should_rate_limit(descriptors, **kwargs): + captured_descriptors.extend(descriptors) + return {"overall_code": "OK", "statuses": []} + + parallel_request_handler.should_rate_limit = mock_should_rate_limit + + user_api_key_dict = UserAPIKeyAuth( + api_key=_api_key, + project_id="proj-mantle", + project_metadata={ + "model_itpm_limit": {"bedrock_mantle/claude-opus": 20000000}, + }, + ) + + await parallel_request_handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data={"model": "gpt-4"}, + call_type="", + ) + + descriptor_keys = [d["key"] for d in captured_descriptors] + assert "model_per_project_itpm" not in descriptor_keys + assert "model_per_project_otpm" not in descriptor_keys + + +@pytest.mark.asyncio +async def test_project_model_itpm_and_tpm_limits_coexist_v3(): + """Combined project TPM and split ITPM/OTPM limits are enforced together.""" + _api_key = hash_token("sk-project-io-test-3") + local_cache = DualCache() + parallel_request_handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + + captured_descriptors = [] + + async def mock_should_rate_limit(descriptors, **kwargs): + captured_descriptors.extend(descriptors) + return {"overall_code": "OK", "statuses": []} + + parallel_request_handler.should_rate_limit = mock_should_rate_limit + + user_api_key_dict = UserAPIKeyAuth( + api_key=_api_key, + project_id="proj-mantle", + project_metadata={ + "model_tpm_limit": {"bedrock_mantle/claude-opus": 1000}, + "model_itpm_limit": {"bedrock_mantle/claude-opus": 20000000}, + "model_otpm_limit": {"bedrock_mantle/claude-opus": 4000000}, + }, + ) + + await parallel_request_handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data={"model": "bedrock_mantle/claude-opus"}, + call_type="", + ) + + descriptor_keys = [d["key"] for d in captured_descriptors] + assert "model_per_project" in descriptor_keys + assert "model_per_project_itpm" in descriptor_keys + assert "model_per_project_otpm" in descriptor_keys + + +@pytest.mark.asyncio +async def test_enforce_project_io_token_quota_for_frame_blocks_over_limit_otpm(): + """VERIA regression: the Responses WebSocket connection-level pre-call + hook only runs once, but a connection accepts many response.create + frames. enforce_project_io_token_quota_for_frame is the per-frame check + that closes that gap; it must reserve against the caller's project OTPM + limit and reject once a frame's estimated output tokens exceed it.""" + _api_key = hash_token("sk-ws-frame-otpm") + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + user_api_key_dict = UserAPIKeyAuth( + api_key=_api_key, + project_id="proj-mantle-ws", + project_metadata={"model_otpm_limit": {"gpt-4o": 50}}, + ) + + await handler.enforce_project_io_token_quota_for_frame( + user_api_key_dict=user_api_key_dict, + requested_model="gpt-4o", + estimated_input_tokens=1, + estimated_output_tokens=30, + ) + + with pytest.raises(HTTPException) as exc: + await handler.enforce_project_io_token_quota_for_frame( + user_api_key_dict=user_api_key_dict, + requested_model="gpt-4o", + estimated_input_tokens=1, + estimated_output_tokens=30, + ) + + assert exc.value.status_code == 429 + assert "model_per_project_otpm" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_enforce_project_io_token_quota_for_frame_noop_without_project_limits(): + """A key with no project ITPM/OTPM configured must never be blocked by + the per-frame check (no descriptors to reserve against).""" + _api_key = hash_token("sk-ws-frame-no-limits") + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + user_api_key_dict = UserAPIKeyAuth(api_key=_api_key) + + await handler.enforce_project_io_token_quota_for_frame( + user_api_key_dict=user_api_key_dict, + requested_model="gpt-4o", + estimated_input_tokens=10_000_000, + estimated_output_tokens=10_000_000, + ) + + @pytest.mark.asyncio async def test_pre_call_hook_keeps_internal_stash_out_of_request_body(): """Regression for #27001 / #35197: the limiter's per-request bookkeeping @@ -3192,7 +3378,7 @@ async def test_responses_route_body_untouched_by_pre_call_hook(caller_metadata): _api_key = hash_token("sk-responses-regression") user_api_key_dict = UserAPIKeyAuth( api_key=_api_key, - tpm_limit=1000, + tpm_limit=100000, rpm_limit=5, ) local_cache = DualCache() @@ -5264,7 +5450,7 @@ async def test_configured_estimate_does_not_apply_to_embeddings(monkeypatch): local_cache, user_api_key_dict, {"model": "text-embedding-3-small", "input": "hello"}, - call_type="embeddings", + call_type="embedding", ) assert reserved == ONE_TOKEN_PROMPT_INPUT_ESTIMATE diff --git a/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py b/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py index f7bd37b412a..8d03857c917 100644 --- a/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py +++ b/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py @@ -15,7 +15,7 @@ Redis. """ import asyncio -from datetime import datetime +from datetime import datetime, timedelta from typing import Any, Dict import pytest @@ -23,15 +23,25 @@ import pytest from litellm.caching.caching import DualCache from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.hooks.parallel_request_limiter_v3 import ( + PROJECT_ITPM_DESCRIPTOR_KEY, + PROJECT_OTPM_DESCRIPTOR_KEY, + _AUDIO_BYTES_PER_TOKEN, _PROXY_MaxParallelRequestsHandler_v3 as RateLimitHandler, ) from litellm.proxy.hooks.parallel_request_limiter_v3 import ( + _call_id_from_callback_kwargs, _request_stash, get_or_create_request_stash, get_request_stash, ) from litellm.proxy.utils import InternalUsageCache, hash_token -from litellm.types.utils import ModelResponse, Usage +from litellm.types.llms.openai import ( + InputTokensDetails, + ResponseAPIUsage, + ResponsesAPIResponse, +) +from litellm.types.rerank import RerankResponse +from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage @pytest.fixture @@ -582,6 +592,47 @@ async def test_estimate_tokens_uses_max_tokens_when_explicit(rate_limiter): assert estimate == 4 + 25 +@pytest.mark.asyncio +async def test_estimate_tokens_honors_explicit_zero_max_tokens(rate_limiter): + """ + Regression for a Greptile finding: explicit_max_tokens was resolved via + `data.get("max_tokens") or data.get("max_completion_tokens") or + data.get("max_output_tokens")`, so an explicit 0 in the first field was + falsy and fell through to the next field (or the no-max_tokens floor), + silently discarding a caller's explicit zero-output request. + """ + handler, _cache = rate_limiter + + estimate = handler._estimate_tokens_for_request( + data={ + "messages": [ + {"role": "user", "content": "abcd" * 4} + ], # 16 chars ~ 4 tokens + "max_tokens": 0, + } + ) + assert estimate == 4, ( + f"expected input-only reservation (4) for an explicit max_tokens=0, got {estimate}" + ) + + +@pytest.mark.asyncio +async def test_estimate_tokens_honors_explicit_zero_max_output_tokens_for_responses( + rate_limiter, +): + handler, _cache = rate_limiter + + estimate = handler._estimate_tokens_for_request( + data={ + "input": "describe this image in detail", # 29 chars ~ 7 tokens + "max_output_tokens": 0, + }, + min_configured_tpm_limit=40, + call_type="aresponses", + ) + assert estimate == 23 + + @pytest.mark.asyncio async def test_estimate_tokens_zero_for_empty_embeddings(rate_limiter): """Embeddings have no output budget — reservation should equal input only.""" @@ -1197,5 +1248,2427 @@ async def test_small_tpm_cap_preserves_explicit_max_tokens(rate_limiter): assert data["max_tokens"] == 500 +@pytest.mark.asyncio +async def test_project_otpm_reservation_prevents_concurrent_bypass(rate_limiter): + """ + Bedrock Mantle-style OTPM: with a 100 OTPM limit and 5 concurrent + requests each reserving 50+ output tokens, upfront reservation must + reject the late arrivals -- not let all 5 through. Exercises the + in-memory fallback in ``atomic_check_and_increment_by_n`` for the + project-scoped ITPM/OTPM descriptors specifically. + """ + handler, cache = rate_limiter + + user_api_key_dict = UserAPIKeyAuth( + api_key=hash_token("sk-otpm-bypass"), + project_id="proj-mantle-bypass", + project_metadata={ + "model_otpm_limit": {"bedrock_mantle/claude-opus": 100}, + }, + ) + + request_data = { + "model": "bedrock_mantle/claude-opus", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 50, + } + + async def make_request(request_id: int) -> Dict[str, Any]: + data = request_data.copy() + try: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data=data, + call_type="", + ) + return {"request_id": request_id, "success": True} + except Exception as e: + return { + "request_id": request_id, + "success": False, + "status_code": getattr(e, "status_code", None), + } + + results = await asyncio.gather(*[make_request(i) for i in range(5)]) + + successful = [r for r in results if r["success"]] + rate_limited = [ + r for r in results if not r["success"] and r.get("status_code") == 429 + ] + + assert len(rate_limited) > 0, ( + f"Expected some OTPM-rate-limited requests but all {len(successful)} succeeded." + ) + + +@pytest.mark.asyncio +async def test_project_otpm_rejects_multiple_completion_candidates(rate_limiter): + handler, cache = rate_limiter + user_api_key_dict = UserAPIKeyAuth( + api_key=hash_token("sk-otpm-multiple-candidates"), + project_id="proj-multiple-candidates", + project_metadata={ + "model_otpm_limit": {"bedrock_mantle/claude-opus": 500}, + }, + ) + data = { + "model": "bedrock_mantle/claude-opus", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 100, + "n": 10, + } + + with pytest.raises(Exception) as exc_info: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data=data, + call_type="acompletion", + ) + + assert getattr(exc_info.value, "status_code", None) == 429 + + +@pytest.mark.asyncio +async def test_project_otpm_reserves_largest_conflicting_output_cap(rate_limiter): + handler, cache = rate_limiter + user_api_key_dict = UserAPIKeyAuth( + api_key=hash_token("sk-otpm-conflicting-caps"), + project_id="proj-conflicting-caps", + project_metadata={ + "model_otpm_limit": {"bedrock_mantle/claude-opus": 50}, + }, + ) + data = { + "model": "bedrock_mantle/claude-opus", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 1, + "max_completion_tokens": 100, + } + + with pytest.raises(Exception) as exc_info: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data=data, + call_type="acompletion", + ) + + assert getattr(exc_info.value, "status_code", None) == 429 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "call_type", + ["agenerate_content", "agenerate_content_stream"], +) +@pytest.mark.parametrize("config_field", ["config", "generationConfig"]) +async def test_project_otpm_rejects_google_genai_native_output_cap( + rate_limiter, + call_type, + config_field, +): + handler, cache = rate_limiter + model = "gemini/gemini-3-flash-preview" + user_api_key_dict = UserAPIKeyAuth( + api_key=hash_token("sk-google-genai-native-otpm"), + project_id="project-google-genai-native-otpm", + project_metadata={"model_otpm_limit": {model: 50}}, + ) + + with pytest.raises(Exception) as exc_info: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data={ + "model": model, + "contents": [{"role": "user", "parts": [{"text": "Hello"}]}], + config_field: {"maxOutputTokens": 100}, + }, + call_type=call_type, + ) + + assert getattr(exc_info.value, "status_code", None) == 429 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "call_type", + ["agenerate_content", "agenerate_content_stream"], +) +@pytest.mark.parametrize("candidate_count_field", ["candidateCount", "candidate_count"]) +async def test_project_otpm_rejects_google_genai_native_candidate_count( + rate_limiter, + call_type, + candidate_count_field, +): + handler, cache = rate_limiter + model = "gemini/gemini-3-flash-preview" + user_api_key_dict = UserAPIKeyAuth( + api_key=hash_token("sk-google-genai-native-candidate-count"), + project_id="project-google-genai-native-candidate-count", + project_metadata={"model_otpm_limit": {model: 150}}, + ) + + with pytest.raises(Exception) as exc_info: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data={ + "model": model, + "contents": [{"role": "user", "parts": [{"text": "Hello"}]}], + "config": { + "maxOutputTokens": 50, + candidate_count_field: 4, + }, + }, + call_type=call_type, + ) + + assert getattr(exc_info.value, "status_code", None) == 429 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "call_type", + ["agenerate_content", "agenerate_content_stream"], +) +@pytest.mark.parametrize("config_field", [None, "config", "generationConfig"]) +async def test_project_otpm_injects_google_genai_native_output_cap( + rate_limiter, + call_type, + config_field, +): + handler, cache = rate_limiter + model = "gemini/gemini-3-flash-preview" + user_api_key_dict = UserAPIKeyAuth( + api_key=hash_token("sk-google-genai-native-implicit-otpm"), + project_id="project-google-genai-native-implicit-otpm", + project_metadata={"model_otpm_limit": {model: 40}}, + ) + data = { + "model": model, + "contents": [{"role": "user", "parts": [{"text": "Hello"}]}], + } + if config_field is not None: + data[config_field] = {"temperature": 0} + + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data=data, + call_type=call_type, + ) + + stash = get_request_stash() + assert stash is not None + assert stash.otpm_reserved_tokens == 10 + expected_config_field = config_field or "config" + assert data[expected_config_field]["maxOutputTokens"] == 10 + assert "max_tokens" not in data + + +@pytest.mark.asyncio +async def test_project_otpm_over_limit_rolls_back_itpm_reservation(rate_limiter): + """ + When ITPM reserves fine but OTPM is then over limit, the ITPM + reservation this same pre-call already made must be rolled back -- + otherwise it leaks until the window's TTL, silently shrinking the ITPM + budget for every other request in that minute. + """ + handler, cache = rate_limiter + + user_api_key_dict = UserAPIKeyAuth( + api_key=hash_token("sk-otpm-rollback"), + project_id="proj-mantle-rollback", + project_metadata={ + "model_itpm_limit": {"bedrock_mantle/claude-opus": 1000000}, + "model_otpm_limit": {"bedrock_mantle/claude-opus": 10}, + }, + ) + + itpm_counter_key = handler.create_rate_limit_keys( + key="model_per_project_itpm", + value="proj-mantle-rollback:bedrock_mantle/claude-opus", + rate_limit_type="tokens", + ) + + data = { + "model": "bedrock_mantle/claude-opus", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 500, # blows past the 10-token OTPM limit + } + + with pytest.raises(Exception) as exc_info: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data=data, + call_type="", + ) + assert getattr(exc_info.value, "status_code", None) == 429 + + cached_value = await cache.async_get_cache(key=itpm_counter_key, local_only=True) + assert int(cached_value or 0) == 0, ( + f"ITPM reservation leaked after OTPM rejection: counter={cached_value}" + ) + + +@pytest.mark.asyncio +async def test_project_itpm_reconciled_on_success_excludes_cached_tokens(rate_limiter): + """ + On success, ITPM reconciles to billable input tokens (prompt_tokens + minus cached_tokens) -- not raw prompt_tokens. Cached prompt-read tokens + are free under Bedrock Mantle and must not count against the ITPM quota, + even though they still appear in usage/cost logging elsewhere. + """ + handler, _cache = rate_limiter + + itpm_scope = ("model_per_project_itpm", "proj-mantle:model") + otpm_scope = ("model_per_project_otpm", "proj-mantle:model") + + stash = get_or_create_request_stash() + stash.itpm_reserved_tokens = 100 + stash.itpm_reserved_scopes = frozenset({itpm_scope}) + stash.otpm_reserved_tokens = 60 + stash.otpm_reserved_scopes = frozenset({otpm_scope}) + mock_kwargs = {} + + mock_response = ModelResponse( + id="test", + object="chat.completion", + created=int(datetime.now().timestamp()), + model="bedrock_mantle/claude-opus", + usage=Usage( + prompt_tokens=80, + completion_tokens=40, + total_tokens=120, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=30), + ), + choices=[], + ) + + increments = [] + + async def mock_increment(increment_list, **kwargs): + for op in increment_list: + increments.append({"key": op["key"], "increment": op["increment_value"]}) + + handler.internal_usage_cache.dual_cache.async_increment_cache_pipeline = ( + mock_increment + ) + + await handler.async_log_success_event( + kwargs=mock_kwargs, + response_obj=mock_response, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + itpm_adjustments = [i for i in increments if "model_per_project_itpm" in i["key"]] + otpm_adjustments = [i for i in increments if "model_per_project_otpm" in i["key"]] + + # billable_input = 80 - 30 cached = 50; delta = 50 - 100 reserved = -50 + assert any(i["increment"] == -50 for i in itpm_adjustments), ( + f"Expected a -50 ITPM adjustment (50 billable - 100 reserved), got: {itpm_adjustments}" + ) + # delta = 40 actual completion - 60 reserved = -20 + assert any(i["increment"] == -20 for i in otpm_adjustments), ( + f"Expected a -20 OTPM adjustment (40 actual - 60 reserved), got: {otpm_adjustments}" + ) + + +@pytest.mark.asyncio +async def test_project_reconciliation_does_not_decrement_later_window(): + current_time = datetime(2026, 8, 5, 12, 0, 0) + cache = DualCache() + handler = RateLimitHandler( + internal_usage_cache=InternalUsageCache(cache), + time_provider=lambda: current_time, + ) + handler.window_size = 60 + scope = (PROJECT_ITPM_DESCRIPTOR_KEY, "project:model") + descriptor = { + "key": scope[0], + "value": scope[1], + "rate_limit": {"tokens_per_unit": 1000, "window_size": 60}, + } + + reservation = await handler.atomic_check_and_increment_by_n( + descriptors=[descriptor], + increments=[{"tokens": 100}], + ) + counter_key = handler.create_rate_limit_keys(*scope, rate_limit_type="tokens") + window_identity = next( + identity + for identity in reservation["reservation_windows"] + if identity[0] == counter_key + ) + stash = get_or_create_request_stash() + stash.itpm_reserved_tokens = 100 + stash.itpm_reserved_scopes = frozenset({scope}) + stash.itpm_reserved_window_identities = frozenset( + {window_identity} + ) + + current_time += timedelta(seconds=61) + later_reservation = await handler.atomic_check_and_increment_by_n( + descriptors=[descriptor], + increments=[{"tokens": 20}], + ) + assert window_identity not in later_reservation["reservation_windows"] + + await handler.async_log_success_event( + kwargs={}, + response_obj=ModelResponse( + usage=Usage(prompt_tokens=10, completion_tokens=0, total_tokens=10) + ), + start_time=current_time, + end_time=current_time, + ) + + assert float(await cache.async_get_cache(key=counter_key, local_only=True) or 0) == 20 + + +@pytest.mark.asyncio +async def test_project_reconciliation_decrements_its_active_window(rate_limiter): + handler, cache = rate_limiter + scope = (PROJECT_ITPM_DESCRIPTOR_KEY, "project:model") + descriptor = { + "key": scope[0], + "value": scope[1], + "rate_limit": {"tokens_per_unit": 1000, "window_size": 60}, + } + reservation = await handler.atomic_check_and_increment_by_n( + descriptors=[descriptor], + increments=[{"tokens": 100}], + ) + counter_key = handler.create_rate_limit_keys(*scope, rate_limit_type="tokens") + window_identity = next( + identity + for identity in reservation["reservation_windows"] + if identity[0] == counter_key + ) + stash = get_or_create_request_stash() + stash.itpm_reserved_tokens = 100 + stash.itpm_reserved_scopes = frozenset({scope}) + stash.itpm_reserved_window_identities = frozenset( + {window_identity} + ) + + await handler.async_log_success_event( + kwargs={}, + response_obj=ModelResponse( + usage=Usage(prompt_tokens=10, completion_tokens=0, total_tokens=10) + ), + start_time=datetime.now(), + end_time=datetime.now(), + ) + + assert float(await cache.async_get_cache(key=counter_key, local_only=True) or 0) == 10 + + +@pytest.mark.asyncio +async def test_redis_window_guard_uses_reservation_identity_and_never_falls_back_negative( + rate_limiter, +): + handler, _cache = rate_limiter + calls = [] + + async def failing_guard(*, keys, args): + calls.append((keys, args)) + raise RuntimeError("redis unavailable") + + unguarded_calls = [] + + async def capture_unguarded(pipeline_operations, **_kwargs): + unguarded_calls.extend(pipeline_operations) + + handler.window_guarded_token_increment_script = failing_guard + handler.async_increment_tokens_with_ttl_preservation = capture_unguarded + await handler.async_increment_reservation_aware_tokens( + pipeline_operations=[ + { + "key": "{model_per_project_itpm:project:model}:tokens", + "increment_value": -90, + "ttl": 60, + "window_key": "{model_per_project_itpm:project:model}:window", + "expected_window_start": "1234", + "reservation_backend": "redis", + } + ] + ) + + assert calls == [ + ( + [ + "{model_per_project_itpm:project:model}:window", + "{model_per_project_itpm:project:model}:tokens", + ], + ["1234", -90, 60], + ) + ] + assert unguarded_calls == [] + + +@pytest.mark.asyncio +async def test_atomic_lua_response_carries_redis_window_identity(rate_limiter): + handler, _cache = rate_limiter + counter_key = "{model_per_project_itpm:project:model}:tokens" + meta = [ + { + "descriptor_key": PROJECT_ITPM_DESCRIPTOR_KEY, + "descriptor_value": "project:model", + "current_limit": 100, + "rate_limit_type": "tokens", + "counter_key": counter_key, + } + ] + + async def successful_reservation(*, keys, args): + return [0, 25, 1234] + + handler.check_and_increment_by_n_script = successful_reservation + assert await handler._atomic_lua_per_descriptor([]) == { + "overall_code": "OK", + "statuses": [], + } + + response = await handler._atomic_lua_per_descriptor( + descriptor_groups=[ + ( + [ + "{model_per_project_itpm:project:model}:window", + counter_key, + ], + [100, 25, 60, 60], + meta, + ) + ] + ) + + assert response["statuses"][0]["limit_remaining"] == 75 + assert response["reservation_windows"] == frozenset( + {(counter_key, "1234", "redis")} + ) + + +@pytest.mark.asyncio +async def test_project_itpm_otpm_released_on_failure(rate_limiter): + """On failure, the full ITPM and OTPM reservations must be refunded.""" + handler, _cache = rate_limiter + + itpm_scope = ("model_per_project_itpm", "proj-mantle:model") + otpm_scope = ("model_per_project_otpm", "proj-mantle:model") + + stash = get_or_create_request_stash() + stash.itpm_reserved_tokens = 100 + stash.itpm_reserved_scopes = frozenset({itpm_scope}) + stash.otpm_reserved_tokens = 60 + stash.otpm_reserved_scopes = frozenset({otpm_scope}) + mock_kwargs = {} + + increments = [] + + async def mock_increment(increment_list, **kwargs): + for op in increment_list: + increments.append({"key": op["key"], "increment": op["increment_value"]}) + + handler.internal_usage_cache.dual_cache.async_increment_cache_pipeline = ( + mock_increment + ) + + await handler.async_log_failure_event( + kwargs=mock_kwargs, + response_obj=None, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + itpm_releases = [i for i in increments if "model_per_project_itpm" in i["key"]] + otpm_releases = [i for i in increments if "model_per_project_otpm" in i["key"]] + + assert any(i["increment"] == -100 for i in itpm_releases), itpm_releases + assert any(i["increment"] == -60 for i in otpm_releases), otpm_releases + + +@pytest.mark.asyncio +async def test_proxy_rejection_refunds_itpm_otpm_by_their_own_amount_not_combined( + rate_limiter, +): + """ + Regression for a Greptile-flagged bug: when a project configures both a + combined model_tpm_limit and split model_itpm_limit/model_otpm_limit for + the same model, async_post_call_failure_hook's proxy-side refund path + used to decrement every token descriptor -- including the ITPM/OTPM + ones -- by the flat combined reservation amount, instead of each + bucket's own reserved amount. That drives the split counters negative + (or under-refunds them) instead of returning them to exactly zero. + """ + handler, cache = rate_limiter + + api_key = hash_token("sk-mixed-tpm-io") + user_api_key_dict = UserAPIKeyAuth( + api_key=api_key, + project_id="proj-mixed", + project_metadata={ + "model_tpm_limit": {"bedrock_mantle/claude-opus": 100000}, + "model_itpm_limit": {"bedrock_mantle/claude-opus": 100000}, + "model_otpm_limit": {"bedrock_mantle/claude-opus": 100000}, + }, + ) + + data = { + "model": "bedrock_mantle/claude-opus", + "messages": [ + {"role": "user", "content": "hello there, this is a test message"} + ], + "max_tokens": 60, + } + + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data=data, + call_type="", + ) + + tpm_counter_key = handler.create_rate_limit_keys( + key="model_per_project", + value="proj-mixed:bedrock_mantle/claude-opus", + rate_limit_type="tokens", + ) + itpm_counter_key = handler.create_rate_limit_keys( + key="model_per_project_itpm", + value="proj-mixed:bedrock_mantle/claude-opus", + rate_limit_type="tokens", + ) + otpm_counter_key = handler.create_rate_limit_keys( + key="model_per_project_otpm", + value="proj-mixed:bedrock_mantle/claude-opus", + rate_limit_type="tokens", + ) + + tpm_reserved = int( + await cache.async_get_cache(key=tpm_counter_key, local_only=True) or 0 + ) + itpm_reserved = int( + await cache.async_get_cache(key=itpm_counter_key, local_only=True) or 0 + ) + otpm_reserved = int( + await cache.async_get_cache(key=otpm_counter_key, local_only=True) or 0 + ) + assert tpm_reserved > 0 and itpm_reserved > 0 and otpm_reserved > 0 + + await handler.async_post_call_failure_hook( + request_data=data, + original_exception=Exception("guardrail rejected"), + user_api_key_dict=user_api_key_dict, + ) + + tpm_after = int( + await cache.async_get_cache(key=tpm_counter_key, local_only=True) or 0 + ) + itpm_after = int( + await cache.async_get_cache(key=itpm_counter_key, local_only=True) or 0 + ) + otpm_after = int( + await cache.async_get_cache(key=otpm_counter_key, local_only=True) or 0 + ) + + assert tpm_after == 0, f"combined TPM counter leaked: {tpm_after}" + assert itpm_after == 0, ( + f"ITPM counter corrupted by combined-amount refund: {itpm_after}" + ) + assert otpm_after == 0, ( + f"OTPM counter corrupted by combined-amount refund: {otpm_after}" + ) + + +@pytest.mark.asyncio +async def test_proxy_rejection_refunds_itpm_otpm_only_reservation_with_no_combined_tpm( + rate_limiter, +): + """ + Regression for the second half of the same bug: with only + model_itpm_limit/model_otpm_limit configured (no model_tpm_limit), the + combined reserved_tokens is 0, and the proxy-side refund path used to + return immediately on that -- leaking the ITPM/OTPM reservations until + the rate-limit window's TTL expired. + """ + handler, cache = rate_limiter + + api_key = hash_token("sk-io-only") + user_api_key_dict = UserAPIKeyAuth( + api_key=api_key, + project_id="proj-io-only", + project_metadata={ + "model_itpm_limit": {"bedrock_mantle/claude-opus": 100000}, + "model_otpm_limit": {"bedrock_mantle/claude-opus": 100000}, + }, + ) + + data = { + "model": "bedrock_mantle/claude-opus", + "messages": [ + {"role": "user", "content": "hello there, this is a test message"} + ], + "max_tokens": 60, + } + + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data=data, + call_type="", + ) + + itpm_counter_key = handler.create_rate_limit_keys( + key="model_per_project_itpm", + value="proj-io-only:bedrock_mantle/claude-opus", + rate_limit_type="tokens", + ) + otpm_counter_key = handler.create_rate_limit_keys( + key="model_per_project_otpm", + value="proj-io-only:bedrock_mantle/claude-opus", + rate_limit_type="tokens", + ) + assert ( + int(await cache.async_get_cache(key=itpm_counter_key, local_only=True) or 0) > 0 + ) + assert ( + int(await cache.async_get_cache(key=otpm_counter_key, local_only=True) or 0) > 0 + ) + + await handler.async_post_call_failure_hook( + request_data=data, + original_exception=Exception("guardrail rejected"), + user_api_key_dict=user_api_key_dict, + ) + + itpm_after = int( + await cache.async_get_cache(key=itpm_counter_key, local_only=True) or 0 + ) + otpm_after = int( + await cache.async_get_cache(key=otpm_counter_key, local_only=True) or 0 + ) + assert itpm_after == 0, ( + f"ITPM-only reservation leaked on proxy rejection: {itpm_after}" + ) + assert otpm_after == 0, ( + f"OTPM-only reservation leaked on proxy rejection: {otpm_after}" + ) + + +@pytest.mark.asyncio +async def test_otpm_rejection_does_not_double_refund_combined_tpm(rate_limiter): + """ + Regression for a High-severity review finding: when the project ITPM + reservation succeeds but OTPM is then over limit, + _reserve_project_io_tokens_or_raise rolls back the combined-TPM + reservation that already succeeded earlier in the same pre-call, then + raises. If it doesn't also mark that reservation released, + async_post_call_failure_hook -- which fires next in the real request + lifecycle, since raising from async_pre_call_hook triggers it -- sees + the same still-stashed reservation and refunds it a second time, + driving the combined TPM counter negative and letting a caller push + past the project's real TPM budget by repeatedly triggering OTPM + rejections. + """ + handler, cache = rate_limiter + + api_key = hash_token("sk-double-refund") + user_api_key_dict = UserAPIKeyAuth( + api_key=api_key, + project_id="proj-double-refund", + project_metadata={ + "model_tpm_limit": {"bedrock_mantle/claude-opus": 100000}, + "model_itpm_limit": {"bedrock_mantle/claude-opus": 100000}, + "model_otpm_limit": {"bedrock_mantle/claude-opus": 5}, + }, + ) + + data = { + "model": "bedrock_mantle/claude-opus", + "messages": [ + {"role": "user", "content": "hello there, this is a test message"} + ], + "max_tokens": 60, # blows past the 5-token OTPM limit + } + + tpm_counter_key = handler.create_rate_limit_keys( + key="model_per_project", + value="proj-double-refund:bedrock_mantle/claude-opus", + rate_limit_type="tokens", + ) + + with pytest.raises(Exception) as exc_info: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data=data, + call_type="", + ) + assert getattr(exc_info.value, "status_code", None) == 429 + + tpm_after_pre_call = int( + await cache.async_get_cache(key=tpm_counter_key, local_only=True) or 0 + ) + assert tpm_after_pre_call == 0, ( + f"combined TPM reservation not rolled back: {tpm_after_pre_call}" + ) + + # In the real request lifecycle, async_post_call_failure_hook fires next + # for a pre-call rejection. It must not refund the same reservation again. + await handler.async_post_call_failure_hook( + request_data=data, + original_exception=exc_info.value, + user_api_key_dict=user_api_key_dict, + ) + + tpm_after_failure_hook = int( + await cache.async_get_cache(key=tpm_counter_key, local_only=True) or 0 + ) + assert tpm_after_failure_hook == 0, ( + f"combined TPM counter went negative from a double refund: {tpm_after_failure_hook}" + ) + + +@pytest.mark.parametrize( + "embedding_input", + [ + list(range(51)), + [list(range(25)), list(range(26))], + ], +) +@pytest.mark.asyncio +async def test_project_itpm_rejects_pretokenized_embedding_input( + rate_limiter, + embedding_input, +): + handler, cache = rate_limiter + user_api_key_dict = UserAPIKeyAuth( + api_key=hash_token("sk-pretokenized-embedding-itpm"), + project_id="proj-pretokenized-embedding", + project_metadata={ + "model_itpm_limit": {"text-embedding-3-small": 50}, + }, + ) + data = { + "model": "text-embedding-3-small", + "input": embedding_input, + } + + with pytest.raises(Exception) as exc_info: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data=data, + call_type="aembedding", + ) + + assert getattr(exc_info.value, "status_code", None) == 429 + + +@pytest.mark.asyncio +async def test_responses_api_not_misclassified_as_embedding_for_output_estimate( + rate_limiter, +): + """ + Regression for a High-severity review finding: the Responses API also + puts its prompt in data["input"], the same field embeddings use, so the + output-token estimate treated every Responses call as an embedding and + reserved zero output tokens. call_type now disambiguates the two: the + same input-only payload gets zero output tokens for an embedding call + but a real floor for a Responses API call. + """ + handler, _cache = rate_limiter + + data = {"input": "describe this image in detail"} + + _, embedding_output_estimate = handler._estimate_input_and_output_tokens( + data=data, call_type="aembedding" + ) + assert embedding_output_estimate == 0 + + _, responses_output_estimate = handler._estimate_input_and_output_tokens( + data=data, call_type="aresponses" + ) + assert responses_output_estimate > 0, ( + "Responses API call was misclassified as an embedding and reserved zero output tokens" + ) + + +@pytest.mark.parametrize( + ("data", "call_type", "expected_output_tokens"), + [ + ( + { + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 100, + "n": 10, + }, + "acompletion", + 1000, + ), + ( + { + "prompt": "hello", + "max_tokens": 100, + "n": 2, + "best_of": 5, + }, + "text_completion", + 500, + ), + ( + { + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 100, + "n": 0, + "best_of": "invalid", + }, + "acompletion", + 100, + ), + ], +) +def test_output_estimate_accounts_for_completion_candidates( + rate_limiter, + data, + call_type, + expected_output_tokens, +): + handler, _cache = rate_limiter + + _, estimated_output_tokens = handler._estimate_input_and_output_tokens( + data=data, + call_type=call_type, + ) + + assert estimated_output_tokens == expected_output_tokens + + +@pytest.mark.asyncio +async def test_responses_api_usage_reconciles_using_input_output_tokens_fields( + rate_limiter, +): + """ + Regression for the other half of the same finding: ResponseAPIUsage + exposes input_tokens/output_tokens, not prompt_tokens/completion_tokens. + Before this fix, _resolve_io_token_reconcile_usage couldn't resolve + Responses API usage at all, so the reservation was silently kept as-is + instead of being trued up to the much larger actual usage. + """ + handler, _cache = rate_limiter + + itpm_scope = ("model_per_project_itpm", "proj-responses:model") + otpm_scope = ("model_per_project_otpm", "proj-responses:model") + + stash = get_or_create_request_stash() + stash.itpm_reserved_tokens = 10 + stash.itpm_reserved_scopes = frozenset({itpm_scope}) + stash.otpm_reserved_tokens = 10 + stash.otpm_reserved_scopes = frozenset({otpm_scope}) + mock_kwargs = {} + + mock_response = ResponsesAPIResponse( + id="resp_test", + created_at=int(datetime.now().timestamp()), + output=[], + usage=ResponseAPIUsage(input_tokens=80, output_tokens=400, total_tokens=480), + ) + + increments = [] + + async def mock_increment(increment_list, **kwargs): + for op in increment_list: + increments.append({"key": op["key"], "increment": op["increment_value"]}) + + handler.internal_usage_cache.dual_cache.async_increment_cache_pipeline = ( + mock_increment + ) + + await handler.async_log_success_event( + kwargs=mock_kwargs, + response_obj=mock_response, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + itpm_adjustments = [i for i in increments if "model_per_project_itpm" in i["key"]] + otpm_adjustments = [i for i in increments if "model_per_project_otpm" in i["key"]] + + # delta = 80 actual input - 10 reserved = +70 + assert any(i["increment"] == 70 for i in itpm_adjustments), ( + f"ITPM reservation was never trued up to actual Responses API usage: {itpm_adjustments}" + ) + # delta = 400 actual output - 10 reserved = +390 + assert any(i["increment"] == 390 for i in otpm_adjustments), ( + f"OTPM reservation was never trued up to actual Responses API usage: {otpm_adjustments}" + ) + + +@pytest.mark.asyncio +async def test_itpm_reservation_accounts_for_audio_content_not_just_text(rate_limiter): + """ + Regression for the audio half of a Medium-severity review finding: + litellm.token_counter has no per-type handling for `input_audio` + content blocks (unlike images, which it does count via + use_default_image_token_count), so it silently contributes 0 tokens for + them. Without DEFAULT_AUDIO_TOKEN_ESTIMATE, a burst of audio-heavy + requests with minimal text would each reserve only the one-token floor + and blow past the project ITPM limit before post-call reconciliation. + """ + handler, cache = rate_limiter + + api_key = hash_token("sk-audio-itpm") + user_api_key_dict = UserAPIKeyAuth( + api_key=api_key, + project_id="proj-audio", + project_metadata={ + # Tighter than DEFAULT_AUDIO_TOKEN_ESTIMATE (300), but far bigger + # than the handful of tokens the bare text "hi" would cost. + "model_itpm_limit": {"bedrock_mantle/claude-opus": 50}, + }, + ) + + data = { + "model": "bedrock_mantle/claude-opus", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "hi"}, + { + "type": "input_audio", + "input_audio": {"data": "base64-audio-bytes", "format": "wav"}, + }, + ], + } + ], + } + + with pytest.raises(Exception) as exc_info: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data=data, + call_type="", + ) + assert getattr(exc_info.value, "status_code", None) == 429, ( + "Expected the audio content to push the ITPM reservation over the " + "50-token limit; if this doesn't raise, audio content isn't being " + "counted again." + ) + + +def test_audio_token_estimate_scales_with_payload_size(): + """ + Regression for veria-ai Low finding: audio token reservation was flat + 300 per block regardless of duration. A short clip and a long clip both + reserved the same amount, letting a caller hide long audio in one block + to exhaust ITPM quota while reserving almost nothing. + + The estimate must now grow proportionally with the base64 payload size + (len(b64) * 3 // 4 // _AUDIO_BYTES_PER_TOKEN), floored at + DEFAULT_AUDIO_TOKEN_ESTIMATE so reference-only blocks and genuinely + short clips still get a non-trivial reservation. + + To exceed the floor the decoded payload must be > 300 * 1600 = 480 000 + bytes. We synthesise a fake b64-length string of 650 000 chars + (decoded ≈ 487 500 bytes → 304 tokens) to avoid actually allocating + and encoding ~480 kB of audio in every test run. + """ + large_b64 = "A" * 650_000 + very_large_b64 = "A" * 12_900_000 + small_b64 = "A" * 1_000 + + large_block = { + "type": "input_audio", + "input_audio": {"data": large_b64, "format": "wav"}, + } + small_block = { + "type": "input_audio", + "input_audio": {"data": small_b64, "format": "wav"}, + } + very_large_block = { + "type": "input_audio", + "input_audio": {"data": very_large_b64, "format": "wav"}, + } + no_data_block = {"type": "input_audio", "input_audio": {"format": "wav"}} + + large_estimate = RateLimitHandler._estimate_audio_block_tokens(large_block) + very_large_estimate = RateLimitHandler._estimate_audio_block_tokens( + very_large_block + ) + small_estimate = RateLimitHandler._estimate_audio_block_tokens(small_block) + no_data_estimate = RateLimitHandler._estimate_audio_block_tokens(no_data_block) + + assert large_estimate > small_estimate, ( + f"Large payload ({large_estimate}) must reserve more than small payload " + f"({small_estimate}); flat-rate bug is back" + ) + assert very_large_estimate == len(very_large_b64) * 3 // 4 // _AUDIO_BYTES_PER_TOKEN + assert very_large_estimate > 6_000 + assert no_data_estimate >= 300, ( + f"Reference-only block (no data) must use the DEFAULT_AUDIO_TOKEN_ESTIMATE floor; got {no_data_estimate}" + ) + assert small_estimate >= 300, ( + f"Small payload must be floored at DEFAULT_AUDIO_TOKEN_ESTIMATE=300; got {small_estimate}" + ) + + +@pytest.mark.asyncio +async def test_itpm_rejects_large_audio_payload_that_would_pass_flat_estimate( + rate_limiter, +): + """ + Regression: a caller placing a long audio clip in one block previously + reserved only 300 tokens (the flat estimate). With the size-proportional + estimate, the same clip now reserves proportionally more and must trip + the ITPM limit when the limit is tuned to exactly expose the difference. + + 1 100 000 b64 chars → decoded ≈ 825 000 bytes → 825 000 // 1600 ≈ 515 + tokens > the 400-token limit. The flat estimate (300) would have passed. + """ + handler, cache = rate_limiter + + large_b64 = "A" * 1_100_000 + + user_api_key_dict = UserAPIKeyAuth( + api_key=hash_token("sk-large-audio"), + project_id="proj-large-audio", + project_metadata={ + "model_itpm_limit": {"bedrock_mantle/claude-opus": 400}, + }, + ) + + data = { + "model": "bedrock_mantle/claude-opus", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "transcribe this"}, + { + "type": "input_audio", + "input_audio": {"data": large_b64, "format": "wav"}, + }, + ], + } + ], + } + + with pytest.raises(Exception) as exc_info: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data=data, + call_type="", + ) + assert getattr(exc_info.value, "status_code", None) == 429, ( + "Large audio payload must exceed the 400-token ITPM limit under the " + "size-proportional estimate; the old flat-rate estimate (300 tokens) " + "would have passed this limit silently" + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("call_type", "request_data"), + [ + ( + "acompletion", + { + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "describe this"}, + { + "type": "image_url", + "image_url": { + "url": "https://example.com/high-resolution.png", + "detail": "high", + }, + }, + ], + } + ] + }, + ), + ( + "acompletion", + { + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "summarize this"}, + { + "type": "file", + "file": { + "filename": "document.pdf", + "file_data": "data:application/pdf;base64,dGVzdA==", + }, + }, + ], + } + ] + }, + ), + ( + "aresponses", + { + "input": [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "describe this"}, + { + "type": "input_image", + "image_url": "https://example.com/high-resolution.png", + "detail": "high", + }, + ], + } + ] + }, + ), + ( + "aresponses", + {"input": "continue", "previous_response_id": "resp-123"}, + ), + ], +) +async def test_multimodal_requests_reserve_measured_project_itpm_not_full_limit( + rate_limiter, + call_type, + request_data, +): + """ + Regression: image, file, and previous_response_id requests used to + reserve the project's whole ITPM limit up front. Because the atomic + check is ``current + increment > limit``, that made every such request + 429 as soon as the window carried any usage at all and, while in + flight, blocked every other request for the same project + model. They + now reserve the token_counter estimate like everything else, so two + multimodal requests fit in the same window. + """ + handler, cache = rate_limiter + model = "bedrock_mantle/claude-opus" + project_itpm_limit = 10_000 + user_api_key_dict = UserAPIKeyAuth( + api_key=hash_token("sk-multimodal-measured"), + project_id="project-multimodal-measured", + project_metadata={"model_itpm_limit": {model: project_itpm_limit}}, + ) + + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data={"model": model, **request_data}, + call_type=call_type, + ) + first_stash = get_request_stash() + assert first_stash is not None + first_reservation = first_stash.itpm_reserved_tokens + assert 0 < first_reservation < project_itpm_limit // 2 + + _request_stash.set(None) + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data={"model": model, **request_data}, + call_type=call_type, + ) + second_stash = get_request_stash() + assert second_stash is not None + assert second_stash is not first_stash + assert second_stash.itpm_reserved_tokens == first_reservation + + +@pytest.mark.asyncio +async def test_itpm_otpm_reservation_is_kept_on_stream_disconnect(rate_limiter): + handler, cache = rate_limiter + + api_key = hash_token("sk-disconnect-test") + user_api_key_dict = UserAPIKeyAuth( + api_key=api_key, + project_id="proj-disconnect", + project_metadata={ + "model_itpm_limit": {"bedrock_mantle/claude-opus": 1000}, + "model_otpm_limit": {"bedrock_mantle/claude-opus": 500}, + }, + ) + + data: Dict[str, Any] = { + "model": "bedrock_mantle/claude-opus", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 50, + } + + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data=data, + call_type="", + ) + + stash = get_request_stash() + assert stash is not None + assert stash.itpm_reserved_tokens > 0, ( + "pre-call hook must stash an ITPM reservation" + ) + assert stash.otpm_reserved_tokens > 0, ( + "pre-call hook must stash an OTPM reservation" + ) + + increment_calls: list[dict] = [] + + async def mock_increment(increment_list, litellm_parent_otel_span=None): + for op in increment_list: + increment_calls.append( + {"key": op["key"], "increment": op["increment_value"]} + ) + + handler.internal_usage_cache.dual_cache.async_increment_cache_pipeline = ( + mock_increment + ) + + await handler.async_release_max_parallel_requests_on_disconnect( + user_api_key_dict=user_api_key_dict + ) + + itpm_refunds = [ + c + for c in increment_calls + if "model_per_project_itpm" in c["key"] and c["increment"] < 0 + ] + otpm_refunds = [ + c + for c in increment_calls + if "model_per_project_otpm" in c["key"] and c["increment"] < 0 + ] + + assert not itpm_refunds + assert not otpm_refunds + assert stash.reservation_released is False + + +@pytest.mark.asyncio +async def test_responses_api_otpm_output_cap_applied_not_skipped_as_embedding( + rate_limiter, +): + """ + Regression for a Greptile P1 finding: _reserve_project_io_tokens_or_raise + classified any request with data["input"] set as an embedding (no output + tokens), which also misclassifies the Responses API -- it puts its prompt + in "input" too, but does generate output. That skipped the output cap + applied whenever the configured OTPM limit is small enough to need it, + letting an unbounded Responses generation blow past OTPM before + post-call reconciliation catches up. + + The cap must land on data["max_output_tokens"], not data["max_tokens"]: + the Responses-to-chat-completion transformation only reads + max_output_tokens, so a max_tokens cap is silently dropped before + provider dispatch (a second Greptile finding on the same code path). + """ + handler, cache = rate_limiter + + api_key = hash_token("sk-responses-otpm-cap") + user_api_key_dict = UserAPIKeyAuth( + api_key=api_key, + project_id="proj-responses-otpm", + project_metadata={ + "model_otpm_limit": {"bedrock_mantle/claude-opus": 40}, + }, + ) + + data: Dict[str, Any] = { + "model": "bedrock_mantle/claude-opus", + "input": "describe this image in detail", + } + + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data=data, + call_type="aresponses", + ) + + assert data.get("max_output_tokens") is not None, ( + "Responses call was misclassified as an embedding and skipped the OTPM output cap" + ) + assert data["max_output_tokens"] == 16 + assert data.get("max_tokens") is None, ( + "OTPM output cap was written to max_tokens, which the Responses transformation ignores" + ) + + +@pytest.mark.asyncio +async def test_explicit_zero_output_responses_call_reserves_effective_provider_minimum( + rate_limiter, +): + handler, cache = rate_limiter + + user_api_key_dict = UserAPIKeyAuth( + api_key=hash_token("sk-responses-zero-output"), + project_id="proj-responses-zero-output", + project_metadata={ + "model_otpm_limit": {"bedrock_mantle/claude-opus": 5}, + }, + ) + + with pytest.raises(Exception) as exc_info: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data={ + "model": "bedrock_mantle/claude-opus", + "input": "describe this image in detail", + "max_output_tokens": 0, + }, + call_type="aresponses", + ) + + assert getattr(exc_info.value, "status_code", None) == 429 + + +@pytest.mark.asyncio +async def test_responses_api_combined_tpm_output_cap_applied_not_skipped_as_embedding( + rate_limiter, +): + """ + Regression for the same misclassification bug in the combined-TPM + output-cap block of async_pre_call_hook (a second, independent + `is_embedding = data.get("input") is not None` check). A project with + only a combined model_tpm_limit (no split itpm/otpm) configured small + enough to need the output cap must still apply it to a Responses call, + and must write it to max_output_tokens for the same reason as the OTPM + case above. + """ + handler, cache = rate_limiter + + api_key = hash_token("sk-responses-tpm-cap") + user_api_key_dict = UserAPIKeyAuth( + api_key=api_key, + project_id="proj-responses-tpm", + project_metadata={ + "model_tpm_limit": {"bedrock_mantle/claude-opus": 40}, + }, + ) + + data: Dict[str, Any] = { + "model": "bedrock_mantle/claude-opus", + "input": "describe this image in detail", + } + + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data=data, + call_type="aresponses", + ) + + assert data.get("max_output_tokens") is not None, ( + "Responses call was misclassified as an embedding and skipped the combined-TPM output cap" + ) + assert data["max_output_tokens"] == 16 + assert data.get("max_tokens") is None, ( + "combined-TPM output cap was written to max_tokens, which the Responses transformation ignores" + ) + + +@pytest.mark.asyncio +async def test_responses_api_multimodal_input_counts_image_content(rate_limiter): + """ + Regression for a Low-severity veria-ai finding: the Responses API's + `input` is commonly a list of message/content-block dicts, but + litellm.token_counter's `text` argument only joins plain string entries + in a list and silently drops everything else -- so an `input_image` + block contributed ~0 tokens to the ITPM estimate instead of the real + image token count. _estimate_precise_input_tokens now converts Responses + `input` to chat messages first (via the standard + transform_responses_api_input_to_messages helper) so image content is + counted the same way a chat completion's image content already is. + """ + handler, _cache = rate_limiter + + text_only_estimate = handler._estimate_precise_input_tokens( + data={"input": "hi"}, + model="bedrock_mantle/claude-opus", + call_type="aresponses", + ) + + multimodal_estimate = handler._estimate_precise_input_tokens( + data={ + "input": [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "hi"}, + { + "type": "input_image", + "image_url": "https://example.com/some-image.png", + }, + ], + } + ], + }, + model="bedrock_mantle/claude-opus", + call_type="aresponses", + ) + + assert multimodal_estimate > text_only_estimate + 100, ( + "Responses API input_image content block was not counted; got " + f"text_only={text_only_estimate}, multimodal={multimodal_estimate}" + ) + + +@pytest.mark.asyncio +async def test_refund_reserved_tokens_noop_when_amount_zero(rate_limiter): + """_refund_reserved_tokens returns immediately without calling Redis when amount=0.""" + handler, _cache = rate_limiter + + calls = [] + + async def mock_increment(pipeline_operations, **kwargs): + calls.extend(pipeline_operations) + + handler.async_increment_tokens_with_ttl_preservation = mock_increment + + await handler._refund_reserved_tokens( + scopes=[("api_key", "sk-test")], + amount=0, + ) + + assert not calls, "No Redis ops expected when amount is zero" + + +@pytest.mark.asyncio +async def test_reserve_io_tokens_noop_when_no_itpm_otpm_descriptors(rate_limiter): + """reserve_io_tokens returns OK immediately when no ITPM/OTPM descriptors present.""" + handler, _cache = rate_limiter + + non_io_descriptor = { + "key": "api_key", + "value": "sk-test", + "rate_limit": {"tokens_per_unit": 1000, "window_size": 60}, + } + response, itpm_reserved, otpm_reserved = await handler.reserve_io_tokens( + descriptors=[non_io_descriptor], + estimated_input_tokens=50, + estimated_output_tokens=50, + ) + + assert response["overall_code"] == "OK" + assert itpm_reserved == 0 + assert otpm_reserved == 0 + + +@pytest.mark.asyncio +async def test_reserve_io_tokens_itpm_only_no_otpm(rate_limiter): + """When only ITPM descriptors are present (no OTPM), returns itpm_reserved with otpm=0.""" + handler, cache = rate_limiter + + itpm_descriptor = { + "key": PROJECT_ITPM_DESCRIPTOR_KEY, + "value": "proj-a:model", + "rate_limit": {"tokens_per_unit": 10000, "window_size": 60}, + } + response, itpm_reserved, otpm_reserved = await handler.reserve_io_tokens( + descriptors=[itpm_descriptor], + estimated_input_tokens=100, + estimated_output_tokens=50, + ) + + assert response["overall_code"] == "OK" + assert itpm_reserved == 100 + assert otpm_reserved == 0 + + +def test_strip_audio_content_blocks_passthrough_non_list_messages(): + """Non-list input is returned unchanged (early return on line 2605).""" + result = RateLimitHandler._strip_audio_content_blocks("not a list") + assert result == "not a list" + + +def test_strip_audio_content_blocks_passthrough_non_dict_message(): + """Non-dict entries in the message list are appended unchanged.""" + messages = ["plain string message"] + result = RateLimitHandler._strip_audio_content_blocks(messages) + assert result == ["plain string message"] + + +def test_strip_audio_content_blocks_passthrough_non_list_content(): + """Messages with non-list content (e.g. plain string) pass through unchanged.""" + messages = [{"role": "user", "content": "hello"}] + result = RateLimitHandler._strip_audio_content_blocks(messages) + assert result == [{"role": "user", "content": "hello"}] + + +@pytest.mark.asyncio +async def test_otpm_rejection_releases_stashed_parallel_slot(rate_limiter): + """ + When OTPM is over limit and a parallel slot was already acquired, the + disconnect cleanup path in _reserve_project_io_tokens_or_raise must + release that slot. Exercises lines 2773-2777. + """ + handler, cache = rate_limiter + + user_api_key_dict = UserAPIKeyAuth( + api_key=hash_token("sk-otpm-slot"), + project_id="proj-slot", + project_metadata={"model_otpm_limit": {"m": 5}}, + ) + + data: Dict[str, Any] = { + "model": "m", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 50, + } + + slot_released = [] + + async def mock_release(acquisition, parent_otel_span=None): + slot_released.append(acquisition) + + handler._release_parallel_request_slots = mock_release + + stash = get_or_create_request_stash() + stash.parallel_slot = { + "slot_id": "test-slot-id", + "counter_keys": ["some-key"], + } + + otpm_descriptor = { + "key": PROJECT_OTPM_DESCRIPTOR_KEY, + "value": "proj-slot:m", + "rate_limit": {"tokens_per_unit": 5, "window_size": 60}, + } + + with pytest.raises(Exception) as exc_info: + await handler._reserve_project_io_tokens_or_raise( + descriptors=[otpm_descriptor], + data=data, + requested_model="m", + user_api_key_dict=user_api_key_dict, + tpm_reservation_scopes=[], + tpm_reservation_amount=0, + ) + assert getattr(exc_info.value, "status_code", None) == 429 + assert slot_released, "Parallel slot must be released when OTPM rejects" + assert stash.parallel_slot is None + + +@pytest.mark.asyncio +async def test_itpm_only_status_stored_when_no_prior_rate_limit_response(rate_limiter): + """ + When only ITPM is configured (no combined TPM/RPM to pre-populate + the request stash), a successful ITPM reservation must store its status + there so post-call headers can read it. + """ + handler, cache = rate_limiter + + user_api_key_dict = UserAPIKeyAuth( + api_key=hash_token("sk-itpm-only-store"), + project_id="proj-store", + ) + + data: Dict[str, Any] = {"model": "m", "messages": []} + + itpm_descriptor = { + "key": PROJECT_ITPM_DESCRIPTOR_KEY, + "value": "proj-store:m", + "rate_limit": {"tokens_per_unit": 100000, "window_size": 60}, + } + + await handler._reserve_project_io_tokens_or_raise( + descriptors=[itpm_descriptor], + data=data, + requested_model="m", + user_api_key_dict=user_api_key_dict, + tpm_reservation_scopes=[], + tpm_reservation_amount=0, + ) + + stash = get_request_stash() + assert stash is not None + stored = stash.rate_limit_response + assert stored is not None, ( + "ITPM status must be stored in litellm_proxy_rate_limit_response" + ) + assert stored.get("statuses"), "Stored response must contain statuses" + + +def test_resolve_io_token_usage_responses_api_with_cached_tokens(rate_limiter): + """ + ResponsesAPIResponse whose usage.input_tokens_details.cached_tokens is set + subtracts the cached portion from billable input. Covers line 3501. + """ + handler, _cache = rate_limiter + + response_obj = ResponsesAPIResponse( + id="resp_cached", + created_at=int(datetime.now().timestamp()), + output=[], + usage=ResponseAPIUsage( + input_tokens=100, + output_tokens=50, + total_tokens=150, + input_tokens_details=InputTokensDetails(cached_tokens=25), + ), + ) + billable_input, completion_tokens, resolved = ( + handler._resolve_io_token_reconcile_usage(response_obj) + ) + + assert resolved is True + assert billable_input == 75, f"Expected 100 - 25 cached = 75, got {billable_input}" + assert completion_tokens == 50 + + +def test_resolve_io_token_usage_dict_format(rate_limiter): + """ + Dict-shaped usage on a ModelResponse (older SDK versions or raw dicts in + the usage field) is parsed correctly. Covers lines 3502-3506. + """ + handler, _cache = rate_limiter + + response_obj = ModelResponse.model_construct( + usage={ + "prompt_tokens": 80, + "completion_tokens": 40, + "prompt_tokens_details": {"cached_tokens": 20}, + } + ) + billable_input, completion_tokens, resolved = ( + handler._resolve_io_token_reconcile_usage(response_obj) + ) + + assert resolved is True + assert billable_input == 60, f"Expected 80 - 20 cached = 60, got {billable_input}" + assert completion_tokens == 40 + + +def test_resolve_io_token_usage_unknown_type_returns_unresolved(rate_limiter): + """ + A ModelResponse whose usage attribute is not a Usage, ResponseAPIUsage, + or dict (e.g. a plain int) returns (0, 0, False) so the reservation is + kept rather than guessed. Covers lines 3507-3508. + """ + handler, _cache = rate_limiter + + response_obj = ModelResponse.model_construct(usage=42) + billable_input, completion_tokens, resolved = ( + handler._resolve_io_token_reconcile_usage(response_obj) + ) + + assert resolved is False + assert billable_input == 0 + assert completion_tokens == 0 + + +@pytest.mark.parametrize( + ("combined_usage", "expected_increments"), + [ + (None, ()), + ( + Usage(prompt_tokens=40, completion_tokens=15, total_tokens=55), + (-60, -45), + ), + ], +) +def test_zero_usage_keeps_reservations_unless_measured_fallback_exists( + rate_limiter, + combined_usage, + expected_increments, +): + handler, _cache = rate_limiter + itpm_scope = (PROJECT_ITPM_DESCRIPTOR_KEY, "project:model") + otpm_scope = (PROJECT_OTPM_DESCRIPTOR_KEY, "project:model") + stash = get_or_create_request_stash() + stash.itpm_reserved_tokens = 100 + stash.itpm_reserved_scopes = frozenset({itpm_scope}) + stash.otpm_reserved_tokens = 60 + stash.otpm_reserved_scopes = frozenset({otpm_scope}) + kwargs = {} if combined_usage is None else {"combined_usage_object": combined_usage} + response_obj = ModelResponse( + usage=Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0) + ) + + operations = handler._build_io_token_reservation_ops(kwargs, response_obj) + + assert tuple(operation["increment_value"] for operation in operations) == expected_increments + + +@pytest.mark.parametrize( + ("usage", "expected_increments"), + [ + ( + Usage(prompt_tokens=40, completion_tokens=15, total_tokens=55), + (40, 15), + ), + ( + Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0), + (100, 60), + ), + ], +) +def test_retry_success_charges_released_project_io_reservations( + rate_limiter, + usage, + expected_increments, +): + handler, _cache = rate_limiter + itpm_scope = (PROJECT_ITPM_DESCRIPTOR_KEY, "project:model") + otpm_scope = (PROJECT_OTPM_DESCRIPTOR_KEY, "project:model") + stash = get_or_create_request_stash() + stash.itpm_reserved_tokens = 100 + stash.itpm_reserved_scopes = frozenset({itpm_scope}) + stash.otpm_reserved_tokens = 60 + stash.otpm_reserved_scopes = frozenset({otpm_scope}) + stash.reservation_released = True + + operations = handler._build_io_token_reservation_ops( + {}, + ModelResponse(usage=usage), + ) + + assert tuple(operation["increment_value"] for operation in operations) == expected_increments + + +@pytest.mark.asyncio +async def test_build_io_token_reservation_ops_skips_unresolvable_usage(rate_limiter): + """ + When response_obj has no parseable usage, _build_io_token_reservation_ops + returns [] to keep the reservation as-is rather than zeroing it out on a + bad guess. Covers line 3538. + """ + handler, _cache = rate_limiter + + itpm_scope = ("model_per_project_itpm", "proj-b:model") + stash = get_or_create_request_stash() + stash.itpm_reserved_tokens = 50 + stash.itpm_reserved_scopes = frozenset({itpm_scope}) + mock_kwargs = {} + + ops = handler._build_io_token_reservation_ops( + kwargs=mock_kwargs, + response_obj=object(), + ) + + assert not ops, f"Expected empty ops for unresolvable usage, got {ops}" + + +@pytest.mark.asyncio +async def test_post_call_failure_skips_rpm_only_descriptor_in_tpm_refund(rate_limiter): + """ + async_post_call_failure_hook skips descriptors without tokens_per_unit + (e.g. an RPM-only api_key scope) when building the combined-TPM refund ops, + so a key with rpm_limit but no tpm_limit doesn't receive a spurious refund + that would drive its counter negative. Covers the continue guard at line 4250. + """ + handler, cache = rate_limiter + + api_key = hash_token("sk-rpm-only-desc") + user_api_key_dict = UserAPIKeyAuth( + api_key=api_key, + rpm_limit=100, + project_id="proj-rpm-only-desc", + project_metadata={"model_tpm_limit": {"gpt-3.5-turbo": 100000}}, + ) + + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 20, + } + + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data=data, + call_type="", + ) + + rpm_tokens_key = handler.create_rate_limit_keys( + key="api_key", value=api_key, rate_limit_type="tokens" + ) + + await handler.async_post_call_failure_hook( + request_data=data, + original_exception=Exception("rejected"), + user_api_key_dict=user_api_key_dict, + ) + + api_key_tokens_after = int( + await cache.async_get_cache(key=rpm_tokens_key, local_only=True) or 0 + ) + assert api_key_tokens_after >= 0, ( + f"RPM-only api_key scope must not receive a negative TPM refund; got {api_key_tokens_after}" + ) + + +@pytest.mark.asyncio +async def test_max_output_tokens_prevents_cap_injection(rate_limiter): + """ + Regression for veria-ai comment: when a Responses API request supplies + max_output_tokens (the canonical Responses output bound) but not max_tokens + or max_completion_tokens, the has_explicit_max_tokens check was False, so + the code injected data["max_tokens"] = capped_floor and silently truncated + the response. + + With the fix, max_output_tokens is included in the explicit-cap check and + data["max_tokens"] must NOT be injected when max_output_tokens is already + set. + """ + handler, cache = rate_limiter + + user_api_key_dict = UserAPIKeyAuth( + api_key=hash_token("sk-max-output-tokens"), + project_id="proj-responses-max-output", + project_metadata={ + "model_otpm_limit": {"mock-model": 100}, + }, + ) + + data: dict = { + "model": "mock-model", + "input": "Summarise the document", + "max_output_tokens": 80, + "litellm_call_id": "test-max-output-tokens", + "metadata": {}, + } + + try: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data=data, + call_type="responses", + ) + except Exception: + pass + + assert "max_tokens" not in data, ( + "data['max_tokens'] must not be injected when max_output_tokens is already " + "set; the cap injection was overriding the caller's explicit output bound" + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("call_type", "request_data", "cap_field", "reserved_tokens"), + [ + ("aresponses", {"input": "hello", "max_tokens": 1}, "max_output_tokens", 16), + ( + "acompletion", + { + "messages": [{"role": "user", "content": "hello"}], + "max_output_tokens": 1, + }, + "max_tokens", + 10, + ), + ], +) +async def test_output_reservation_ignores_cap_fields_from_other_endpoints( + rate_limiter, + call_type, + request_data, + cap_field, + reserved_tokens, +): + handler, cache = rate_limiter + user_api_key_dict = UserAPIKeyAuth( + api_key=hash_token(f"sk-{call_type}"), + project_id=f"project-{call_type}", + project_metadata={"model_otpm_limit": {"model": 40}}, + ) + data = {"model": "model", **request_data} + + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data=data, + call_type=call_type, + ) + + assert data[cap_field] == reserved_tokens + stash = get_request_stash() + assert stash is not None + assert stash.otpm_reserved_tokens == reserved_tokens + + +def test_responses_input_is_counted_even_when_messages_is_present(rate_limiter): + handler, _cache = rate_limiter + small_estimate = handler._estimate_precise_input_tokens( + data={"input": "short", "messages": [{"role": "user", "content": "ignored"}]}, + model="", + call_type="aresponses", + ) + large_estimate = handler._estimate_precise_input_tokens( + data={"input": "large input " * 500, "messages": []}, + model="", + call_type="aresponses", + ) + + assert large_estimate > small_estimate + + +def test_anthropic_messages_usage_reconciles_split_project_quota(rate_limiter): + handler, _cache = rate_limiter + + billable_input, output_tokens, resolved = handler._resolve_io_token_reconcile_usage( + { + "usage": { + "input_tokens": 100, + "output_tokens": 25, + "cache_read_input_tokens": 30, + } + } + ) + + assert resolved is True + assert billable_input == 70 + assert output_tokens == 25 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "call_type", + ["agenerate_content", "agenerate_content_stream"], +) +async def test_google_genai_native_contents_reserve_project_itpm( + rate_limiter, + call_type, +): + handler, cache = rate_limiter + model = "gemini/gemini-2.5-flash" + user_api_key_dict = UserAPIKeyAuth( + api_key=hash_token("sk-google-genai-native-itpm"), + project_id="project-google-genai-native-itpm", + project_metadata={"model_itpm_limit": {model: 10_000}}, + ) + + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data={ + "model": model, + "contents": [ + { + "role": "user", + "parts": [{"text": "Gemini quota input " * 200}], + } + ], + }, + call_type=call_type, + ) + + stash = get_request_stash() + assert stash is not None + assert stash.itpm_reserved_tokens > 100 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call_type", ["rerank", "arerank"]) +async def test_rerank_query_and_documents_enforce_project_itpm( + rate_limiter, + monkeypatch, + call_type, +): + handler, cache = rate_limiter + captured = {} + + def token_counter(**kwargs): + captured.update(kwargs) + return 101 + + monkeypatch.setattr("litellm.token_counter", token_counter) + user_api_key_dict = UserAPIKeyAuth( + api_key=hash_token(f"sk-{call_type}-itpm"), + project_id=f"project-{call_type}-itpm", + project_metadata={"model_itpm_limit": {"rerank-model": 100}}, + ) + + with pytest.raises(Exception) as exc_info: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data={ + "model": "rerank-model", + "query": "Which document is most relevant?", + "documents": ["first document", {"text": "second document"}], + }, + call_type=call_type, + ) + + assert getattr(exc_info.value, "status_code", None) == 429 + assert captured["text"] == ( + "Which document is most relevant?\n" + "first document\n" + "{'text': 'second document'}" + ) + + +def test_rerank_input_estimate_falls_back_to_character_count( + rate_limiter, + monkeypatch, +): + handler, _cache = rate_limiter + data = { + "query": "query text", + "documents": ["first document", "second document"], + } + + def token_counter(**_kwargs): + raise ValueError("tokenizer unavailable") + + monkeypatch.setattr("litellm.token_counter", token_counter) + rerank_text = handler._rerank_input_to_text(data) + + assert handler._estimate_precise_input_tokens( + data, + model="custom-rerank-model", + call_type="rerank", + ) == len(rerank_text) // 4 + + +@pytest.mark.parametrize( + ("response_obj", "expected"), + [ + ( + RerankResponse( + meta={"tokens": {"input_tokens": 42, "output_tokens": 3}} + ), + (42, 3, True), + ), + ( + RerankResponse( + meta={ + "tokens": {"input_tokens": 0, "output_tokens": 0}, + "billed_units": {"total_tokens": 57}, + } + ), + (57, 0, True), + ), + ( + RerankResponse( + meta={ + "tokens": {"input_tokens": 0, "output_tokens": 0}, + "billed_units": {"total_tokens": 0}, + } + ), + (0, 0, False), + ), + ], +) +def test_rerank_usage_reconciles_project_split_token_quota( + rate_limiter, + response_obj, + expected, +): + handler, _cache = rate_limiter + + assert handler._resolve_io_token_reconcile_usage(response_obj) == expected + + +def test_split_quota_helpers_handle_non_mapping_inputs(rate_limiter): + handler, _cache = rate_limiter + + assert _call_id_from_callback_kwargs(object()) is None + assert handler._is_embedding_request(object(), None) is False + assert handler._get_explicit_output_cap(object(), None) is None + assert handler.get_output_candidate_count(object()) == 1 + assert handler.get_output_candidate_count({"n": 1e309}) == 1 + assert ( + handler._get_explicit_output_cap({"max_output_tokens": []}, "responses") is None + ) + assert handler._apply_implicit_output_cap(object(), 100, "responses") is None + assert handler._estimate_input_and_output_tokens(object()) == (0, 0) + assert handler._build_io_token_reservation_ops(object(), object()) == () + + +@pytest.mark.parametrize( + ("data", "call_type", "expected"), + [ + ({"max_tokens": "30.0"}, "", 30), + ({"max_tokens": "not-a-number"}, "", None), + ({"max_tokens": True}, "", None), + ({"max_output_tokens": "30.0"}, "responses", 30), + ({"max_output_tokens": "nan"}, "responses", None), + ({"generationConfig": {"maxOutputTokens": "12.5"}}, "agenerate_content", 12), + ({"generationConfig": {"maxOutputTokens": "oops"}}, "agenerate_content", None), + ], +) +def test_get_explicit_output_cap_tolerates_unparseable_values( + rate_limiter, data, call_type, expected +): + """A client-supplied cap the proxy cannot parse must fall back to the + no-cap output estimate instead of raising ValueError and 500ing the + request before it ever reaches the provider.""" + handler, _cache = rate_limiter + + assert handler._get_explicit_output_cap(data, call_type) == expected + + +@pytest.mark.asyncio +async def test_project_io_counters_not_double_charged_when_reservation_disabled( + monkeypatch, +): + """With LITELLM_TPM_TOKEN_RESERVATION_ENABLED=false the first + should_rate_limit pass used to +1 every ITPM/OTPM counter on top of the + full reservation _reserve_project_io_tokens_or_raise always makes, + permanently inflating each bucket by one token per request.""" + monkeypatch.setenv("LITELLM_TPM_TOKEN_RESERVATION_ENABLED", "false") + cache = DualCache() + handler = RateLimitHandler(internal_usage_cache=InternalUsageCache(cache)) + assert handler.tpm_reservation_enabled is False + + user_api_key_dict = UserAPIKeyAuth( + api_key=hash_token("sk-io-no-reservation"), + project_id="proj-io-no-reservation", + project_metadata={ + "model_itpm_limit": {"bedrock_mantle/claude-opus": 1000000}, + "model_otpm_limit": {"bedrock_mantle/claude-opus": 1000000}, + }, + ) + data = { + "model": "bedrock_mantle/claude-opus", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 50, + } + + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data=data, + call_type="", + ) + + stash = get_request_stash() + assert stash is not None + assert stash.itpm_reserved_tokens > 0 + assert stash.otpm_reserved_tokens > 0 + + for descriptor_key, reserved in ( + ("model_per_project_itpm", stash.itpm_reserved_tokens), + ("model_per_project_otpm", stash.otpm_reserved_tokens), + ): + counter_key = handler.create_rate_limit_keys( + key=descriptor_key, + value="proj-io-no-reservation:bedrock_mantle/claude-opus", + rate_limit_type="tokens", + ) + cached = await cache.async_get_cache(key=counter_key, local_only=True) + assert int(cached or 0) == reserved, ( + f"{descriptor_key} counter {cached} != reserved {reserved}: " + "first-pass should_rate_limit double-charged the bucket" + ) + + +@pytest.mark.parametrize( + ("call_type", "data"), + [ + ( + "text_completion", + { + "messages": [{"role": "user", "content": "ignored"}], + "prompt": "abcd", + "input": "ignored", + "max_tokens": 1, + }, + ), + (None, {"prompt": "abcd", "max_tokens": 1}), + (None, {"prompt": ["abcd", "efgh"], "max_tokens": 1}), + ], +) +def test_split_token_estimate_selects_endpoint_input(rate_limiter, call_type, data): + handler, _cache = rate_limiter + + estimated_input, estimated_output = handler._estimate_input_and_output_tokens( + data=data, + call_type=call_type, + ) + + assert estimated_input > 0 + assert estimated_output == 1 + + +def test_split_quota_multimodal_guards_handle_non_mapping_inputs(rate_limiter): + handler, _cache = rate_limiter + + assert handler._estimate_audio_block_tokens( + object() + ) == handler._estimate_audio_block_tokens({}) + assert handler._responses_input_to_chat_messages(object()) == () + assert handler._estimate_precise_input_tokens(object(), model=None) == 0 + + +@pytest.mark.parametrize( + ("call_type", "data", "expected_text"), + [ + ("embedding", {"input": "embedding input"}, "embedding input"), + ( + "embedding", + {"input": ["first embedding", "second embedding"]}, + ["first embedding", "second embedding"], + ), + ("text_completion", {"prompt": "completion prompt"}, "completion prompt"), + ], +) +def test_precise_input_estimate_selects_endpoint_text( + rate_limiter, + monkeypatch, + call_type, + data, + expected_text, +): + handler, _cache = rate_limiter + captured = {} + + def token_counter(**kwargs): + captured.update(kwargs) + return 7 + + monkeypatch.setattr("litellm.token_counter", token_counter) + + assert ( + handler._estimate_precise_input_tokens(data, model="test", call_type=call_type) + == 7 + ) + assert captured["messages"] is None + assert captured["text"] == expected_text + + +@pytest.mark.asyncio +async def test_project_io_reservation_ignores_non_mapping_request_data(rate_limiter): + handler, _cache = rate_limiter + + await handler._reserve_project_io_tokens_or_raise( + descriptors=[], + data=object(), + requested_model=None, + user_api_key_dict=UserAPIKeyAuth(), + tpm_reservation_scopes=(), + tpm_reservation_amount=0, + ) + + +@pytest.mark.asyncio +async def test_streaming_combined_usage_reconciles_project_io_reservations( + rate_limiter, +): + handler, _cache = rate_limiter + itpm_scope = (PROJECT_ITPM_DESCRIPTOR_KEY, "project:model") + otpm_scope = (PROJECT_OTPM_DESCRIPTOR_KEY, "project:model") + stash = get_or_create_request_stash() + stash.itpm_reserved_tokens = 100 + stash.itpm_reserved_scopes = frozenset({itpm_scope}) + stash.otpm_reserved_tokens = 60 + stash.otpm_reserved_scopes = frozenset({otpm_scope}) + kwargs = { + "combined_usage_object": Usage( + prompt_tokens=40, + completion_tokens=15, + total_tokens=55, + ), + } + increments = [] + + async def capture_increments(increment_list, **_kwargs): + increments.extend(increment_list) + + handler.internal_usage_cache.dual_cache.async_increment_cache_pipeline = ( + capture_increments + ) + + await handler.async_log_success_event( + kwargs=kwargs, + response_obj={"response": "stream body"}, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + itpm_adjustments = [ + operation + for operation in increments + if PROJECT_ITPM_DESCRIPTOR_KEY in operation["key"] + ] + otpm_adjustments = [ + operation + for operation in increments + if PROJECT_OTPM_DESCRIPTOR_KEY in operation["key"] + ] + assert [operation["increment_value"] for operation in itpm_adjustments] == [-60] + assert [operation["increment_value"] for operation in otpm_adjustments] == [-45] + + +def test_aggregate_only_combined_usage_reconciles_project_io_reservations(rate_limiter): + handler, _cache = rate_limiter + stash = get_or_create_request_stash() + stash.itpm_reserved_tokens = 100 + stash.itpm_reserved_scopes = frozenset( + {(PROJECT_ITPM_DESCRIPTOR_KEY, "project:model")} + ) + stash.otpm_reserved_tokens = 80 + stash.otpm_reserved_scopes = frozenset( + {(PROJECT_OTPM_DESCRIPTOR_KEY, "project:model")} + ) + kwargs = { + "combined_usage_object": Usage(total_tokens=55), + } + + operations = handler._build_io_token_reservation_ops(kwargs, object()) + + assert [operation["increment_value"] for operation in operations] == [-45, -25] + + +def test_raw_split_usage_dict_reconciles_project_io_tokens(rate_limiter): + handler, _cache = rate_limiter + + assert handler._resolve_io_token_reconcile_usage( + { + "input_tokens": 30, + "output_tokens": 12, + "input_tokens_details": {"cached_tokens": 5}, + } + ) == (25, 12, True) + + +@pytest.mark.asyncio +async def test_post_call_success_hook_contains_header_merge_failures( + rate_limiter, monkeypatch +): + handler, _cache = rate_limiter + response = ModelResponse() + response._hidden_params = {} + + def raise_on_merge(**_kwargs): + raise RuntimeError("header merge failed") + + monkeypatch.setattr( + handler, + "_merge_ratelimit_statuses_into_additional_headers", + raise_on_merge, + ) + + await handler.async_post_call_success_hook( + data={ + "litellm_proxy_rate_limit_response": { + "overall_code": "OK", + "statuses": (), + } + }, + user_api_key_dict=UserAPIKeyAuth(), + response=response, + ) + + if __name__ == "__main__": pytest.main([__file__, "-v", "-s"]) diff --git a/tests/test_litellm/proxy/management_endpoints/test_access_group_management.py b/tests/test_litellm/proxy/management_endpoints/test_access_group_management.py index 3240ad20edb..c973c6a8346 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_access_group_management.py +++ b/tests/test_litellm/proxy/management_endpoints/test_access_group_management.py @@ -13,6 +13,9 @@ sys.path.insert( ) # Adds the parent directory to the system path from litellm import Router +from litellm.proxy.management_endpoints.model_management_endpoints import ( + ReconcileOutcome, +) @pytest.mark.asyncio @@ -121,7 +124,7 @@ async def test_create_access_group_with_model_ids_tags_only_specific_deployments patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), patch( "litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache", - new_callable=AsyncMock, + new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)), ), ): response = await create_model_group( @@ -186,7 +189,7 @@ async def test_create_access_group_with_model_names_tags_all_deployments(): patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), patch( "litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache", - new_callable=AsyncMock, + new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)), ), ): response = await create_model_group( @@ -236,7 +239,7 @@ async def test_create_access_group_model_ids_takes_priority_over_model_names(): patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), patch( "litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache", - new_callable=AsyncMock, + new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)), ), ): response = await create_model_group( @@ -313,7 +316,7 @@ async def test_create_access_group_invalid_model_id_returns_400(): patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), patch( "litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache", - new_callable=AsyncMock, + new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)), ), ): with pytest.raises(HTTPException) as exc_info: @@ -352,7 +355,7 @@ async def test_create_access_group_surfaces_dropped_models(): patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), patch( "litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache", - new=AsyncMock(return_value=None), + new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)), ), ): with pytest.raises(HTTPException) as exc_info: @@ -365,6 +368,50 @@ async def test_create_access_group_surfaces_dropped_models(): assert "deploy-A" in str(exc_info.value.detail) + +@pytest.mark.asyncio +async def test_create_access_group_trusts_reload_snapshot_over_post_lock_fresh_read(): + """A concurrent reconcile sampled after the lock is released must not make this + write's reload look like it dropped the tagged model: the verdict has to judge from + the ReconcileOutcome the reload captured under the lock, not a fresh router read.""" + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + create_model_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + NewModelGroupRequest, + ) + + deploy_a = MagicMock(model_id="deploy-A", model_name="gpt-4o", model_info={}) + + mock_prisma = MagicMock() + mock_prisma.db.litellm_proxymodeltable.find_many = AsyncMock(return_value=[]) + mock_prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=deploy_a) + mock_prisma.db.litellm_proxymodeltable.update = AsyncMock() + + concurrently_wiped_router = MagicMock() + concurrently_wiped_router.get_model_ids.side_effect = [["deploy-A"], []] + with ( + patch("litellm.proxy.proxy_server.llm_router", concurrently_wiped_router), + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch( + "litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache", + new=AsyncMock( + return_value=ReconcileOutcome( + still_desired=frozenset({"deploy-A"}), live_after=frozenset({"deploy-A"}) + ) + ), + ), + ): + response = await create_model_group( + data=NewModelGroupRequest(access_group="production-models", model_ids=["deploy-A"]), + user_api_key_dict=UserAPIKeyAuth(user_id="test_admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert response.models_updated == 1 + assert concurrently_wiped_router.get_model_ids.call_count == 1 + + @pytest.mark.asyncio async def test_tag_deployment_parses_string_model_info_and_refuses_corrupt(): """The model_info column can arrive as its JSON string; tagging must parse it rather @@ -420,7 +467,7 @@ async def test_delete_access_group_ignores_models_that_were_already_dead(): patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), patch( "litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache", - new=AsyncMock(return_value=None), + new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)), ), ): response = await delete_access_group( @@ -430,3 +477,99 @@ async def test_delete_access_group_ignores_models_that_were_already_dead(): assert response.models_updated == 1 mock_prisma.db.litellm_proxymodeltable.update.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_create_access_group_read_through_recovers_model_created_on_sibling_replica(): + """Regression: an access group referencing a model that another replica just wrote + to the DB must be created instead of 400ing until the periodic config reload.""" + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + create_model_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + NewModelGroupRequest, + ) + + from types import SimpleNamespace + + model_name = "e2e-ag-sibling-replica-model" + db_row = SimpleNamespace( + model_id=f"{model_name}-id", + model_name=model_name, + litellm_params={"model": "openai/gpt-4o", "api_key": "fake", "mock_response": "hi"}, + model_info={}, + blocked=False, + ) + + mock_router = Router( + model_list=[ + { + "model_name": "some-other-model", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"}, + } + ] + ) + + mock_prisma = MagicMock() + mock_prisma.db.litellm_proxymodeltable.find_many = AsyncMock(side_effect=[[db_row], [], [db_row]]) + mock_prisma.db.litellm_proxymodeltable.update = AsyncMock() + + with ( + patch("litellm.proxy.proxy_server.llm_router", mock_router), + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + patch( + "litellm.proxy.management_endpoints.model_access_group_management_endpoints.clear_cache", + new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)), + ), + ): + response = await create_model_group( + data=NewModelGroupRequest(access_group="replica-lag-group", model_names=[model_name]), + user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert response.models_updated == 1 + assert response.model_names == [model_name] + assert mock_prisma.db.litellm_proxymodeltable.find_many.await_args_list[0].kwargs["where"] == { + "model_name": model_name + } + + +@pytest.mark.asyncio +async def test_create_access_group_model_missing_everywhere_still_400s(): + from fastapi import HTTPException + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + create_model_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + NewModelGroupRequest, + ) + + model_name = "e2e-ag-model-nobody-created" + mock_router = Router( + model_list=[ + { + "model_name": "some-other-model", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"}, + } + ] + ) + mock_prisma = MagicMock() + mock_prisma.db.litellm_proxymodeltable.find_many = AsyncMock(return_value=[]) + + with ( + patch("litellm.proxy.proxy_server.llm_router", mock_router), + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + ): + with pytest.raises(HTTPException) as exc_info: + await create_model_group( + data=NewModelGroupRequest(access_group="ghost-group", model_names=[model_name]), + user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert exc_info.value.status_code == 400 + assert model_name in str(exc_info.value.detail) diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index 412e6f0f83c..62c05841197 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -14,6 +14,7 @@ sys.path.insert(0, os.path.abspath("../../../..")) # Adds the parent directory from litellm.proxy.management_endpoints.common_daily_activity import ( _adjust_dates_for_timezone, _build_aggregated_sql_query, + _build_entity_rollup_sql_query, _is_user_agent_tag, _record_to_spend_metrics, get_api_key_metadata, @@ -982,6 +983,58 @@ class TestBuildAggregatedSqlQuery: assert "COALESCE(model_group, model)" not in normalized +class TestAggregatedEmptyEntityFilter: + _BUILDERS: Final = (_build_aggregated_sql_query, _build_entity_rollup_sql_query) + + @pytest.mark.parametrize("build", _BUILDERS) + def test_empty_entity_list_emits_no_degenerate_in_clause(self, build): + sql, params = build( + table_name="litellm_dailyteamspend", + entity_id_field="team_id", + entity_id=[], + start_date="2026-08-01", + end_date="2026-08-19", + model=None, + api_key=None, + ) + + normalized = " ".join(sql.split()) + assert "IN ()" not in normalized + assert '"team_id" IN' not in normalized + assert params == ["2026-08-01", "2026-08-19"] + + @pytest.mark.parametrize("build", _BUILDERS) + def test_empty_entity_list_matches_nothing_rather_than_everything(self, build): + sql, _ = build( + table_name="litellm_dailyteamspend", + entity_id_field="team_id", + entity_id=[], + start_date="2026-08-01", + end_date="2026-08-19", + model=None, + api_key=None, + ) + + assert "FALSE" in " ".join(sql.split()) + + @pytest.mark.parametrize("build", _BUILDERS) + def test_populated_entity_list_still_filters_on_its_ids(self, build): + sql, params = build( + table_name="litellm_dailyteamspend", + entity_id_field="team_id", + entity_id=["team-alpha", "team-beta"], + start_date="2026-08-01", + end_date="2026-08-19", + model=None, + api_key=None, + ) + + normalized = " ".join(sql.split()) + assert '"team_id" IN ($3, $4)' in normalized + assert "FALSE" not in normalized + assert params == ["2026-08-01", "2026-08-19", "team-alpha", "team-beta"] + + @pytest.mark.asyncio async def test_get_daily_activity_aggregated_empty_result_set(): """Regression test for the empty-range 500. diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py index 664015003e4..69819318800 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py @@ -1286,6 +1286,75 @@ class TestOpenAIPassthroughIntegration: mock_chat_handler.assert_called_once() assert result == {"result": None, "kwargs": {}} + def test_openai_passthrough_handler_embeddings_unmapped_model_logs_zero_cost(self): + response_body = { + "object": "list", + "model": "lit5787-unmapped-embeddings-deployment", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], + "usage": {"prompt_tokens": 9, "total_tokens": 9}, + } + mock_logging_obj = self._create_mock_logging_obj() + result = OpenAIPassthroughLoggingHandler.openai_passthrough_handler( + httpx_response=self._create_mock_httpx_response(response_body), + response_body=response_body, + logging_obj=mock_logging_obj, + url_route="https://my-resource.openai.azure.com/openai/v1/embeddings", + result="", + start_time=self.start_time, + end_time=self.end_time, + cache_hit=False, + request_body={ + "model": "lit5787-unmapped-embeddings-deployment", + "input": "spend probe", + }, + passthrough_logging_payload=PassthroughStandardLoggingPayload( + url="https://my-resource.openai.azure.com/openai/v1/embeddings", + request_body={ + "model": "lit5787-unmapped-embeddings-deployment", + "input": "spend probe", + }, + request_method="POST", + ), + litellm_params={}, + ) + + assert result["result"] is not None + assert result["result"].usage.prompt_tokens == 9 + assert result["kwargs"]["response_cost"] == 0.0 + assert result["kwargs"]["model"] == "lit5787-unmapped-embeddings-deployment" + assert result["result"]._hidden_params["response_cost"] == 0.0 + assert mock_logging_obj.model_call_details["response_cost"] == 0.0 + + def test_openai_passthrough_handler_embeddings_error_skips_chat_fallback(self): + response_body = { + "object": "list", + "model": "text-embedding-3-small", + "usage": {"prompt_tokens": 9, "total_tokens": 9}, + } + kwargs_in = { + "passthrough_logging_payload": PassthroughStandardLoggingPayload( + url="https://api.openai.com/v1/embeddings", + request_body={"model": "text-embedding-3-small", "input": "spend probe"}, + request_method="POST", + ), + "litellm_params": {}, + } + result = OpenAIPassthroughLoggingHandler.openai_passthrough_handler( + httpx_response=self._create_mock_httpx_response(response_body), + response_body=response_body, + logging_obj=self._create_mock_logging_obj(), + url_route="https://api.openai.com/v1/embeddings", + result="", + start_time=self.start_time, + end_time=self.end_time, + cache_hit=False, + request_body={"model": "text-embedding-3-small", "input": "spend probe"}, + **kwargs_in, + ) + + assert result["result"] is None + assert result["kwargs"]["passthrough_logging_payload"] == kwargs_in["passthrough_logging_payload"] + @patch( "litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler.OpenAIPassthroughLoggingHandler.openai_passthrough_handler" ) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 050070e2fcf..b56a8da7c66 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -1,7 +1,10 @@ +import contextlib import json import os import sys import traceback +from collections.abc import Mapping +from types import MappingProxyType, SimpleNamespace from typing import Final from unittest import mock from unittest.mock import AsyncMock, MagicMock, Mock, patch @@ -22,6 +25,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( _join_url_paths, azure_proxy_route, bedrock_llm_proxy_route, + bedrock_proxy_route, create_pass_through_route, cursor_proxy_route, get_azure_ai_search_index_from_endpoint, @@ -1730,6 +1734,116 @@ class TestBedrockLLMProxyRoute: assert "Blocked by guardrail" in str(exc_info.value.detail) +class TestBedrockAgentRuntimePassthroughToggle: + AGENT_RUNTIME_ENDPOINT: Final = "knowledgebases/KB1234567/retrieve" + MODEL_ENDPOINT: Final = "model/us.anthropic.claude-sonnet-4-5-20250929-v1:0/converse" + DISABLED: Final = MappingProxyType({"disable_bedrock_agent_runtime_passthrough": True}) + + @staticmethod + def _mock_request() -> Mock: + request: Final = Mock() + request.method = "POST" + request.state = SimpleNamespace() + request.json = AsyncMock(return_value={"retrievalQuery": {"text": "hi"}}) # mutable-ok: must be json.dumps-able + return request + + @contextlib.contextmanager + def _patched_dispatch(self, general_settings: Mapping[str, object]): + from botocore.credentials import Credentials + + bedrock_llm: Final = Mock() + bedrock_llm.get_credentials = Mock(return_value=Credentials("ak", "sk")) + forwarder: Final = AsyncMock(return_value="forwarded") + + with ( + patch("litellm.proxy.proxy_server.general_settings", general_settings), + patch("litellm.utils.get_secret", return_value="us-east-1"), + patch("litellm.llms.bedrock.chat.BedrockConverseLLM", return_value=bedrock_llm), + patch( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_request_copy", + Mock(), + ), + patch( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route", + return_value=forwarder, + ) as create_route, + ): + yield create_route, forwarder + + @pytest.mark.asyncio + async def test_agent_runtime_dispatch_allowed_by_default(self): + with self._patched_dispatch(MappingProxyType({})) as (create_route, forwarder): + result: Final = await bedrock_proxy_route( + endpoint=self.AGENT_RUNTIME_ENDPOINT, + request=self._mock_request(), + fastapi_response=Mock(), + user_api_key_dict=UserAPIKeyAuth(), + ) + + assert result == "forwarded" + forwarder.assert_awaited_once() + assert "bedrock-agent-runtime.us-east-1.amazonaws.com" in create_route.call_args.kwargs["target"] + + @pytest.mark.asyncio + @pytest.mark.parametrize("value", (True, "true", "True")) + async def test_agent_runtime_dispatch_rejected_when_disabled(self, value: bool | str): + settings: Final = MappingProxyType({"disable_bedrock_agent_runtime_passthrough": value}) + + with self._patched_dispatch(settings) as (create_route, forwarder): + with pytest.raises(HTTPException) as exc_info: + await bedrock_proxy_route( + endpoint=self.AGENT_RUNTIME_ENDPOINT, + request=self._mock_request(), + fastapi_response=Mock(), + user_api_key_dict=UserAPIKeyAuth(), + ) + + assert exc_info.value.status_code == 403 + assert "bedrock-agent-runtime pass-through is disabled" in str(exc_info.value.detail) + create_route.assert_not_called() + forwarder.assert_not_awaited() + + @pytest.mark.asyncio + async def test_model_invoke_still_routed_when_agent_runtime_disabled(self): + with ( + patch("litellm.proxy.proxy_server.general_settings", self.DISABLED), + patch("litellm.utils.get_secret", return_value="us-east-1"), + patch( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_request_copy", + Mock(), + ), + patch( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.bedrock_llm_proxy_route", + new=AsyncMock(return_value="llm-route"), + ) as llm_route, + ): + result: Final = await bedrock_proxy_route( + endpoint=self.MODEL_ENDPOINT, + request=self._mock_request(), + fastapi_response=Mock(), + user_api_key_dict=UserAPIKeyAuth(), + ) + + assert result == "llm-route" + llm_route.assert_awaited_once() + + @pytest.mark.asyncio + @pytest.mark.parametrize("value", (False, "false", None, "", "yes")) + async def test_agent_runtime_dispatch_allowed_for_non_true_values(self, value: object): + settings: Final = MappingProxyType({"disable_bedrock_agent_runtime_passthrough": value}) + + with self._patched_dispatch(settings) as (create_route, forwarder): + result: Final = await bedrock_proxy_route( + endpoint=self.AGENT_RUNTIME_ENDPOINT, + request=self._mock_request(), + fastapi_response=Mock(), + user_api_key_dict=UserAPIKeyAuth(), + ) + + assert result == "forwarded" + create_route.assert_called_once() + + class TestLLMPassthroughFactoryProxyRoute: @pytest.mark.asyncio async def test_llm_passthrough_factory_proxy_route_success(self): diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 33d835652cd..e5add059260 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -124,6 +124,30 @@ def test_get_logging_payload_maps_openai_cache_write_tokens_to_cache_creation_in assert additional_usage_values["prompt_tokens_details"]["cache_write_tokens"] == 800 +def test_get_logging_payload_maps_nested_cache_creation_input_tokens(): + """ + Regression (LIT-5757): DashScope nests cache_creation_input_tokens inside + prompt_tokens_details; SpendLogs must record it as cache_creation_input_tokens. + """ + additional_usage_values: Final = _get_additional_usage_values_for_usage( + litellm.Usage( + prompt_tokens=2059, + completion_tokens=31, + total_tokens=2090, + prompt_tokens_details={ + "cached_tokens": 0, + "text_tokens": 2059, + "cache_type": "ephemeral", + "cache_creation_input_tokens": 2048, + "cache_creation": {"ephemeral_5m_input_tokens": 2048}, + }, + ) + ) + + assert additional_usage_values["cache_creation_input_tokens"] == 2048 + assert additional_usage_values["prompt_tokens_details"]["cache_write_tokens"] == 2048 + + def test_get_logging_payload_preserves_anthropic_cache_creation_input_tokens(): additional_usage_values = _get_additional_usage_values_for_usage( litellm.Usage( diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 5545ee92e84..75aa716bb85 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -11098,3 +11098,100 @@ async def test_moderations_reraises_proxy_exception_unwrapped(): assert exc_info.value.code == "400" assert exc_info.value.param == "metadata" mock_logging.post_call_failure_hook.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_init_agents_in_db_rebuilds_registry_under_agent_reconcile_lock(monkeypatch): + from litellm.proxy.agent_endpoints.agent_registry import ( + AGENT_RECONCILE_LOCK, + global_agent_registry, + ) + from litellm.proxy.proxy_server import ProxyConfig + + lock_states: list[bool] = [] + + async def fake_get_all_agents_from_db(prisma_client) -> list: + lock_states.append(AGENT_RECONCILE_LOCK.locked()) + return [] + + def fake_load_agents_from_db_and_config(db_agents) -> None: + lock_states.append(AGENT_RECONCILE_LOCK.locked()) + + monkeypatch.setattr(global_agent_registry, "get_all_agents_from_db", fake_get_all_agents_from_db) + monkeypatch.setattr(global_agent_registry, "load_agents_from_db_and_config", fake_load_agents_from_db_and_config) + + await ProxyConfig()._init_agents_in_db(prisma_client=MagicMock()) + + assert lock_states == [True, True] + assert not AGENT_RECONCILE_LOCK.locked() + + +@pytest.mark.asyncio +async def test_init_guardrails_in_db_snapshots_and_reconciles_under_guardrail_reconcile_lock(monkeypatch): + from litellm.proxy.guardrails.guardrail_registry import ( + GUARDRAIL_RECONCILE_LOCK, + IN_MEMORY_GUARDRAIL_HANDLER, + GuardrailRegistry, + ) + from litellm.proxy.proxy_server import ProxyConfig + + lock_states: list[bool] = [] + + async def fake_get_all_guardrails_from_db(prisma_client) -> list: + lock_states.append(GUARDRAIL_RECONCILE_LOCK.locked()) + return [] + + def fake_reconcile_db_guardrails(db_guardrail_ids) -> list: + lock_states.append(GUARDRAIL_RECONCILE_LOCK.locked()) + return [] + + monkeypatch.setattr(GuardrailRegistry, "get_all_guardrails_from_db", fake_get_all_guardrails_from_db) + monkeypatch.setattr(IN_MEMORY_GUARDRAIL_HANDLER, "reconcile_db_guardrails", fake_reconcile_db_guardrails) + + await ProxyConfig()._init_guardrails_in_db(prisma_client=MagicMock()) + + assert lock_states == [True, True] + assert not GUARDRAIL_RECONCILE_LOCK.locked() + + +class TestEmbeddingsFailureHookRequestData: + @pytest.mark.asyncio + async def test_failure_hook_gets_post_setup_data_with_logging_obj(self): + """Request setup replaces the processor's data dict (adding the logging + object the failure hook needs to lift token usage from); the embeddings + exception handler must pass that replaced dict, not the raw request body + dict it was rebuilt from.""" + from litellm.proxy._types import ProxyException + + captured = {} + logging_obj_sentinel = MagicMock() + + async def fake_process(self, **kwargs): + self.data = {**self.data, "litellm_logging_obj": logging_obj_sentinel} + captured["processor_data"] = self.data + raise RuntimeError("provider timeout") + + with ( + patch.object( + proxy_server_module, + "_read_request_body", + new=AsyncMock(return_value={"model": "my-embed", "input": "hello"}), + ), + patch.object( + proxy_server_module.ProxyBaseLLMRequestProcessing, + "base_process_llm_request", + new=fake_process, + ), + patch.object(proxy_server_module, "proxy_logging_obj") as mock_logging, + ): + mock_logging.post_call_failure_hook = AsyncMock(return_value=None) + with pytest.raises(ProxyException): + await proxy_server_module.embeddings( + request=MagicMock(), + fastapi_response=MagicMock(), + user_api_key_dict=UserAPIKeyAuth(), + ) + + hook_request_data = mock_logging.post_call_failure_hook.await_args.kwargs["request_data"] + assert hook_request_data is captured["processor_data"] + assert hook_request_data["litellm_logging_obj"] is logging_obj_sentinel diff --git a/tests/test_litellm/proxy/test_proxy_types.py b/tests/test_litellm/proxy/test_proxy_types.py index 5354de182a0..4a93e9ac7ba 100644 --- a/tests/test_litellm/proxy/test_proxy_types.py +++ b/tests/test_litellm/proxy/test_proxy_types.py @@ -159,3 +159,21 @@ def test_update_key_request_requires_key_or_key_alias(): by_alias = UpdateKeyRequest(key_alias="my-alias") assert by_alias.key is None assert by_alias.key_alias == "my-alias" + + +@pytest.mark.parametrize("request_type", ["new", "update"]) +def test_project_io_token_limits_are_stored_in_metadata(request_type): + from litellm.proxy._types import NewProjectRequest, UpdateProjectRequest + + limits = { + "model_itpm_limit": {"bedrock_mantle/openai.gpt-oss-120b": 20_000_000}, + "model_otpm_limit": {"bedrock_mantle/openai.gpt-oss-120b": 4_000_000}, + } + request = ( + NewProjectRequest(team_id="team-1", **limits) + if request_type == "new" + else UpdateProjectRequest(project_id="project-1", **limits) + ) + + assert request.metadata == limits + assert request.model_dump(exclude_none=True)["metadata"] == limits diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index 1504c3c3103..d6cf0e30139 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -478,6 +478,307 @@ class TestPostCallFailureHookLiftsRecoveredPartialSpend: assert "response_cost" not in request_data +class TestPostCallFailureHookEstimatesDispatchedInputTokens: + """A non-stream request that failed after dispatch (timeout, provider + error) consumed provider-billed input tokens but recovered no usage. + post_call_failure_hook must estimate the input side onto request_data so + the spend log's failure row records what was sent instead of zero, while + never charging spend for the failure (LIT-5690). + """ + + async def _run(self, request_data): + from unittest.mock import AsyncMock, patch + + from litellm.proxy._types import UserAPIKeyAuth + + proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache()) + proxy_logging_obj.alert_types = [] + with patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()): + await proxy_logging_obj.post_call_failure_hook( + request_data=request_data, + original_exception=Exception("boom"), + user_api_key_dict=UserAPIKeyAuth(), + ) + + def _logging_obj(self, model_call_details): + logging_obj = MagicMock() + logging_obj.model_call_details = model_call_details + return logging_obj + + @pytest.mark.asyncio + async def test_dispatched_failure_estimates_input_tokens_with_zero_cost(self): + from datetime import datetime + + from litellm.types.utils import Usage + + request_data = { + "litellm_logging_obj": self._logging_obj( + { + "first_api_call_start_time": datetime.now(), + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "count these input tokens please"}], + "call_type": "acompletion", + } + ), + "metadata": {}, + "response_cost": 123.0, + } + await self._run(request_data) + + estimated = request_data["combined_usage_object"] + assert isinstance(estimated, Usage) + assert estimated.prompt_tokens > 0 + assert estimated.completion_tokens == 0 + assert estimated.total_tokens == estimated.prompt_tokens + assert request_data["response_cost"] == 0.0 + + @pytest.mark.asyncio + async def test_failure_before_dispatch_stays_zero(self): + request_data = { + "litellm_logging_obj": self._logging_obj( + { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "never dispatched"}], + } + ), + "metadata": {}, + } + await self._run(request_data) + + assert "combined_usage_object" not in request_data + assert "response_cost" not in request_data + + @pytest.mark.asyncio + async def test_proxy_only_error_never_dispatched_stays_zero(self): + from datetime import datetime + + from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL + + request_data = { + "litellm_logging_obj": self._logging_obj( + { + "first_api_call_start_time": datetime.now(), + "model": "no-such-model", + "messages": [{"role": "user", "content": "hi"}], + LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL: True, + } + ), + "metadata": {}, + } + await self._run(request_data) + + assert "combined_usage_object" not in request_data + assert "response_cost" not in request_data + + @pytest.mark.asyncio + async def test_recovered_partial_usage_wins_over_estimate(self): + from datetime import datetime + + from litellm.types.utils import Usage + + recovered_usage = Usage(prompt_tokens=30, completion_tokens=7, total_tokens=37) + request_data = { + "litellm_logging_obj": self._logging_obj( + { + "first_api_call_start_time": datetime.now(), + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "mid-stream failure"}], + "call_type": "acompletion", + "combined_usage_object": recovered_usage, + "response_cost": 3.5e-05, + } + ), + "metadata": {}, + } + await self._run(request_data) + + assert request_data["combined_usage_object"] is recovered_usage + assert request_data["response_cost"] == 3.5e-05 + + @pytest.mark.asyncio + async def test_dispatched_failure_with_text_completion_prompt(self): + from datetime import datetime + + from litellm.types.utils import Usage + + request_data = { + "litellm_logging_obj": self._logging_obj( + { + "first_api_call_start_time": datetime.now(), + "model": "gpt-3.5-turbo", + "messages": "a plain text-completion prompt string", + "call_type": "atext_completion", + } + ), + "metadata": {}, + } + await self._run(request_data) + + estimated = request_data["combined_usage_object"] + assert isinstance(estimated, Usage) + assert estimated.prompt_tokens > 0 + assert estimated.completion_tokens == 0 + + def _dispatched_request_data(self, messages, optional_params, call_type="acompletion"): + from datetime import datetime + + return { + "litellm_logging_obj": self._logging_obj( + { + "first_api_call_start_time": datetime.now(), + "model": "gpt-3.5-turbo", + "messages": messages, + "optional_params": optional_params, + "call_type": call_type, + } + ), + "metadata": {}, + } + + @pytest.mark.asyncio + async def test_image_message_estimated_without_fetching_image(self): + import litellm as litellm_module + from litellm.types.utils import Usage + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "describe this image"}, + { + "type": "image_url", + "image_url": {"url": "http://127.0.0.1:1/unreachable.png", "detail": "high"}, + }, + ], + } + ] + request_data = self._dispatched_request_data(messages, {}) + await self._run(request_data) + + estimated = request_data["combined_usage_object"] + assert isinstance(estimated, Usage) + expected = litellm_module.token_counter( + model="gpt-3.5-turbo", messages=messages, use_default_image_token_count=True + ) + assert estimated.prompt_tokens == expected + assert estimated.prompt_tokens > 0 + + @pytest.mark.asyncio + async def test_embedding_string_list_input_counted_in_estimate(self): + import litellm as litellm_module + from litellm.types.utils import Usage + + embedding_input = ["first embedding text", "second embedding text"] + request_data = self._dispatched_request_data(embedding_input, {}, call_type="aembedding") + await self._run(request_data) + + estimated = request_data["combined_usage_object"] + assert isinstance(estimated, Usage) + expected = litellm_module.token_counter(model="gpt-3.5-turbo", text="".join(embedding_input)) + assert estimated.prompt_tokens == expected + + @pytest.mark.asyncio + async def test_transcription_checksum_not_estimated(self): + request_data = self._dispatched_request_data("a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6", {}, call_type="atranscription") + await self._run(request_data) + + assert "combined_usage_object" not in request_data + assert "response_cost" not in request_data + + @pytest.mark.asyncio + async def test_anthropic_system_prompt_counted_in_estimate(self): + import litellm as litellm_module + from litellm.types.utils import Usage + + system_prompt = "You are a verbose historian who narrates every fact in exhaustive detail." + messages = [{"role": "user", "content": "write a short essay"}] + request_data = self._dispatched_request_data(messages, {"system": system_prompt, "max_tokens": 100}) + await self._run(request_data) + + estimated = request_data["combined_usage_object"] + assert isinstance(estimated, Usage) + expected = litellm_module.token_counter(model="gpt-3.5-turbo", messages=messages) + litellm_module.token_counter( + model="gpt-3.5-turbo", text=system_prompt + ) + assert estimated.prompt_tokens == expected + + @pytest.mark.asyncio + async def test_anthropic_system_text_blocks_counted_in_estimate(self): + import litellm as litellm_module + from litellm.types.utils import Usage + + system_blocks = [ + {"type": "text", "text": "part one of the system prompt. "}, + {"type": "text", "text": "part two of the system prompt."}, + ] + messages = [{"role": "user", "content": "write a short essay"}] + request_data = self._dispatched_request_data(messages, {"system": system_blocks}) + await self._run(request_data) + + estimated = request_data["combined_usage_object"] + assert isinstance(estimated, Usage) + expected = litellm_module.token_counter(model="gpt-3.5-turbo", messages=messages) + litellm_module.token_counter( + model="gpt-3.5-turbo", text="part one of the system prompt. part two of the system prompt." + ) + assert estimated.prompt_tokens == expected + + @pytest.mark.asyncio + async def test_responses_instructions_counted_in_estimate(self): + import litellm as litellm_module + from litellm.types.utils import Usage + + instructions = "Answer every question as a meticulous archivist." + request_data = self._dispatched_request_data("summarize the archive", {"instructions": instructions}) + await self._run(request_data) + + estimated = request_data["combined_usage_object"] + assert isinstance(estimated, Usage) + expected = litellm_module.token_counter( + model="gpt-3.5-turbo", text="summarize the archive" + ) + litellm_module.token_counter(model="gpt-3.5-turbo", text=instructions) + assert estimated.prompt_tokens == expected + + @pytest.mark.asyncio + async def test_request_body_system_counted_when_optional_params_empty(self): + import litellm as litellm_module + from litellm.types.utils import Usage + + system_prompt = "You are a meticulous cartographer who labels every landmark." + messages = [{"role": "user", "content": "draw me a map"}] + request_data = { + **self._dispatched_request_data(messages, {}, call_type="aanthropic_messages"), + "system": system_prompt, + } + await self._run(request_data) + + estimated = request_data["combined_usage_object"] + assert isinstance(estimated, Usage) + expected = litellm_module.token_counter(model="gpt-3.5-turbo", messages=messages) + litellm_module.token_counter( + model="gpt-3.5-turbo", text=system_prompt + ) + assert estimated.prompt_tokens == expected + + @pytest.mark.asyncio + async def test_optional_params_system_wins_over_request_body_system(self): + import litellm as litellm_module + from litellm.types.utils import Usage + + dispatched_system = "short dispatched system prompt" + messages = [{"role": "user", "content": "hello"}] + request_data = { + **self._dispatched_request_data(messages, {"system": dispatched_system}), + "system": "a much longer request body system prompt that must not be double counted here", + } + await self._run(request_data) + + estimated = request_data["combined_usage_object"] + assert isinstance(estimated, Usage) + expected = litellm_module.token_counter(model="gpt-3.5-turbo", messages=messages) + litellm_module.token_counter( + model="gpt-3.5-turbo", text=dispatched_system + ) + assert estimated.prompt_tokens == expected + + from typing import cast import litellm diff --git a/tests/test_litellm/proxy/test_route_a2a_models.py b/tests/test_litellm/proxy/test_route_a2a_models.py index 02e4bddcee0..0523e796543 100644 --- a/tests/test_litellm/proxy/test_route_a2a_models.py +++ b/tests/test_litellm/proxy/test_route_a2a_models.py @@ -50,6 +50,7 @@ async def test_route_a2a_model_bypasses_router(): ) mock_registry = Mock() + mock_registry.get_agent_by_id = Mock(return_value=None) mock_registry.get_agent_by_name = Mock(return_value=mock_agent) # Mock litellm.acompletion to verify it's called @@ -106,3 +107,79 @@ async def test_route_non_a2a_model_raises_error_if_not_in_router(): user_model=None, route_type="acompletion", ) + + +class _DbAgentRow: + def __init__(self, agent_id: str, agent_name: str) -> None: + self.agent_id = agent_id + self.agent_name = agent_name + self.object_permission = None + self.spend = 0.0 + + def model_dump(self): + return { + "agent_id": self.agent_id, + "agent_name": self.agent_name, + "agent_card_params": {"name": self.agent_name, "url": "http://sibling-db-agent.example.com"}, + "litellm_params": {}, + "object_permission": None, + "spend": self.spend, + } + + +def _router_without_models(): + mock_router = Mock() + mock_router.model_names = [] + mock_router.deployment_names = [] + mock_router.has_model_id = Mock(return_value=False) + mock_router.model_group_alias = None + mock_router.router_general_settings = Mock(pass_through_all_models=False) + mock_router.default_deployment = None + mock_router.pattern_router = Mock(patterns=[]) + mock_router.map_team_model = Mock(return_value=None) + mock_router.is_recognized_model = Mock(return_value=False) + mock_router.team_public_model_names = [] + return mock_router + + +@pytest.mark.asyncio +async def test_route_a2a_model_read_through_recovers_agent_created_on_sibling_replica(monkeypatch): + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry + + agent_name = "a2a-sibling-replica-agent" + prisma_client = Mock() + prisma_client.db.litellm_agentstable.find_unique = AsyncMock( + side_effect=[None, _DbAgentRow("a2a-sibling-replica-agent-id", agent_name)] + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + + original_agents = list(global_agent_registry.agent_list) + original_config_agents = getattr(global_agent_registry, "config_agents", ()) + global_agent_registry.agent_list = [] + global_agent_registry.config_agents = () + + data = { + "model": f"a2a/{agent_name}", + "messages": [{"role": "user", "content": "Hello"}], + } + mock_acompletion = AsyncMock(return_value={"id": "read-through-response"}) + + try: + with patch("litellm.acompletion", mock_acompletion): + await route_request( + data=data, + llm_router=_router_without_models(), + user_model=None, + route_type="acompletion", + ) + finally: + global_agent_registry.agent_list = original_agents + global_agent_registry.config_agents = original_config_agents + + mock_acompletion.assert_called_once() + call_kwargs = mock_acompletion.call_args.kwargs + assert call_kwargs["model"] == f"a2a/{agent_name}" + assert call_kwargs["api_base"] == "http://sibling-db-agent.example.com" + prisma_client.db.litellm_agentstable.find_unique.assert_awaited() diff --git a/tests/test_litellm/proxy/test_route_llm_request.py b/tests/test_litellm/proxy/test_route_llm_request.py index fc3b14592cd..1e716f7c148 100644 --- a/tests/test_litellm/proxy/test_route_llm_request.py +++ b/tests/test_litellm/proxy/test_route_llm_request.py @@ -1119,6 +1119,126 @@ async def test_route_request_rejects_chat_completion_without_messages(): llm_router.acompletion.assert_not_called() +class FakeProxyModelTable: + def __init__(self, rows): + self.rows = rows + self.find_many_wheres = [] + + async def find_many(self, where=None, **kwargs): + self.find_many_wheres.append(where) + return list(self.rows) + + +def _fake_prisma_client_with_models(rows): + from types import SimpleNamespace + + table = FakeProxyModelTable(rows) + return SimpleNamespace(db=SimpleNamespace(litellm_proxymodeltable=table)), table + + +def _db_model_row(model_name: str, mock_response: str): + from types import SimpleNamespace + + return SimpleNamespace( + model_id=f"{model_name}-id", + model_name=model_name, + litellm_params={"model": "openai/gpt-4o", "api_key": "fake", "mock_response": mock_response}, + model_info={}, + blocked=False, + ) + + +@pytest.mark.asyncio +async def test_route_request_read_through_recovers_model_created_on_sibling_replica(monkeypatch): + """Regression: a model written to the DB by another replica must be served on + first request instead of 400ing until the periodic config reload.""" + import litellm + import litellm.proxy.proxy_server as proxy_server + + model_name = "e2e-sibling-replica-model" + router = litellm.Router( + model_list=[ + { + "model_name": "some-other-model", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"}, + } + ] + ) + fake_prisma, table = _fake_prisma_client_with_models([_db_model_row(model_name, "hello-from-db")]) + monkeypatch.setattr(proxy_server, "prisma_client", fake_prisma) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + monkeypatch.setattr(proxy_server, "llm_router", router) + + llm_call = await route_request( + data={"model": model_name, "messages": [{"role": "user", "content": "hi"}]}, + llm_router=router, + user_model=None, + route_type="acompletion", + ) + response = await llm_call + + assert response.choices[0].message.content == "hello-from-db" + assert len(table.find_many_wheres) == 1 + assert table.find_many_wheres[0] == {"model_name": model_name} + + +@pytest.mark.asyncio +async def test_route_request_unknown_model_raises_and_hits_db_once_within_ttl(monkeypatch): + import litellm + import litellm.proxy.proxy_server as proxy_server + + model_name = "e2e-model-nobody-created" + router = litellm.Router( + model_list=[ + { + "model_name": "some-other-model", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"}, + } + ] + ) + fake_prisma, table = _fake_prisma_client_with_models([]) + monkeypatch.setattr(proxy_server, "prisma_client", fake_prisma) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + monkeypatch.setattr(proxy_server, "llm_router", router) + + data = {"model": model_name, "messages": [{"role": "user", "content": "hi"}]} + with pytest.raises(ProxyModelNotFoundError): + await route_request(data=data, llm_router=router, user_model=None, route_type="acompletion") + with pytest.raises(ProxyModelNotFoundError): + await route_request(data=data, llm_router=router, user_model=None, route_type="acompletion") + + assert table.find_many_wheres == [{"model_name": model_name}, {"model_id": model_name}] + + +@pytest.mark.asyncio +async def test_route_request_read_through_disabled_without_store_model_in_db(monkeypatch): + import litellm + import litellm.proxy.proxy_server as proxy_server + + model_name = "e2e-config-only-proxy-model" + router = litellm.Router( + model_list=[ + { + "model_name": "some-other-model", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"}, + } + ] + ) + fake_prisma, table = _fake_prisma_client_with_models([_db_model_row(model_name, "should-not-load")]) + monkeypatch.setattr(proxy_server, "prisma_client", fake_prisma) + monkeypatch.setattr(proxy_server, "store_model_in_db", False) + monkeypatch.setattr(proxy_server, "llm_router", router) + + with pytest.raises(ProxyModelNotFoundError): + await route_request( + data={"model": model_name, "messages": [{"role": "user", "content": "hi"}]}, + llm_router=router, + user_model=None, + route_type="acompletion", + ) + + assert table.find_many_wheres == [] + @pytest.mark.asyncio async def test_route_request_routing_group_name_passes_model_gate(): from unittest.mock import AsyncMock, patch @@ -1141,3 +1261,39 @@ async def test_route_request_routing_group_name_passes_model_gate(): assert response == "group_response" spy.assert_called_once_with(**data) + + +@pytest.mark.asyncio +async def test_route_request_a2a_agent_miss_does_not_consume_model_read_through(monkeypatch): + from types import SimpleNamespace + from unittest.mock import AsyncMock + + import litellm + import litellm.proxy.proxy_server as proxy_server + + model_name = "a2a/agent-nobody-created" + router = litellm.Router( + model_list=[ + { + "model_name": "some-other-model", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"}, + } + ] + ) + fake_prisma, model_table = _fake_prisma_client_with_models([]) + agents_find_unique = AsyncMock(return_value=None) + fake_prisma.db.litellm_agentstable = SimpleNamespace(find_unique=agents_find_unique) + monkeypatch.setattr(proxy_server, "prisma_client", fake_prisma) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + monkeypatch.setattr(proxy_server, "llm_router", router) + + with pytest.raises(ProxyModelNotFoundError): + await route_request( + data={"model": model_name, "messages": [{"role": "user", "content": "hi"}]}, + llm_router=router, + user_model=None, + route_type="acompletion", + ) + + assert agents_find_unique.await_count == 2 + assert model_table.find_many_wheres == [] diff --git a/tests/test_litellm/responses/test_responses_websocket_all_providers.py b/tests/test_litellm/responses/test_responses_websocket_all_providers.py index 4509abc7749..2d523bfdeb3 100644 --- a/tests/test_litellm/responses/test_responses_websocket_all_providers.py +++ b/tests/test_litellm/responses/test_responses_websocket_all_providers.py @@ -1030,6 +1030,171 @@ class TestWebSocketErrorHandling: assert "Invalid JSON" in error_event +class TestWebSocketProjectQuotaEnforcement: + """VERIA regression: the connection-level pre-call hook only runs once, + but a WebSocket connection accepts many response.create frames. Every + frame must be checked against any registered project ITPM/OTPM quota + callback, not just the first one.""" + + @pytest.mark.asyncio + async def test_managed_handler_blocks_frame_rejected_by_quota_callback(self, monkeypatch): + from unittest.mock import AsyncMock, MagicMock + + import litellm + from litellm.exceptions import RateLimitError + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.responses.streaming_iterator import ( + ManagedResponsesWebSocketHandler, + ) + + aresponses_called = False + + async def fake_aresponses(*args, **kwargs): + nonlocal aresponses_called + aresponses_called = True + + monkeypatch.setattr(litellm, "aresponses", fake_aresponses) + + quota_callback = MagicMock() + quota_callback.enforce_project_io_token_quota_for_frame = AsyncMock( + side_effect=RateLimitError(message="project OTPM exceeded", llm_provider="", model="") + ) + + mock_websocket = MagicMock() + mock_websocket.send_text = AsyncMock() + mock_logging_obj = Logging( + model="test-model", + messages=[], + stream=True, + call_type="aresponses", + start_time=0, + litellm_call_id="test-id", + function_id="test-func", + ) + handler = ManagedResponsesWebSocketHandler( + websocket=mock_websocket, + model="test-model", + logging_obj=mock_logging_obj, + quota_callbacks=[quota_callback], + ) + + await handler._process_response_create(json.dumps({"type": "response.create", "input": "hi"})) + + quota_callback.enforce_project_io_token_quota_for_frame.assert_awaited_once() + assert aresponses_called is False + mock_websocket.send_text.assert_called_once() + error_event = mock_websocket.send_text.call_args[0][0] + assert "rate_limit_exceeded" in error_event + + @pytest.mark.asyncio + async def test_managed_handler_forwards_frame_allowed_by_quota_callback(self, monkeypatch): + from unittest.mock import AsyncMock, MagicMock + + import litellm + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.responses.streaming_iterator import ( + ManagedResponsesWebSocketHandler, + ) + + aresponses_called = False + + async def fake_aresponses(*args, **kwargs): + nonlocal aresponses_called + aresponses_called = True + + async def _empty(): + return + yield + + return _empty() + + monkeypatch.setattr(litellm, "aresponses", fake_aresponses) + + quota_callback = MagicMock() + quota_callback.enforce_project_io_token_quota_for_frame = AsyncMock(return_value=None) + + mock_websocket = MagicMock() + mock_websocket.send_text = AsyncMock() + mock_logging_obj = Logging( + model="test-model", + messages=[], + stream=True, + call_type="aresponses", + start_time=0, + litellm_call_id="test-id", + function_id="test-func", + ) + handler = ManagedResponsesWebSocketHandler( + websocket=mock_websocket, + model="test-model", + logging_obj=mock_logging_obj, + quota_callbacks=[quota_callback], + ) + + await handler._process_response_create(json.dumps({"type": "response.create", "input": "hi"})) + + quota_callback.enforce_project_io_token_quota_for_frame.assert_awaited_once() + assert aresponses_called is True + + @pytest.mark.asyncio + async def test_native_handler_blocks_frame_rejected_by_quota_callback(self): + from unittest.mock import AsyncMock, MagicMock + + from litellm.exceptions import RateLimitError + from litellm.responses.streaming_iterator import ResponsesWebSocketStreaming + + quota_callback = MagicMock() + quota_callback.enforce_project_io_token_quota_for_frame = AsyncMock( + side_effect=RateLimitError(message="project OTPM exceeded", llm_provider="", model="") + ) + + mock_backend_ws = MagicMock() + mock_backend_ws.send = AsyncMock() + mock_websocket = MagicMock() + mock_websocket.send_text = AsyncMock() + + handler = ResponsesWebSocketStreaming( + websocket=mock_websocket, + backend_ws=mock_backend_ws, + logging_obj=MagicMock(), + authorized_model="gpt-4o", + quota_callbacks=[quota_callback], + ) + + allowed = await handler._enforce_or_reject_frame( + json.dumps({"type": "response.create", "input": "hi"}) + ) + + assert allowed is False + mock_backend_ws.send.assert_not_called() + mock_websocket.send_text.assert_called_once() + assert "rate_limit_exceeded" in mock_websocket.send_text.call_args[0][0] + + @pytest.mark.asyncio + async def test_native_handler_forwards_frame_allowed_by_quota_callback(self): + from unittest.mock import AsyncMock, MagicMock + + from litellm.responses.streaming_iterator import ResponsesWebSocketStreaming + + quota_callback = MagicMock() + quota_callback.enforce_project_io_token_quota_for_frame = AsyncMock(return_value=None) + + handler = ResponsesWebSocketStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=MagicMock(), + authorized_model="gpt-4o", + quota_callbacks=[quota_callback], + ) + + allowed = await handler._enforce_or_reject_frame( + json.dumps({"type": "response.create", "input": "hi"}) + ) + + assert allowed is True + quota_callback.enforce_project_io_token_quota_for_frame.assert_awaited_once() + + class TestNativeWebSocketGuardrails: @pytest.mark.asyncio async def test_response_create_injects_authorized_model(self): diff --git a/tests/test_litellm/router_utils/test_fallback_event_handlers.py b/tests/test_litellm/router_utils/test_fallback_event_handlers.py index 68395737469..24477248a8a 100644 --- a/tests/test_litellm/router_utils/test_fallback_event_handlers.py +++ b/tests/test_litellm/router_utils/test_fallback_event_handlers.py @@ -1,4 +1,5 @@ import json +from typing import NoReturn from unittest.mock import MagicMock, patch import httpx @@ -167,6 +168,10 @@ async def _acreate_batch(*args, **kwargs): raise AssertionError("only used for its __name__") +async def _acreate_file(*args: object, **kwargs: object) -> NoReturn: + raise AssertionError("only used for its __name__") + + @pytest.mark.asyncio async def test_run_async_fallback_keeps_uploaded_file_requests_in_their_model_group(): """An input_file_id only exists under the credentials of the group it was uploaded @@ -229,6 +234,46 @@ async def test_run_async_fallback_allows_same_model_group_retry_for_uploaded_fil assert router.attempted_model_groups == ["openai-group"] +@pytest.mark.asyncio +async def test_run_async_fallback_keeps_file_creation_in_its_model_group(): + """A file created for batches lands in the account of the deployment that stored it, + and its id is only usable against the model group the caller named. A cross-group + fallback silently stores the file with the wrong provider.""" + router = AttemptRecordingRouter() + + with pytest.raises(RuntimeError, match="azure connection error"): + await run_async_fallback( + litellm_router=router, + fallback_model_group=["openai-group"], + original_model_group="azure-group", + original_exception=RuntimeError("azure connection error"), + max_fallbacks=3, + fallback_depth=0, + model="azure-group", + original_function=_acreate_file, + ) + + assert router.attempted_model_groups == [] + + +@pytest.mark.asyncio +async def test_run_async_fallback_allows_same_model_group_retry_for_file_creation(): + router = AttemptRecordingRouter() + + await run_async_fallback( + litellm_router=router, + fallback_model_group=[{"model": "azure-group", "_target_order": 2}], + original_model_group="azure-group", + original_exception=RuntimeError("first deployment failed"), + max_fallbacks=3, + fallback_depth=0, + model="azure-group", + original_function=_acreate_file, + ) + + assert router.attempted_model_groups == ["azure-group"] + + @pytest.mark.asyncio async def test_run_async_fallback_still_crosses_model_groups_without_an_uploaded_file(): router = AttemptRecordingRouter() diff --git a/tests/test_litellm/test_conftest.py b/tests/test_litellm/test_conftest.py new file mode 100644 index 00000000000..cca4f7c3ef2 --- /dev/null +++ b/tests/test_litellm/test_conftest.py @@ -0,0 +1,43 @@ +import os +import subprocess +import sys +from pathlib import Path +from typing import Final + +REPO_ROOT: Final = Path(__file__).resolve().parents[2] + +PROXY_BASE_URL_SENSITIVE_NODE: Final = ( + "tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py" + "::TestTemporaryMCPSessionEndpoints" + "::test_mcp_token_opens_sealed_passthrough_code_and_exchanges_with_minted_client" +) + +COVERAGE_SUBPROCESS_VARS: Final = frozenset( + {"COV_CORE_SOURCE", "COV_CORE_CONFIG", "COV_CORE_DATAFILE", "COV_CORE_CONTEXT", "COVERAGE_PROCESS_START"} +) + + +def test_host_proxy_base_url_cannot_reach_request_derived_url_tests(): + child_env: Final = { + key: value for key, value in os.environ.items() if key not in COVERAGE_SUBPROCESS_VARS + } | {"PROXY_BASE_URL": "https://leaked-host-origin.example.com"} + + completed: Final = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + PROXY_BASE_URL_SENSITIVE_NODE, + "-q", + "--no-header", + "-p", + "no:cacheprovider", + ], + cwd=REPO_ROOT, + env=child_env, + capture_output=True, + text=True, + check=False, + ) + + assert completed.returncode == 0, completed.stdout + completed.stderr diff --git a/tests/test_litellm/test_logging.py b/tests/test_litellm/test_logging.py index 9ab362f6cd5..784ec5b6cf4 100644 --- a/tests/test_litellm/test_logging.py +++ b/tests/test_litellm/test_logging.py @@ -177,7 +177,10 @@ def test_json_formatter_parses_embedded_python_dict_repr(): # Python dict parsed and promoted to first-class properties assert obj["model_name"] == "text-embedding-3-large" assert "litellm_params" in obj - assert obj["litellm_params"]["api_key"] == "sk**********" + # Redacted, not passed through: SecretRedactionFilter already collapses this + # pair in the plain path before any formatter sees it, so the JSON path matching + # it is production parity. The key survives because redaction is per-value here. + assert obj["litellm_params"]["api_key"] == "REDACTED" assert obj["litellm_params"]["tpm"] == 1000000 assert obj["litellm_params"]["use_in_pass_through"] is False assert "model_info" in obj @@ -185,6 +188,38 @@ def test_json_formatter_parses_embedded_python_dict_repr(): assert obj["model_info"]["db_model"] is False +def test_json_formatter_output_stays_parseable_when_a_secret_is_redacted(): + """Redaction must collapse the value only, never the surrounding JSON member. + + Redacting the serialized document turned '"api_key": "sk-..."' into a bare + REDACTED token, so the line stopped being valid JSON entirely. + """ + formatter = JsonFormatter() + record = logging.LogRecord( + name="LiteLLM", + level=logging.INFO, + pathname="", + lineno=0, + msg="calling deployment", + args=(), + exc_info=None, + ) + record.deployment = { + "api_key": "sk-abcdefghijklmnopqrstuvwxyz0123456789", + "aws_secret_access_key": "wJalrXUtnFEMIQAfakeKEYbPxRfiCYEXAMPLEKEY", + "aws_region_name": "us-east-1", + "nested": {"tokens": ["Bearer abcdefghijklmnop", "keep-me"]}, + } + + obj = json.loads(formatter.format(record)) + + assert obj["deployment"]["api_key"] == "REDACTED" + assert obj["deployment"]["aws_secret_access_key"] == "REDACTED" + # Non-secret siblings stay legible so the logs remain useful + assert obj["deployment"]["aws_region_name"] == "us-east-1" + assert obj["deployment"]["nested"]["tokens"] == ["REDACTED", "keep-me"] + + def test_json_formatter_includes_component_field(): """ Test that JsonFormatter always emits a 'component' field equal to the logger name. diff --git a/tests/test_litellm/test_responses_api_bridge_non_stream.py b/tests/test_litellm/test_responses_api_bridge_non_stream.py index 25a3bc2dbba..c272b151865 100644 --- a/tests/test_litellm/test_responses_api_bridge_non_stream.py +++ b/tests/test_litellm/test_responses_api_bridge_non_stream.py @@ -1,6 +1,6 @@ import os import sys -from typing import Optional +from typing import Final, Optional from unittest.mock import Mock import pytest @@ -192,6 +192,32 @@ def test_transform_usage_with_cached_tokens_only(): print("✓ Transformation works with cached_tokens only") +def test_transform_usage_maps_nested_cache_creation_input_tokens(): + """ + Regression (LIT-5757): DashScope nests cache_creation_input_tokens inside + prompt_tokens_details; the bridge must surface it as cache_write_tokens. + """ + usage: Final = Usage( + prompt_tokens=2059, + completion_tokens=31, + total_tokens=2090, + prompt_tokens_details={ + "cached_tokens": 0, + "text_tokens": 2059, + "cache_type": "ephemeral", + "cache_creation_input_tokens": 2048, + "cache_creation": {"ephemeral_5m_input_tokens": 2048}, + }, + ) + + responses_usage: Final = LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage( + usage + ) + + assert responses_usage.input_tokens_details is not None + assert responses_usage.input_tokens_details.cache_write_tokens == 2048 + + def test_transform_usage_with_reasoning_tokens_only(): """ Test transformation when only reasoning_tokens is provided (no cached_tokens). diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 49ed236356c..65debae9a16 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -492,6 +492,54 @@ async def test_async_router_acreate_file_with_jsonl(): assert first_call_content == non_jsonl_content +@pytest.mark.asyncio +async def test_async_router_acreate_file_does_not_fall_back_across_model_groups(): + """A file created for batches only exists under the credentials of the model group + the caller named. A cross-group fallback silently stores it with the wrong provider + and the later batch create against the named group permanently fails.""" + from unittest.mock import MagicMock, patch + + router = litellm.Router( + model_list=[ + { + "model_name": "azure-gpt", + "litellm_params": { + "model": "azure/my-azure-deployment", + "api_base": "http://127.0.0.1:9", + "api_key": "dummy-key", + "api_version": "2024-06-01", + }, + }, + { + "model_name": "openai-gpt", + "litellm_params": {"model": "gpt-4o-mini"}, + }, + ], + fallbacks=[{"azure-gpt": ["openai-gpt"]}], + ) + + def fail_azure(*args: object, **kwargs: object) -> MagicMock: + if kwargs.get("model") == "azure/my-azure-deployment": + raise litellm.APIConnectionError( + message="Connection error.", + llm_provider="azure", + model="azure/my-azure-deployment", + ) + return MagicMock() + + with patch("litellm.acreate_file", side_effect=fail_azure) as mock_acreate_file: + with pytest.raises(litellm.APIConnectionError): + await router.acreate_file( + model="azure-gpt", + purpose="batch", + file=MagicMock(), + ) + + called_models = [call.kwargs.get("model") for call in mock_acreate_file.call_args_list] + assert "azure/my-azure-deployment" in called_models + assert "gpt-4o-mini" not in called_models + + @pytest.mark.asyncio async def test_async_router_acreate_file_uses_deployment_custom_llm_provider(): """ @@ -7595,6 +7643,95 @@ def test_pre_call_checks_keeps_deployment_when_provider_is_unresolvable(monkeypa assert len(result) == 1 +class TestUpsertDeploymentRollback: + """ + Regression tests: `upsert_deployment` pops the previous deployment before + re-adding the edited one. When the re-add raises under + `ignore_invalid_deployments=True`, the pop must be rolled back so this pod + keeps serving the previous configuration instead of silently dropping a live + deployment (the "Error upserting deployment" drop behind the access-group + reload 500 in the 2-replica e2e suite). + """ + + def test_failed_upsert_keeps_previous_deployment_serving(self): + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + router = litellm.Router( + model_list=[ + { + "model_name": "prod-model", + "litellm_params": {"model": "gpt-4o", "api_key": "sk-old"}, + "model_info": {"id": "prod-1", "db_model": True}, + } + ], + ignore_invalid_deployments=True, + ) + + result = router.upsert_deployment( + deployment=Deployment( + model_name="prod-model", + litellm_params=LiteLLM_Params(model="auto_router/broken"), + model_info=ModelInfo(id="prod-1", db_model=True), + ) + ) + + assert result is None + restored = router.get_deployment(model_id="prod-1") + assert restored is not None + assert restored.litellm_params.model == "gpt-4o" + assert [model["model_name"] for model in router.model_list] == ["prod-model"] + + def test_failed_fresh_add_returns_none_without_restore(self): + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + router = litellm.Router(model_list=[], ignore_invalid_deployments=True) + + result = router.upsert_deployment( + deployment=Deployment( + model_name="fresh-router", + litellm_params=LiteLLM_Params(model="auto_router/broken"), + model_info=ModelInfo(id="fresh-1", db_model=True), + ) + ) + + assert result is None + assert router.get_deployment(model_id="fresh-1") is None + assert router.model_list == [] + + def test_restore_re_adds_popped_deployment(self): + router = litellm.Router( + model_list=[ + { + "model_name": "prod-model", + "litellm_params": {"model": "gpt-4o", "api_key": "sk-old"}, + "model_info": {"id": "prod-1", "db_model": True}, + } + ], + ignore_invalid_deployments=True, + ) + previous = router.get_deployment(model_id="prod-1") + router.delete_deployment(id="prod-1") + assert router.has_model_id("prod-1") is False + + router._restore_deployment_after_failed_upsert( + previous_deployment=previous, model_id="prod-1" + ) + + restored = router.get_deployment(model_id="prod-1") + assert restored is not None + assert restored.litellm_params.model == "gpt-4o" + + router._restore_deployment_after_failed_upsert( + previous_deployment=previous, model_id="prod-1" + ) + assert len(router.model_list) == 1 + + router._restore_deployment_after_failed_upsert( + previous_deployment=None, model_id="prod-1" + ) + assert len(router.model_list) == 1 + + class TestConsumedRequestTagsStamp: """Issue #36621: when a request's tags select a tagged pre-routing strategy, those tags are consumed by the selection; the hook must stamp the rewritten model group so @@ -7794,6 +7931,21 @@ class TestTaggedAutoRouterOnSharedModelName: assert response is not None assert response.model == "gemini-flash" + @pytest.mark.asyncio + async def test_request_level_tag_filtering_from_key_settings_bypasses_the_marker(self): + router = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=False) + + assert await self._hook_response(router, {"enable_tag_filtering": True}) is None + + @pytest.mark.asyncio + async def test_globally_disabled_filtering_still_lets_the_marker_capture_untagged_requests(self): + router = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=False) + + response = await self._hook_response(router, {}) + + assert response is not None + assert response.model == "gemini-flash" + @pytest.mark.asyncio async def test_marker_only_alias_still_captures_untagged_requests(self): router = self._router(marker_tags=["route"], include_plain_sibling=False, enable_tag_filtering=True) diff --git a/tests/test_litellm/test_secret_redaction.py b/tests/test_litellm/test_secret_redaction.py index a5f3f912339..0188d87dfdb 100644 --- a/tests/test_litellm/test_secret_redaction.py +++ b/tests/test_litellm/test_secret_redaction.py @@ -495,3 +495,54 @@ def test_redaction_survives_uvicorn_logging_reconfiguration(): lg.handlers[:] = handlers lg.setLevel(level) lg.propagate = True + + +def test_aws_credential_redaction_catches_quoted_values(): + """AWS creds appear as quoted dict-repr values, not just bare key=value.""" + cases = ( + "{'aws_secret_access_key': 'wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY'}", + '{"aws_session_token": "IQoJb3JpZ2luX2VjEaCXVzLWVhc3QtMSJHMEUCIQ"}', + "aws_session_token: 'FwoGZXIvYXdzEBYaDHh4eHh4eHh4eHh4eCLLAe'", + "aws_secret_access_key=wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY", + "{'aws_access_key_id': 'not-an-akia-shaped-value'}", + ) + for secret_line in cases: + result = redact_string(secret_line) + assert "REDACTED" in result, f"AWS redaction missed: {secret_line!r}" + assert "wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY" not in result + assert "IQoJb3JpZ2luX2VjEaCXVzLWVhc3QtMSJHMEUCIQ" not in result + + safe = "'aws_region_name': 'us-east-1'" + assert redact_string(safe) == safe + + +@pytest.mark.parametrize( + "extra", + ( + {"api_base": {f"https://host/v1?key={SECRET}"}}, + {"blob": {"authorization": f"Bearer {SECRET}"}}, + {"blob": [f"Bearer {SECRET}"]}, + {"blob": ({"nested": {"deep": SECRET}},)}, + ), + ids=("set", "dict", "list", "nested"), +) +def test_json_formatter_redacts_non_string_extra_values(extra): + """SecretRedactionFilter only scrubs str attrs, so containers must be caught on render.""" + buf = StringIO() + handler = logging.StreamHandler(buf) + handler.setFormatter(JsonFormatter()) + handler.addFilter(_secret_filter) + + logger = logging.getLogger("test_json_extra_redaction") + logger.handlers = [handler] + logger.setLevel(logging.DEBUG) + logger.propagate = False + try: + logger.warning("request sent", extra=extra) + finally: + logger.handlers = [] + + output = buf.getvalue() + assert output.strip(), "no record captured" + assert SECRET not in output, f"non-string extra leaked a secret: {output}" + assert "REDACTED" in output diff --git a/tests/test_litellm/test_thinking_enabled.py b/tests/test_litellm/test_thinking_enabled.py index 8ba406c395a..744b258e617 100644 --- a/tests/test_litellm/test_thinking_enabled.py +++ b/tests/test_litellm/test_thinking_enabled.py @@ -60,6 +60,7 @@ class TestIsThinkingEnabled: ({"reasoning_effort": "medium"}, True), # both thinking enabled and reasoning_effort returns True ({"thinking": {"type": "enabled"}, "reasoning_effort": "high"}, True), + ({"thinking": True}, True), # falsy thinking values should not crash ({"thinking": False}, False), ({"thinking": 0}, False), diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index afdfdf170ac..efccdc4a986 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -3766,6 +3766,20 @@ class TestValidateAndFixThinkingParam: assert "budgetTokens" in thinking assert "budget_tokens" not in thinking + def test_bool_true_maps_to_enabled_with_default_budget(self): + from litellm.constants import DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET + from litellm.utils import validate_and_fix_thinking_param + + assert validate_and_fix_thinking_param(thinking=True) == { + "type": "enabled", + "budget_tokens": DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, + } + + def test_bool_false_returns_none(self): + from litellm.utils import validate_and_fix_thinking_param + + assert validate_and_fix_thinking_param(thinking=False) is None + def test_deepseek_v4_models_in_cost_map(): """ diff --git a/tests/test_litellm/types/test_types_utils.py b/tests/test_litellm/types/test_types_utils.py index cd5e8dda012..672aa84cc73 100644 --- a/tests/test_litellm/types/test_types_utils.py +++ b/tests/test_litellm/types/test_types_utils.py @@ -1,5 +1,6 @@ import os import sys +from typing import Final import pytest @@ -75,6 +76,29 @@ def test_usage_dump(): assert new_usage.prompt_tokens_details.web_search_requests == 1 +def test_prompt_tokens_details_maps_nested_cache_creation_input_tokens(): + """Regression (LIT-5757): DashScope nests the Anthropic-spelled + cache_creation_input_tokens inside prompt_tokens_details. It must populate + the canonical cache_write_tokens/cache_creation_tokens pair, without + overriding an explicitly provided canonical value.""" + from litellm.types.utils import PromptTokensDetailsWrapper + + nested: Final = PromptTokensDetailsWrapper( + cached_tokens=0, text_tokens=2059, cache_creation_input_tokens=2048 + ) + assert nested.cache_write_tokens == 2048 + assert nested.cache_creation_tokens == 2048 + + explicit: Final = PromptTokensDetailsWrapper( + cache_write_tokens=100, cache_creation_input_tokens=2048 + ) + assert explicit.cache_write_tokens == 100 + assert explicit.cache_creation_tokens == 100 + + non_int: Final = PromptTokensDetailsWrapper(cache_creation_input_tokens=None) + assert not hasattr(non_int, "cache_write_tokens") + + def test_usage_server_tool_use_dict_is_coerced_and_round_trips(): from litellm.types.utils import ServerToolUse, Usage diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 94c1f9b86f9..e7cfff93aa4 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,9 +1,9 @@ { "LIT001": { - "limit": 22897 + "limit": 22892 }, "LIT002": { - "limit": 26888 + "limit": 26886 }, "LIT003": { "limit": 269 @@ -27,7 +27,7 @@ "limit": 0 }, "LIT010": { - "limit": 16713 + "limit": 16699 }, "LIT011": { "limit": 5590 diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index ed72e784a35..c471355db68 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -793,16 +793,6 @@ "count": 1 } }, - "src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/models-and-endpoints/utils/modelDataTransformer.ts": { "prefer-const": { "count": 6 @@ -1055,11 +1045,6 @@ "count": 1 } }, - "src/app/(dashboard)/policies/_components/impact_popover.test.tsx": { - "react/display-name": { - "count": 1 - } - }, "src/app/(dashboard)/policies/_components/impact_popover.tsx": { "local/filename-pascal-case": { "count": 1 @@ -1073,11 +1058,6 @@ "count": 1 } }, - "src/app/(dashboard)/policies/_components/index.test.tsx": { - "react/display-name": { - "count": 1 - } - }, "src/app/(dashboard)/policies/_components/index.tsx": { "local/filename-pascal-case": { "count": 1 @@ -1750,7 +1730,7 @@ }, "src/components/add_model/AddModelForm.test.tsx": { "no-restricted-imports": { - "count": 2 + "count": 1 } }, "src/components/add_model/AddModelForm.tsx": { @@ -1761,7 +1741,7 @@ "count": 1 }, "no-restricted-imports": { - "count": 3 + "count": 2 } }, "src/components/add_model/ClassificationMethodConfig.tsx": { @@ -1813,9 +1793,6 @@ }, "no-restricted-imports": { "count": 3 - }, - "prefer-const": { - "count": 2 } }, "src/components/add_model/auto_router_connection_test.tsx": { @@ -1826,17 +1803,6 @@ "src/components/add_model/cache_control_settings.tsx": { "local/filename-pascal-case": { "count": 1 - }, - "no-restricted-imports": { - "count": 1 - }, - "prefer-const": { - "count": 1 - } - }, - "src/components/add_model/conditional_public_model_name.test.tsx": { - "no-restricted-imports": { - "count": 1 } }, "src/components/add_model/conditional_public_model_name.tsx": { @@ -1863,11 +1829,6 @@ "count": 1 } }, - "src/components/add_model/litellm_model_name.test.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/add_model/litellm_model_name.tsx": { "local/filename-pascal-case": { "count": 1 @@ -1887,11 +1848,6 @@ "count": 2 } }, - "src/components/add_model/provider_specific_fields.test.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/add_model/provider_specific_fields.tsx": { "local/filename-pascal-case": { "count": 1 @@ -2063,14 +2019,6 @@ "count": 1 } }, - "src/components/common_components/chartUtils.tsx": { - "local/filename-pascal-case": { - "count": 1 - }, - "no-nested-ternary": { - "count": 1 - } - }, "src/components/common_components/check_openapi_schema.tsx": { "local/filename-pascal-case": { "count": 1 @@ -2294,15 +2242,6 @@ "local/filename-pascal-case": { "count": 1 }, - "local/no-complex-jsx-arrow": { - "count": 1 - }, - "max-lines": { - "count": 1 - }, - "no-nested-ternary": { - "count": 14 - }, "no-restricted-imports": { "count": 1 }, @@ -2361,21 +2300,10 @@ "count": 1 } }, - "src/components/organisms/create_key_button.test.tsx": { - "@typescript-eslint/no-require-imports": { - "count": 1 - }, - "react/display-name": { - "count": 8 - } - }, "src/components/organisms/create_key_button.tsx": { "local/filename-pascal-case": { "count": 1 }, - "local/no-complex-jsx-arrow": { - "count": 2 - }, "max-lines": { "count": 1 }, @@ -2669,15 +2597,6 @@ "src/components/templates/key_edit_view.tsx": { "local/filename-pascal-case": { "count": 1 - }, - "local/no-complex-jsx-arrow": { - "count": 2 - }, - "no-nested-ternary": { - "count": 2 - }, - "no-restricted-imports": { - "count": 1 } }, "src/components/templates/key_info_view.tsx": { @@ -3062,9 +2981,6 @@ "tests/setupTests.ts": { "@typescript-eslint/no-this-alias": { "count": 1 - }, - "react/display-name": { - "count": 1 } } } \ No newline at end of file diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 93ee979f37d..186d38234d5 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -18,7 +18,6 @@ "@tanstack/react-pacer": "0.22.1", "@tanstack/react-query": "5.100.7", "@tanstack/react-table": "8.21.3", - "@tremor/react": "3.18.7", "@types/papaparse": "5.5.2", "antd": "5.29.3", "cva": "1.0.0-beta.4", @@ -1460,87 +1459,12 @@ "@floating-ui/utils": "^0.2.11" } }, - "node_modules/@floating-ui/react": { - "version": "0.19.2", - "resolved": "https://registry.npmjs.org/@floating-ui/react/-/react-0.19.2.tgz", - "integrity": "sha512-JyNk4A0Ezirq8FlXECvRtQOX/iBe5Ize0W/pLkrZjfHW9GUV7Xnq6zm6fyZuQzaHHqEnVizmvlA96e1/CkZv+w==", - "license": "MIT", - "dependencies": { - "@floating-ui/react-dom": "^1.3.0", - "aria-hidden": "^1.1.3", - "tabbable": "^6.0.1" - }, - "peerDependencies": { - "react": ">=16.8.0", - "react-dom": ">=16.8.0" - } - }, - "node_modules/@floating-ui/react-dom": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-1.3.0.tgz", - "integrity": "sha512-htwHm67Ji5E/pROEAr7f8IKFShuiCKHwUC/UY4vC3I5jiSvGFAYnSYiZO5MlGmads+QqvUkR9ANHEguGrDv72g==", - "license": "MIT", - "dependencies": { - "@floating-ui/dom": "^1.2.1" - }, - "peerDependencies": { - "react": ">=16.8.0", - "react-dom": ">=16.8.0" - } - }, "node_modules/@floating-ui/utils": { "version": "0.2.11", "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.11.tgz", "integrity": "sha512-RiB/yIh78pcIxl6lLMG0CgBXAZ2Y0eVHqMPYugu+9U0AeT6YBeiJpf7lbdJNIugFP5SIjwNRgo4DhR1Qxi26Gg==", "license": "MIT" }, - "node_modules/@headlessui/react": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@headlessui/react/-/react-2.2.0.tgz", - "integrity": "sha512-RzCEg+LXsuI7mHiSomsu/gBJSjpupm6A1qIZ5sWjd7JhARNlMiSA4kKfJpCKwU9tE+zMRterhhrP74PvfJrpXQ==", - "license": "MIT", - "dependencies": { - "@floating-ui/react": "^0.26.16", - "@react-aria/focus": "^3.17.1", - "@react-aria/interactions": "^3.21.3", - "@tanstack/react-virtual": "^3.8.1" - }, - "engines": { - "node": ">=10" - }, - "peerDependencies": { - "react": "^18 || ^19 || ^19.0.0-rc", - "react-dom": "^18 || ^19 || ^19.0.0-rc" - } - }, - "node_modules/@headlessui/react/node_modules/@floating-ui/react": { - "version": "0.26.28", - "resolved": "https://registry.npmjs.org/@floating-ui/react/-/react-0.26.28.tgz", - "integrity": "sha512-yORQuuAtVpiRjpMhdc0wJj06b9JFjrYF4qp96j++v2NBpbi6SEGF7donUJ3TMieerQ6qVkAv1tgr7L4r5roTqw==", - "license": "MIT", - "dependencies": { - "@floating-ui/react-dom": "^2.1.2", - "@floating-ui/utils": "^0.2.8", - "tabbable": "^6.0.0" - }, - "peerDependencies": { - "react": ">=16.8.0", - "react-dom": ">=16.8.0" - } - }, - "node_modules/@headlessui/react/node_modules/@floating-ui/react-dom": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.1.8.tgz", - "integrity": "sha512-cC52bHwM/n/CxS87FH0yWdngEZrjdtLW/qVruo68qg+prK7ZQ4YGdut2GyDVpoGeAYe/h899rVeOVm6Oi40k2A==", - "license": "MIT", - "dependencies": { - "@floating-ui/dom": "^1.7.6" - }, - "peerDependencies": { - "react": ">=16.8.0", - "react-dom": ">=16.8.0" - } - }, "node_modules/@headlessui/tailwindcss": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/@headlessui/tailwindcss/-/tailwindcss-0.2.2.tgz", @@ -2141,33 +2065,6 @@ "url": "https://opencollective.com/libvips" } }, - "node_modules/@internationalized/date": { - "version": "3.12.1", - "resolved": "https://registry.npmjs.org/@internationalized/date/-/date-3.12.1.tgz", - "integrity": "sha512-6IedsVWXyq4P9Tj+TxuU8WGWM70hYLl12nbYU8jkikVpa6WXapFazPUcHUMDMoWftIDE2ILDkFFte6W2nFCkRQ==", - "license": "Apache-2.0", - "dependencies": { - "@swc/helpers": "^0.5.0" - } - }, - "node_modules/@internationalized/number": { - "version": "3.6.6", - "resolved": "https://registry.npmjs.org/@internationalized/number/-/number-3.6.6.tgz", - "integrity": "sha512-iFgmQaXHE0vytNfpLZWOC2mEJCBRzcUxt53Xf/yCXG93lRvqas237i3r7X4RKMwO3txiyZD4mQjKAByFv6UGSQ==", - "license": "Apache-2.0", - "dependencies": { - "@swc/helpers": "^0.5.0" - } - }, - "node_modules/@internationalized/string": { - "version": "3.2.8", - "resolved": "https://registry.npmjs.org/@internationalized/string/-/string-3.2.8.tgz", - "integrity": "sha512-NdbMQUSfXLYIQol5VyMtinm9pZDciiMfN7RtmSuSB78io1hqwJ0naYfxyW6vgxWBkzWymQa/3uLDlbfmshtCaA==", - "license": "Apache-2.0", - "dependencies": { - "@swc/helpers": "^0.5.0" - } - }, "node_modules/@istanbuljs/schema": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.6.tgz", @@ -2876,44 +2773,6 @@ "react-dom": ">=16.9.0" } }, - "node_modules/@react-aria/focus": { - "version": "3.22.0", - "resolved": "https://registry.npmjs.org/@react-aria/focus/-/focus-3.22.0.tgz", - "integrity": "sha512-ZfDOVuVhqDsM9mkNji3QUZ/d40JhlVgXrDkrfXylM1035QCrcTHN7m2DpbE95sU2A8EQb4wikvt5jM6K/73BPg==", - "license": "Apache-2.0", - "dependencies": { - "@swc/helpers": "^0.5.0", - "react-aria": "3.48.0" - }, - "peerDependencies": { - "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1", - "react-dom": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1" - } - }, - "node_modules/@react-aria/interactions": { - "version": "3.28.0", - "resolved": "https://registry.npmjs.org/@react-aria/interactions/-/interactions-3.28.0.tgz", - "integrity": "sha512-OXwdU1EWFdMxmr/K1CXNGJzmNlCClByb+PuCaqUyzBymHPCGVhawirLIon/CrIN5psh3AiWpHSh4H0WeJdVpng==", - "license": "Apache-2.0", - "dependencies": { - "@react-types/shared": "^3.34.0", - "@swc/helpers": "^0.5.0", - "react-aria": "3.48.0" - }, - "peerDependencies": { - "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1", - "react-dom": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1" - } - }, - "node_modules/@react-types/shared": { - "version": "3.34.0", - "resolved": "https://registry.npmjs.org/@react-types/shared/-/shared-3.34.0.tgz", - "integrity": "sha512-gp6xo/s2lX54AlTjOiqwDnxA7UW79BNvI9dB9pr3LZTzRKCd1ZA+ZbgKw/ReIiWuvvVw/8QFJpnqeeFyLocMcQ==", - "license": "Apache-2.0", - "peerDependencies": { - "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1" - } - }, "node_modules/@redocly/ajv": { "version": "8.11.2", "resolved": "https://registry.npmjs.org/@redocly/ajv/-/ajv-8.11.2.tgz", @@ -3839,23 +3698,6 @@ "react-dom": ">=16.8" } }, - "node_modules/@tanstack/react-virtual": { - "version": "3.13.24", - "resolved": "https://registry.npmjs.org/@tanstack/react-virtual/-/react-virtual-3.13.24.tgz", - "integrity": "sha512-aIJvz5OSkhNIhZIpYivrxrPTKYsjW9Uzy+sP/mx0S3sev2HyvPb7xmjbYvokzEpfgYHy/HjzJ2zFAETuUfgCpg==", - "license": "MIT", - "dependencies": { - "@tanstack/virtual-core": "3.14.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/tannerlinsley" - }, - "peerDependencies": { - "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", - "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" - } - }, "node_modules/@tanstack/store": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/@tanstack/store/-/store-0.11.0.tgz", @@ -3879,16 +3721,6 @@ "url": "https://github.com/sponsors/tannerlinsley" } }, - "node_modules/@tanstack/virtual-core": { - "version": "3.14.0", - "resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.14.0.tgz", - "integrity": "sha512-JLANqGy/D6k4Ujmh8Tr25lGimuOXNiaVyXaCAZS0W+1390sADdGnyUdSWNIfd49gebtIxGMij4IktRVzrdr12Q==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/tannerlinsley" - } - }, "node_modules/@testing-library/dom": { "version": "10.4.1", "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.1.tgz", @@ -3978,93 +3810,6 @@ "@testing-library/dom": ">=7.21.4" } }, - "node_modules/@tremor/react": { - "version": "3.18.7", - "resolved": "https://registry.npmjs.org/@tremor/react/-/react-3.18.7.tgz", - "integrity": "sha512-nmqvf/1m0GB4LXc7v2ftdfSLoZhy5WLrhV6HNf0SOriE6/l8WkYeWuhQq8QsBjRi94mUIKLJ/VC3/Y/pj6VubQ==", - "license": "Apache 2.0", - "dependencies": { - "@floating-ui/react": "^0.19.2", - "@headlessui/react": "2.2.0", - "date-fns": "^3.6.0", - "react-day-picker": "^8.10.1", - "react-transition-state": "^2.1.2", - "recharts": "^2.13.3", - "tailwind-merge": "^2.5.2" - }, - "peerDependencies": { - "react": "^18.0.0", - "react-dom": ">=16.6.0" - } - }, - "node_modules/@tremor/react/node_modules/eventemitter3": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", - "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", - "license": "MIT" - }, - "node_modules/@tremor/react/node_modules/react-is": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", - "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", - "license": "MIT" - }, - "node_modules/@tremor/react/node_modules/recharts": { - "version": "2.15.4", - "resolved": "https://registry.npmjs.org/recharts/-/recharts-2.15.4.tgz", - "integrity": "sha512-UT/q6fwS3c1dHbXv2uFgYJ9BMFHu3fwnd7AYZaEQhXuYQ4hgsxLvsUXzGdKeZrW5xopzDCvuA2N41WJ88I7zIw==", - "deprecated": "1.x and 2.x branches are no longer active. Bump to Recharts v3 to receive latest features and bugfixes. See https://github.com/recharts/recharts/wiki/3.0-migration-guide", - "license": "MIT", - "dependencies": { - "clsx": "^2.0.0", - "eventemitter3": "^4.0.1", - "lodash": "^4.17.21", - "react-is": "^18.3.1", - "react-smooth": "^4.0.4", - "recharts-scale": "^0.4.4", - "tiny-invariant": "^1.3.1", - "victory-vendor": "^36.6.8" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "react": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", - "react-dom": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" - } - }, - "node_modules/@tremor/react/node_modules/tailwind-merge": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-2.6.1.tgz", - "integrity": "sha512-Oo6tHdpZsGpkKG88HJ8RR1rg/RdnEkQEfMoEk2x1XRI3F1AxeU+ijRXpiVUF4UbLfcxxRGw6TbUINKYdWVsQTQ==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/dcastil" - } - }, - "node_modules/@tremor/react/node_modules/victory-vendor": { - "version": "36.9.2", - "resolved": "https://registry.npmjs.org/victory-vendor/-/victory-vendor-36.9.2.tgz", - "integrity": "sha512-PnpQQMuxlwYdocC8fIJqVXvkeViHYzotI+NJrCuav0ZYFoq912ZHBk3mCeuj+5/VpodOjPe1z0Fk2ihgzlXqjQ==", - "license": "MIT AND ISC", - "dependencies": { - "@types/d3-array": "^3.0.3", - "@types/d3-ease": "^3.0.0", - "@types/d3-interpolate": "^3.0.1", - "@types/d3-scale": "^4.0.2", - "@types/d3-shape": "^3.1.0", - "@types/d3-time": "^3.0.0", - "@types/d3-timer": "^3.0.0", - "d3-array": "^3.1.6", - "d3-ease": "^3.0.1", - "d3-interpolate": "^3.0.1", - "d3-scale": "^4.0.2", - "d3-shape": "^3.1.0", - "d3-time": "^3.0.0", - "d3-timer": "^3.0.1" - } - }, "node_modules/@tybys/wasm-util": { "version": "0.10.3", "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz", @@ -5203,18 +4948,6 @@ "dev": true, "license": "Python-2.0" }, - "node_modules/aria-hidden": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/aria-hidden/-/aria-hidden-1.2.6.tgz", - "integrity": "sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA==", - "license": "MIT", - "dependencies": { - "tslib": "^2.0.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/aria-query": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz", @@ -6314,16 +6047,6 @@ "dev": true, "license": "MIT" }, - "node_modules/dom-helpers": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/dom-helpers/-/dom-helpers-5.2.1.tgz", - "integrity": "sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA==", - "license": "MIT", - "dependencies": { - "@babel/runtime": "^7.8.7", - "csstype": "^3.0.2" - } - }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -7201,15 +6924,6 @@ "dev": true, "license": "MIT" }, - "node_modules/fast-equals": { - "version": "5.4.1", - "resolved": "https://registry.npmjs.org/fast-equals/-/fast-equals-5.4.1.tgz", - "integrity": "sha512-DjlFSM5Pk9cGcL0q5QXl66eGzx0N6szNgaswwc5ZphlBohjTVJSnGgI+rJVOgOi65qUoQnDZN4nDqi33udtydQ==", - "license": "MIT", - "engines": { - "node": ">=6.0.0" - } - }, "node_modules/fast-glob": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.1.tgz", @@ -9229,12 +8943,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/lodash": { - "version": "4.18.1", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", - "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", - "license": "MIT" - }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -11868,27 +11576,6 @@ "node": ">=0.10.0" } }, - "node_modules/react-aria": { - "version": "3.48.0", - "resolved": "https://registry.npmjs.org/react-aria/-/react-aria-3.48.0.tgz", - "integrity": "sha512-jQjd4rBEIMqecBaAKYJbVGK6EqIHLa5znVQ7jwFyK5vCyljoj6KhgtiahmcIPsG5vG5vEDLw+ba+bEWn6A2P4w==", - "license": "Apache-2.0", - "dependencies": { - "@internationalized/date": "^3.12.1", - "@internationalized/number": "^3.6.6", - "@internationalized/string": "^3.2.8", - "@react-types/shared": "^3.34.0", - "@swc/helpers": "^0.5.0", - "aria-hidden": "^1.2.3", - "clsx": "^2.0.0", - "react-stately": "3.46.0", - "use-sync-external-store": "^1.6.0" - }, - "peerDependencies": { - "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1", - "react-dom": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1" - } - }, "node_modules/react-copy-to-clipboard": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/react-copy-to-clipboard/-/react-copy-to-clipboard-5.1.1.tgz", @@ -11902,20 +11589,6 @@ "react": ">=15.3.0" } }, - "node_modules/react-day-picker": { - "version": "8.10.2", - "resolved": "https://registry.npmjs.org/react-day-picker/-/react-day-picker-8.10.2.tgz", - "integrity": "sha512-LK68OTbHB3oJNhl9cA0qVizzp3o26w61YSjAFkYi67N86iro32wx86kSNeFU/hq+gI8m1yzWhnomMLfZ041RzQ==", - "license": "MIT", - "funding": { - "type": "individual", - "url": "https://github.com/sponsors/gpbl" - }, - "peerDependencies": { - "date-fns": "^2.28.0 || ^3.0.0", - "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" - } - }, "node_modules/react-dom": { "version": "18.3.1", "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", @@ -12013,38 +11686,6 @@ } } }, - "node_modules/react-smooth": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/react-smooth/-/react-smooth-4.0.4.tgz", - "integrity": "sha512-gnGKTpYwqL0Iii09gHobNolvX4Kiq4PKx6eWBCYYix+8cdw+cGo3do906l1NBPKkSWx1DghC1dlWG9L2uGd61Q==", - "license": "MIT", - "dependencies": { - "fast-equals": "^5.0.1", - "prop-types": "^15.8.1", - "react-transition-group": "^4.4.5" - }, - "peerDependencies": { - "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", - "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" - } - }, - "node_modules/react-stately": { - "version": "3.46.0", - "resolved": "https://registry.npmjs.org/react-stately/-/react-stately-3.46.0.tgz", - "integrity": "sha512-OdxhWvHgs2L4OJGIs7hnuTr5WjjMM6enhNEAMRqiekhF8+ITvA2LRwNftOZwcogaoCslGYq5S2VQTQwnm0GbCA==", - "license": "Apache-2.0", - "dependencies": { - "@internationalized/date": "^3.12.1", - "@internationalized/number": "^3.6.6", - "@internationalized/string": "^3.2.8", - "@react-types/shared": "^3.34.0", - "@swc/helpers": "^0.5.0", - "use-sync-external-store": "^1.6.0" - }, - "peerDependencies": { - "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1" - } - }, "node_modules/react-syntax-highlighter": { "version": "15.6.6", "resolved": "https://registry.npmjs.org/react-syntax-highlighter/-/react-syntax-highlighter-15.6.6.tgz", @@ -12062,32 +11703,6 @@ "react": ">= 0.14.0" } }, - "node_modules/react-transition-group": { - "version": "4.4.5", - "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.5.tgz", - "integrity": "sha512-pZcd1MCJoiKiBR2NRxeCRg13uCXbydPnmB4EOeRrY7480qNWO8IIgQG6zlDkm6uRMsURXPuKq0GWtiM59a5Q6g==", - "license": "BSD-3-Clause", - "dependencies": { - "@babel/runtime": "^7.5.5", - "dom-helpers": "^5.0.1", - "loose-envify": "^1.4.0", - "prop-types": "^15.6.2" - }, - "peerDependencies": { - "react": ">=16.6.0", - "react-dom": ">=16.6.0" - } - }, - "node_modules/react-transition-state": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/react-transition-state/-/react-transition-state-2.3.3.tgz", - "integrity": "sha512-wsIyg07ohlWEAYDZHvuXh/DY7mxlcLb0iqVv2aMXJ0gwgPVKNWKhOyNyzuJy/tt/6urSq0WT6BBZ/tdpybaAsQ==", - "license": "MIT", - "peerDependencies": { - "react": ">=16.8.0", - "react-dom": ">=16.8.0" - } - }, "node_modules/recharts": { "version": "3.9.2", "resolved": "https://registry.npmjs.org/recharts/-/recharts-3.9.2.tgz", @@ -12118,15 +11733,6 @@ "react-is": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, - "node_modules/recharts-scale": { - "version": "0.4.5", - "resolved": "https://registry.npmjs.org/recharts-scale/-/recharts-scale-0.4.5.tgz", - "integrity": "sha512-kivNFO+0OcUNu7jQquLXAxz1FIwZj8nrj+YkOKc5694NbjCvcT6aSZiIzNzd2Kul4o4rTto8QVR9lMNtxD4G1w==", - "license": "MIT", - "dependencies": { - "decimal.js-light": "^2.4.1" - } - }, "node_modules/redent": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/redent/-/redent-3.0.0.tgz", @@ -13200,12 +12806,6 @@ "dev": true, "license": "MIT" }, - "node_modules/tabbable": { - "version": "6.4.0", - "resolved": "https://registry.npmjs.org/tabbable/-/tabbable-6.4.0.tgz", - "integrity": "sha512-05PUHKSNE8ou2dwIxTngl4EzcnsCDZGJ/iCLtDflR/SHB/ny14rXc+qU5P4mG9JkusiV7EivzY9Mhm55AzAvCg==", - "license": "MIT" - }, "node_modules/tailwind-merge": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-3.4.0.tgz", diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 483ab39c336..9407ff25a1f 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -31,7 +31,6 @@ "@tanstack/react-pacer": "0.22.1", "@tanstack/react-query": "5.100.7", "@tanstack/react-table": "8.21.3", - "@tremor/react": "3.18.7", "@types/papaparse": "5.5.2", "antd": "5.29.3", "cva": "1.0.0-beta.4", @@ -103,7 +102,6 @@ "axios": "1.13.6", "postcss": "8.5.23", "esbuild": "0.28.1", - "date-fns": "^4.4.0", "sharp": "^0.35.0" }, "engines": { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/TagsInput.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/TagsInput.tsx index 0d799474399..6fb1508497e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/TagsInput.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/TagsInput.tsx @@ -28,6 +28,7 @@ interface TagsInputProps { emptyText?: string; tokenSeparators?: string[]; loading?: boolean; + disabled?: boolean; id?: string; } @@ -48,6 +49,7 @@ export const TagsInput = ({ emptyText = "No matching options", tokenSeparators = [], loading = false, + disabled = false, id, }: TagsInputProps) => { const anchor = useComboboxAnchor(); @@ -103,6 +105,7 @@ export const TagsInput = ({ itemToStringLabel={(option: TagsInputOption) => option.label} filter={matchesQuery} openOnInputClick + disabled={disabled || loading} > } className="min-h-8 py-1 text-sm"> diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx index 320e0b5baef..b6ee130d50a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx @@ -1,5 +1,6 @@ import * as networking from "@/components/networking"; import { fireEvent, render, waitFor, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; import { afterEach, describe, expect, it, vi } from "vitest"; import GuardrailInfoView from "./guardrail_info"; @@ -136,7 +137,7 @@ describe("Guardrail Info", () => { vi.mocked(networking.getGuardrailProviderSpecificParams).mockResolvedValue({}); - const { getByText, container } = render( + const { getByText, findByText, container } = render( {}} accessToken="123" isAdmin={true} />, ); @@ -152,18 +153,9 @@ describe("Guardrail Info", () => { expect(getByText("Guardrail Settings")).toBeInTheDocument(); }); - // Find the info icon and hover over it - const infoIcon = within(container).getByRole("img", { name: "info-circle" }); - expect(infoIcon).toBeInTheDocument(); + await userEvent.hover(within(container).getByRole("img", { name: "info-circle" })); - if (infoIcon) { - fireEvent.mouseEnter(infoIcon); - - // Wait for the tooltip to appear - await waitFor(() => { - expect(getByText("Guardrail is defined in the config file and cannot be edited.")).toBeInTheDocument(); - }); - } + expect(await findByText("Guardrail is defined in the config file and cannot be edited.")).toBeInTheDocument(); }); it("should render the guardrail info", async () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx index caac09a1c83..e80ddac932f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx @@ -6,7 +6,8 @@ import { } from "@/components/networking"; import { copyToClipboard as utilCopyToClipboard } from "@/utils/dataUtils"; import { CodeOutlined, EyeInvisibleOutlined, InfoCircleOutlined, StopOutlined } from "@ant-design/icons"; -import { Button as AntdButton, Tooltip } from "antd"; + +import { Button as AntdButton } from "antd"; import { ArrowLeft, CheckIcon, CopyIcon } from "lucide-react"; import { Badge } from "@/components/ui/badge"; import { Card } from "@/components/ui/card"; @@ -21,7 +22,7 @@ import { Input } from "@/components/ui/input"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Separator } from "@/components/ui/separator"; import { Textarea } from "@/components/ui/textarea"; -import { TooltipProvider } from "@/components/ui/tooltip"; +import { SimpleTooltip, TooltipProvider } from "@/components/ui/tooltip"; import { asText, GuardrailField, @@ -663,9 +664,9 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose,

Guardrail Settings

{isConfigGuardrail && ( - + - + )} {!isEditing && !isConfigGuardrail && diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/AwsSigV4Fields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/AwsSigV4Fields.tsx index d4ae537bffa..09016bda266 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/AwsSigV4Fields.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/AwsSigV4Fields.tsx @@ -1,7 +1,25 @@ import React from "react"; -import { Form, Input, Tooltip } from "antd"; +import { Input, Tooltip } from "antd"; import { InfoCircleOutlined } from "@ant-design/icons"; +import { MountedFormField } from "@/components/common_components/MountedFormField"; +import { antdRequired } from "@/components/common_components/antdFormRules"; +import { requiredWhenSiblingSet, textControl } from "./mcpFieldRules"; + +const fieldClassName = "rounded-lg border-gray-300 focus:border-blue-500 focus:ring-blue-500"; + +const FieldLabel: React.FC<{ label: string; tooltip: string }> = ({ label, tooltip }) => ( + + {label} + + + + +); + +const ACCESS_KEY_PATH = ["credentials", "aws_access_key_id"] as const; +const SECRET_KEY_PATH = ["credentials", "aws_secret_access_key"] as const; + const AwsSigV4Fields: React.FC = () => ( <>

@@ -15,140 +33,120 @@ const AwsSigV4Fields: React.FC = () => ( View docs →

- - AWS Region - - - - - } + } name={["credentials", "aws_region_name"]} - rules={[{ required: true, message: "AWS region is required for SigV4 auth" }]} + required + rules={{ validate: { required: antdRequired("AWS region is required for SigV4 auth") } }} > - - - } + + - AWS Service Name - - - - + } name={["credentials", "aws_service_name"]} > - - - } + + - AWS Access Key ID - - - - + } - name={["credentials", "aws_access_key_id"]} - dependencies={[["credentials", "aws_secret_access_key"]]} - rules={[ - ({ getFieldValue }) => ({ - validator(_, value) { - const secretKey = getFieldValue(["credentials", "aws_secret_access_key"]); - if (secretKey && !value) { - return Promise.reject(new Error("Access Key ID is required when Secret Access Key is provided")); - } - return Promise.resolve(); - }, - }), - ]} + name={ACCESS_KEY_PATH} + rules={{ + deps: ["credentials.aws_secret_access_key"], + validate: { + pairedWithSecret: requiredWhenSiblingSet( + SECRET_KEY_PATH, + "Access Key ID is required when Secret Access Key is provided", + ), + }, + }} > - - - ( + + )} + + - AWS Secret Access Key - - - - + } - name={["credentials", "aws_secret_access_key"]} - dependencies={[["credentials", "aws_access_key_id"]]} - rules={[ - ({ getFieldValue }) => ({ - validator(_, value) { - const accessKeyId = getFieldValue(["credentials", "aws_access_key_id"]); - if (accessKeyId && !value) { - return Promise.reject(new Error("Secret Access Key is required when Access Key ID is provided")); - } - return Promise.resolve(); - }, - }), - ]} + name={SECRET_KEY_PATH} + rules={{ + deps: ["credentials.aws_access_key_id"], + validate: { + pairedWithAccessKey: requiredWhenSiblingSet( + ACCESS_KEY_PATH, + "Secret Access Key is required when Access Key ID is provided", + ), + }, + }} > - - - - AWS Session Token - - - - - } + {(control) => ( + + )} + + } name={["credentials", "aws_session_token"]} > - - - ( + + )} + + - AWS Role ARN - - - - + } name={["credentials", "aws_role_name"]} > - - - ( + + )} + + - AWS Session Name - - - - + } name={["credentials", "aws_session_name"]} > - - + {(control) => ( + + )} + ); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.integration.test.tsx index f08aacb5522..78d89b7e19e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.integration.test.tsx @@ -2147,7 +2147,7 @@ describe("CreateMCPServer dcr_bridge toggle", () => { }); // Forcing dcr_bridge false for every non-client-forwarded auth type is covered in - // createServerPayload.test.ts. The two form-state cases below stay: they prove the Form.Item + // createServerPayload.test.ts. The two form-state cases below stay: they prove the field // unmounts on a switch away, and that the live value survives a client-forwarded swap. it("forces dcr_bridge: false when the auth type is switched away after toggling", async () => { @@ -2179,7 +2179,7 @@ describe("CreateMCPServer dcr_bridge toggle", () => { }); expect(getDcrToggle()).toHaveAttribute("aria-checked", "true"); - // The Form.Item is mounted in both client-forwarded modes, so switching between them keeps the + // The field is mounted in both client-forwarded modes, so switching between them keeps the // live toggle value rather than forcing it back to the default or to false. await selectAntOption("Authentication", "OAuth Delegate (client-supplied upstream token)"); await waitFor(() => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.permissions.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.permissions.integration.test.tsx new file mode 100644 index 00000000000..788be106d8a --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.permissions.integration.test.tsx @@ -0,0 +1,160 @@ +import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import * as networking from "@/components/networking"; +import CreateMCPServer from "./CreateMCPServer"; +import { selectAntOption } from "./testUtils"; + +vi.mock("@/components/networking", () => ({ + createMCPServer: vi.fn(), + fetchOpenAPIRegistry: vi.fn().mockResolvedValue({ apis: [] }), + registerMCPServer: vi.fn(), + storeMCPOAuthUserCredential: vi.fn().mockResolvedValue({}), + testMCPToolsListRequest: vi.fn().mockResolvedValue({ tools: [], error: null }), +})); + +vi.mock("@/utils/mcpTokenStore", () => ({ + setToken: vi.fn(), +})); + +vi.mock("./OpenAPIQuickPicker", () => ({ + default: () => null, +})); + +vi.mock("@/hooks/useMcpOAuthFlow", () => ({ + useMcpOAuthFlow: () => ({ + startOAuthFlow: vi.fn(), + status: "idle", + error: null, + tokenResponse: null, + reset: vi.fn(), + }), +})); + +vi.mock("./mcp_server_cost_config", () => ({ + default: () =>
, +})); + +vi.mock("./mcp_tool_configuration", () => ({ + default: () =>
, +})); + +vi.mock("./mcp_connection_status", () => ({ + default: () =>
, +})); + +vi.mock("./StdioConfiguration", () => ({ + default: () =>
, +})); + +const defaultProps = { + userRole: "Admin", + accessToken: "test-token", + onCreateSuccess: vi.fn(), + isModalVisible: true, + setModalVisible: vi.fn(), + availableAccessGroups: ["group-a", "group-b"], +}; + +const getServerNameInput = () => document.getElementById("server_name") as HTMLInputElement; + +const switchFor = (labelText: string): HTMLElement => { + const label = screen.getByText(labelText); + const row = label.closest(".flex.items-start.justify-between"); + const control = row?.querySelector("button[role='switch']"); + if (control === null || control === undefined) { + throw new Error(`no switch found for "${labelText}"`); + } + return control as HTMLElement; +}; + +const fillMinimalHttpServer = async (name: string) => { + await selectAntOption("Transport Type", "Streamable HTTP"); + await waitFor(() => { + expect(screen.getByPlaceholderText("https://your-mcp-server.com")).toBeInTheDocument(); + }); + const user = userEvent.setup({ delay: null }); + await user.type(getServerNameInput(), name); + await user.type(screen.getByPlaceholderText("https://your-mcp-server.com"), "https://example.com/mcp"); + await selectAntOption("Authentication", "None"); +}; + +const submitAndReadPayload = async () => { + await act(async () => { + fireEvent.click(screen.getByRole("button", { name: "Add MCP Server" })); + }); + await waitFor(() => expect(networking.createMCPServer).toHaveBeenCalledTimes(1)); + return vi.mocked(networking.createMCPServer).mock.calls[0][1]; +}; + +const createdServer = { + server_id: "new-server-1", + server_name: "Perm_Server", + alias: "Perm_Server", + url: "https://example.com/mcp", + transport: "http", + auth_type: "none", + created_at: "2024-01-01T00:00:00Z", + created_by: "user-1", + updated_at: "2024-01-01T00:00:00Z", + updated_by: "user-1", +}; + +describe("CreateMCPServer permission toggles reaching the payload", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(networking.createMCPServer).mockResolvedValue(createdServer); + }); + + it("sends the panel's untouched defaults rather than dropping the keys the panel owns", async () => { + render(); + await fillMinimalHttpServer("Perm_Server"); + + const payload = await submitAndReadPayload(); + + expect(payload.allow_all_keys).toBe(false); + expect(payload.available_on_public_internet).toBe(true); + }); + + it("sends allow_all_keys true once the operator turns the public-to-all-keys switch on", async () => { + render(); + await fillMinimalHttpServer("Perm_Server"); + + await act(async () => { + fireEvent.click(switchFor("Allow All LiteLLM Keys")); + }); + + const payload = await submitAndReadPayload(); + + expect(payload.allow_all_keys).toBe(true); + }); + + it("sends available_on_public_internet false when the operator restricts the server to the internal network", async () => { + render(); + await fillMinimalHttpServer("Perm_Server"); + + const internalOnly = switchFor("Internal network only"); + expect(internalOnly).toHaveAttribute("aria-checked", "false"); + + await act(async () => { + fireEvent.click(internalOnly); + }); + expect(internalOnly).toHaveAttribute("aria-checked", "true"); + + const payload = await submitAndReadPayload(); + + expect(payload.available_on_public_internet).toBe(false); + }); + + it("omits delegate_auth_to_upstream's true value on a none-auth server, whose gate never mounts that switch", async () => { + render(); + await fillMinimalHttpServer("Perm_Server"); + + expect(screen.getByText("Allow All LiteLLM Keys")).toBeInTheDocument(); + expect(screen.queryByText("Delegate auth to upstream (PKCE passthrough)")).not.toBeInTheDocument(); + + const payload = await submitAndReadPayload(); + + expect(payload.delegate_auth_to_upstream).toBe(false); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx index ed8d3bddc98..92bdd14754c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx @@ -1,5 +1,6 @@ import React, { useState } from "react"; -import { Modal, Tooltip, Form, Select, Input as AntdInput, InputNumber, Collapse } from "antd"; +import { Modal, Tooltip, Select, Input as AntdInput, InputNumber, Collapse } from "antd"; +import { FormProvider, useForm, useWatch } from "react-hook-form"; import { InfoCircleOutlined } from "@ant-design/icons"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; @@ -49,6 +50,16 @@ import { validateMCPServerUrl, validateMCPServerName } from "./utils"; import { toast } from "@/lib/toast"; import { useMcpOAuthFlow } from "@/hooks/useMcpOAuthFlow"; import { useTestMCPConnection } from "@/hooks/useTestMCPConnection"; +import { + MountedFormField, + MountedFormProvider, + projectMountedValues, + useMountRegistry, + type MountedFormValues, +} from "@/components/common_components/MountedFormField"; +import { antdRequired, antdRules } from "@/components/common_components/antdFormRules"; +import { allFieldsValue, mountedPaths, resetFields, setFieldsValue, singleBranchChange } from "./mcpFormStore"; +import { numberControl, notOnlyWhitespace, selectControl, textControl } from "./mcpFieldRules"; import mcpLogo from "../../../../../public/assets/logos/mcp_logo.png"; export const mcpLogoImg = mcpLogo.src; @@ -76,6 +87,13 @@ const payloadErrorMessage = (result: Exclude = ({ userID, userRole, @@ -87,7 +105,8 @@ const CreateMCPServer: React.FC = ({ prefillData, onBackToDiscovery, }) => { - const [form] = Form.useForm(); + const form = useForm({ mode: "onChange", defaultValues: CREATE_DEFAULTS }); + const registry = useMountRegistry(); const [isLoading, setIsLoading] = useState(false); const [costConfig, setCostConfig] = useState({}); const [formValues, setFormValues] = useState>({}); @@ -136,6 +155,8 @@ const CreateMCPServer: React.FC = ({ enabled: true, }); + const authSectionMounted = transportType !== "stdio" && transportType !== ""; + const watchedAuthType = useWatch({ control: form.control, name: "auth_type" }) as string | undefined; const authType = formValues.auth_type as string | undefined; const shouldShowAuthValueField = authType ? AUTH_TYPES_REQUIRING_AUTH_VALUE.includes(authType) : false; const isOAuthAuthType = authType === AUTH_TYPE.OAUTH2; @@ -147,7 +168,7 @@ const CreateMCPServer: React.FC = ({ const persistCreateUiState = () => { writeCreateUiSnapshot({ modalVisible: isModalVisible, - formValues: form.getFieldsValue(true), + formValues: allFieldsValue(form), transportType, costConfig, allowedTools, @@ -170,11 +191,11 @@ const CreateMCPServer: React.FC = ({ // Merge the ref-held DCR client so a re-authorize reuses the registered client instead of // re-registering; the form store itself never holds the DCR client (see onTokenReceived). getCredentials: () => ({ - ...((form.getFieldValue("credentials") as Record | undefined) ?? {}), + ...((allFieldsValue(form).credentials as Record | undefined) ?? {}), ...(dcrClientRef.current ?? {}), }), getTemporaryPayload: () => { - const values = form.getFieldsValue(true); + const values = allFieldsValue(form); const transport = values.transport || transportType; // For OpenAPI transport the form has spec_path instead of url. // We pass the spec_path as url so the temp-session endpoint has something @@ -218,12 +239,12 @@ const CreateMCPServer: React.FC = ({ return; } - if (isClientForwardedTokenMode(form.getFieldValue("auth_type"))) { + if (isClientForwardedTokenMode(allFieldsValue(form).auth_type)) { // Browser-only modes: the token is held in local state (oauthAccessToken) for tool preview // and committed to sessionStorage on submit; it must never be written into form.credentials, // which would persist it as server-level credentials on the created server row. Mirrors the // edit form's onTokenReceived early return. - setAuthorizedIdentity(getOAuthAuthorizationIdentity(form.getFieldsValue(true))); + setAuthorizedIdentity(getOAuthAuthorizationIdentity(allFieldsValue(form))); toast.success( "Token held for this browser session. Tools can now be previewed and configured; the token is not saved to LiteLLM.", ); @@ -240,7 +261,7 @@ const CreateMCPServer: React.FC = ({ } : null; - const current = (form.getFieldValue("credentials") as Record | undefined) ?? {}; + const current = (allFieldsValue(form).credentials as Record | undefined) ?? {}; const nextCredentials = { ...(preservedAdminCredentials(current) ?? {}), ...(current.scopes !== undefined && { scopes: current.scopes }), @@ -252,10 +273,10 @@ const CreateMCPServer: React.FC = ({ // Path-replace (not deep-merge) so a re-authorize with fewer token fields does not leave stale // siblings from the previous token behind; the admin-typed client keys and scopes are carried // explicitly above. - form.setFieldValue("credentials", nextCredentials); + form.setValue("credentials", nextCredentials); // Capture the identity AFTER writing the token so the held token is not spuriously invalidated by // its own credential write. - setAuthorizedIdentity(getOAuthAuthorizationIdentity(form.getFieldsValue(true))); + setAuthorizedIdentity(getOAuthAuthorizationIdentity(allFieldsValue(form))); toast.success("OAuth authorization successful! Please click 'Create MCP Server' to save the configuration."); }, @@ -277,10 +298,10 @@ const CreateMCPServer: React.FC = ({ // Capture the admin-typed app before resetFields destroys it, then re-apply it: the app is // upstream-scoped config, not minted material, so it survives every invalidation (the token is // what gets discarded). Token-shaped keys are excluded by the helper's key filter. - const keptAdminCredentials = preservedAdminCredentials(form.getFieldValue("credentials")); - form.resetFields([...CLEARED_ON_INVALIDATION]); + const keptAdminCredentials = preservedAdminCredentials(allFieldsValue(form).credentials); + resetFields(form, [...CLEARED_ON_INVALIDATION]); if (keptAdminCredentials) { - form.setFieldsValue({ credentials: keptAdminCredentials }); + setFieldsValue(form, { credentials: keptAdminCredentials }); } // Re-apply the in-flight edit last; rc-field-form deep-merges nested objects, so a changed // credentials sub-field composes with the preserved sibling instead of replacing the object. @@ -288,7 +309,7 @@ const CreateMCPServer: React.FC = ({ CLEARED_ON_INVALIDATION.filter((key) => key in changedValues).map((key) => [key, changedValues[key]]), ); if (Object.keys(preserved).length > 0) { - form.setFieldsValue(preserved); + setFieldsValue(form, preserved); } }; @@ -337,7 +358,7 @@ const CreateMCPServer: React.FC = ({ // wait until transportType state catches up so the URL field is mounted return; } - form.setFieldsValue(pendingRestoredValues.values); + setFieldsValue(form, pendingRestoredValues.values); setFormValues(pendingRestoredValues.values); setPendingRestoredValues(null); }, [pendingRestoredValues, form, transportType]); @@ -381,11 +402,20 @@ const CreateMCPServer: React.FC = ({ prefillValues.url = prefillData.url; } - form.setFieldsValue(prefillValues); + setFieldsValue(form, prefillValues); setFormValues(prefillValues); setAliasManuallyEdited(false); }, [isModalVisible, prefillData, form]); + const handleSubmit = async (event: React.FormEvent) => { + event.preventDefault(); + const isValid = await form.trigger(mountedPaths(registry) as string[]); + if (!isValid) { + return; + } + await handleCreate(projectMountedValues(registry, form.getValues)); + }; + const handleCreate = async (values: Record) => { const built = buildCreateServerPayload(values, { transportType, @@ -446,7 +476,7 @@ const CreateMCPServer: React.FC = ({ description: "Once an admin approves it, the server will appear in your MCP Servers list.", }); } - form.resetFields(); + form.reset(CREATE_DEFAULTS); setCostConfig({}); clearTools(); setAllowedTools([]); @@ -466,7 +496,7 @@ const CreateMCPServer: React.FC = ({ // state const handleCancel = () => { - form.resetFields(); + form.reset(CREATE_DEFAULTS); setCostConfig({}); clearTools(); setAllowedTools([]); @@ -489,11 +519,11 @@ const CreateMCPServer: React.FC = ({ ? { url: undefined, command: undefined, args: undefined, env: undefined } : { spec_path: undefined, command: undefined, args: undefined, env: undefined }; - form.setFieldsValue(transportValues); - if (isHeldOAuthTokenStale(form.getFieldsValue(true), authorizedIdentity)) { + setFieldsValue(form, transportValues); + if (isHeldOAuthTokenStale(allFieldsValue(form), authorizedIdentity)) { clearHeldOAuthToken(); } - setFormValues(form.getFieldsValue(true)); + setFormValues(allFieldsValue(form)); }; // Generate options with existing groups and potential new group @@ -532,7 +562,7 @@ const CreateMCPServer: React.FC = ({ React.useEffect(() => { if (!aliasManuallyEdited && formValues.server_name) { const normalized = formValues.server_name.replace(/\s+/g, "_"); - form.setFieldsValue({ alias: normalized }); + setFieldsValue(form, { alias: normalized }); setFormValues((prev) => ({ ...prev, alias: normalized })); } }, [formValues.server_name]); @@ -549,7 +579,7 @@ const CreateMCPServer: React.FC = ({ const wasVisible = wasModalVisibleRef.current; wasModalVisibleRef.current = isModalVisible; if (!isModalVisible && wasVisible) { - form.resetFields(); + form.reset(CREATE_DEFAULTS); setFormValues({}); setOauthAccessToken(null); clearTools(); @@ -582,19 +612,35 @@ const CreateMCPServer: React.FC = ({ const upstreamChanged = ["url", "spec_path", "issuer", "authorization_url", "token_url", "registration_url"].some( (key) => key in changedValues, ); - const hasDeclaredApp = preservedDeclaredAppCredentials(form.getFieldValue("credentials")) !== undefined; + const hasDeclaredApp = preservedDeclaredAppCredentials(allFieldsValue(form).credentials) !== undefined; if (upstreamChanged && hasDeclaredApp) { setAppMayNotMatchUpstream(true); } } - if (isHeldOAuthTokenStale(form.getFieldsValue(true), authorizedIdentity)) { + if (isHeldOAuthTokenStale(allFieldsValue(form), authorizedIdentity)) { clearHeldOAuthToken(changedValues); - setFormValues(form.getFieldsValue(true)); + setFormValues(allFieldsValue(form)); return; } setFormValues(allValues); }; + const valuesChangeRef = React.useRef(handleFormValuesChange); + valuesChangeRef.current = handleFormValuesChange; + + React.useEffect(() => { + const subscription = form.watch((values, { name, type }) => { + if (type !== "change" || name === undefined) { + return; + } + valuesChangeRef.current( + singleBranchChange(name, values as MountedFormValues), + projectMountedValues(registry, form.getValues), + ); + }); + return () => subscription.unsubscribe(); + }, [form, registry]); + // rendering return ( = ({ }} >
-
- {!isAdmin && ( -
- Your submission will be sent for admin review. Once approved, the server will appear in your MCP Servers - list. The request must be made with a team-scoped API key. -
- )} -
- - MCP Server Name - - - - - } - name="server_name" - rules={[ - { required: false, message: "Please enter a server name" }, - { validator: (_, value) => validateMCPServerName(value) }, - ]} - > - - + + + + {!isAdmin && ( +
+ Your submission will be sent for admin review. Once approved, the server will appear in your MCP + Servers list. The request must be made with a team-scoped API key. +
+ )} +
+ + MCP Server Name + + + + + } + name="server_name" + rules={{ validate: antdRules({ validator: (_, value) => validateMCPServerName(value) }) }} + > + {(control) => ( + + )} + - - Alias - - - - - } - name="alias" - rules={[{ required: false }, { validator: (_, value) => validateMCPServerName(value) }]} - > - setAliasManuallyEdited(true)} - /> - + + Alias + + + + + } + name="alias" + rules={{ validate: antdRules({ validator: (_, value) => validateMCPServerName(value) }) }} + > + {(control) => ( + { + control.onChange(event); + setAliasManuallyEdited(true); + }} + /> + )} + - Description} - name="description" - rules={[ - { - required: false, - message: "Please enter a server description", - }, - ]} - > - - + Description} + name="description" + > + {(control) => ( + + )} + - + - GitHub / Source URL} - name="source_url" - > - - + GitHub / Source URL} + name="source_url" + > + {(control) => ( + + )} + - Transport Type} - name="transport" - rules={[{ required: true, message: "Please select a transport type" }]} - > - - + Transport Type} + name="transport" + required + rules={{ validate: { required: antdRequired("Please select a transport type") } }} + > + {(control) => ( + + )} + - {/* URL field - only show for HTTP and SSE */} - {(transportType === "http" || transportType === "sse") && ( - MCP Server URL} - name="url" - rules={[ - { required: true, message: "Please enter a server URL" }, - { validator: (_, value) => validateMCPServerUrl(value) }, - ]} - > - - - )} + {/* URL field - only show for HTTP and SSE */} + {(transportType === "http" || transportType === "sse") && ( + MCP Server URL} + name="url" + required + rules={{ + validate: { + required: antdRequired("Please enter a server URL"), + ...antdRules({ validator: (_, value) => validateMCPServerUrl(value) }), + }, + }} + > + {(control) => ( + + )} + + )} - {/* OpenAPI: logo picker + spec URL input */} - {transportType === TRANSPORT.OPENAPI && ( - - handleFormValuesChange(updates, { ...form.getFieldsValue(true), ...updates }) - } - onKeyToolsChange={setKeyTools} - onLogoUrlChange={setLogoUrl} - onOAuthDocsUrlChange={setOauthDocsUrl} - /> - )} + {/* OpenAPI: logo picker + spec URL input */} + {transportType === TRANSPORT.OPENAPI && ( + + handleFormValuesChange(updates, { ...allFieldsValue(form), ...updates }) + } + onKeyToolsChange={setKeyTools} + onLogoUrlChange={setLogoUrl} + onOAuthDocsUrlChange={setOauthDocsUrl} + /> + )} - {/* BYOK toggle - only for OpenAPI */} - {transportType === TRANSPORT.OPENAPI && } + {/* BYOK toggle - only for OpenAPI */} + {transportType === TRANSPORT.OPENAPI && } - - Max Concurrent Requests (optional) - - - - - } - name="max_concurrent_requests" - > - - + + Max Concurrent Requests (optional) + + + + + } + name="max_concurrent_requests" + > + {(control) => ( + + )} + - {/* Authentication - show for HTTP, SSE, and OpenAPI */} - {transportType !== "stdio" && transportType !== "" && ( - Authentication, - children: ( - <> - - - + {/* Authentication - show for HTTP, SSE, and OpenAPI */} + {transportType !== "stdio" && transportType !== "" && ( + Authentication, + children: ( + <> + + {(control) => ( + + )} + - + - - - {shouldShowAuthValueField && ( - - Authentication Value - - - - - } - name={["credentials", "auth_value"]} - rules={[ - { - validator: (_, value) => - value && typeof value === "string" && value.trim() === "" - ? Promise.reject(new Error("Authentication value cannot be empty whitespace")) - : Promise.resolve(), - }, - ]} - > - - - )} - {isOAuthAuthType && ( - - )} + {shouldShowAuthValueField && ( + + Authentication Value + + + + + } + name={["credentials", "auth_value"]} + rules={{ + validate: { + notWhitespace: notOnlyWhitespace("Authentication value cannot be empty whitespace"), + }, + }} + > + {(control) => ( + + )} + + )} - {isTokenExchangeAuthType && } + {isOAuthAuthType && ( + + )} - {isIdJagAuthType && } - - ), - }, - ]} - /> - )} + {isTokenExchangeAuthType && } - {transportType !== "stdio" && transportType !== "" && isAwsSigV4AuthType && } + {isIdJagAuthType && } + + ), + }, + ]} + /> + )} - {/* Stdio Configuration - only show for stdio transport */} - -
+ {transportType !== "stdio" && transportType !== "" && isAwsSigV4AuthType && } - {/* Environment Variables Section */} -
- -
+ {/* Stdio Configuration - only show for stdio transport */} + +
- {/* Permission Management / Access Control Section */} -
- -
+ {/* Environment Variables Section */} +
+ +
- {/* Connection Status Section */} -
- -
+ {/* Permission Management / Access Control Section */} +
+ +
- {/* Tool Configuration Section */} -
- setHasToolAllowlistInteraction(true)} - toolNameToDisplayName={toolNameToDisplayName} - toolNameToDescription={toolNameToDescription} - onToolNameToDisplayNameChange={setToolNameToDisplayName} - onToolNameToDescriptionChange={setToolNameToDescription} - keyTools={keyTools} - externalTools={tools} - externalIsLoading={isLoadingTools} - externalError={toolsError} - externalErrorStatus={toolsErrorStatus} - externalCanFetch={canFetchTools} - /> -
+ {/* Connection Status Section */} +
+ +
- {/* Cost Configuration Section */} -
- allowedTools.includes(tool.name))} - disabled={false} - /> -
+ {/* Tool Configuration Section */} +
+ setHasToolAllowlistInteraction(true)} + toolNameToDisplayName={toolNameToDisplayName} + toolNameToDescription={toolNameToDescription} + onToolNameToDisplayNameChange={setToolNameToDisplayName} + onToolNameToDescriptionChange={setToolNameToDescription} + keyTools={keyTools} + externalTools={tools} + externalIsLoading={isLoadingTools} + externalError={toolsError} + externalErrorStatus={toolsErrorStatus} + externalCanFetch={canFetchTools} + /> +
-
- - -
-
+ {/* Cost Configuration Section */} +
+ allowedTools.includes(tool.name))} + disabled={false} + /> +
+ +
+ + +
+ + +
); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/DcrBridgeToggle.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/DcrBridgeToggle.tsx index 49c182aa6be..35b23f9873c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/DcrBridgeToggle.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/DcrBridgeToggle.tsx @@ -1,16 +1,19 @@ import React from "react"; -import { Form, Switch, Tooltip } from "antd"; +import { Switch, Tooltip } from "antd"; import { InfoCircleOutlined } from "@ant-design/icons"; + +import { MountedFormField } from "@/components/common_components/MountedFormField"; import { isClientForwardedTokenMode } from "@/components/mcp_tools/types"; +import { switchControl } from "./mcpFieldRules"; /** * DCR-bridge toggle for the client-forwarded token modes (true_passthrough / * oauth_delegate); self-gates to those two auth types and renders nothing * otherwise. When on, OAuth-only clients like Claude Desktop can register and * sign in through the gateway; when off, the gateway relays the upstream - * server's own OAuth metadata instead. `initialChecked` seeds the antd - * Form.Item `initialValue` (not the Switch's DOM defaultChecked): the create - * form defaults it on, the edit form seeds it from the stored value. + * server's own OAuth metadata instead. `initialChecked` seeds the field's + * default value (not the Switch's DOM defaultChecked): the create form defaults + * it on, the edit form seeds it from the stored value. */ export default function DcrBridgeToggle({ authType, @@ -21,7 +24,7 @@ export default function DcrBridgeToggle({ }) { if (!isClientForwardedTokenMode(authType)) return null; return ( - Gateway-hosted sign-in (DCR bridge) @@ -31,10 +34,9 @@ export default function DcrBridgeToggle({ } name="dcr_bridge" - valuePropName="checked" - initialValue={initialChecked} + defaultValue={initialChecked} > - - + {(control) => } + ); } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.test.tsx new file mode 100644 index 00000000000..a57bde7eb9d --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.test.tsx @@ -0,0 +1,86 @@ +import React from "react"; +import { describe, it, expect, vi } from "vitest"; +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { FormProvider, useForm } from "react-hook-form"; + +import { + MountedFormProvider, + projectMountedValues, + useMountRegistry, + type MountedFormValues, +} from "@/components/common_components/MountedFormField"; +import EnvVarsSection from "./EnvVarsSection"; + +const renderSection = (defaultValues: MountedFormValues) => { + const onFinish = vi.fn(); + const Harness: React.FC = () => { + const form = useForm({ mode: "onChange", defaultValues }); + const registry = useMountRegistry(); + return ( + + +
{ + event.preventDefault(); + onFinish(projectMountedValues(registry, form.getValues)); + }} + > + + + +
+
+ ); + }; + render(); + return onFinish; +}; + +describe("EnvVarsSection", () => { + it("submits a per-user row whole, keeping the value key whose input the scope hides", async () => { + const onFinish = renderSection({ + env_vars: [{ name: "DB_USER", value: "admin", scope: "user", description: "Your DB username" }], + }); + + expect(screen.queryByPlaceholderText("e.g. postgresql")).not.toBeInTheDocument(); + await userEvent.click(screen.getByText("Submit")); + + expect(onFinish).toHaveBeenCalledWith( + expect.objectContaining({ + env_vars: [{ name: "DB_USER", value: "admin", scope: "user", description: "Your DB username" }], + }), + ); + }); + + it("submits an empty env_vars key when the list has no rows, rather than dropping the key", async () => { + const onFinish = renderSection({ env_vars: [] }); + + await userEvent.click(screen.getByText("Submit")); + + expect(onFinish.mock.calls[0][0]).toHaveProperty("env_vars", []); + }); + + it("carries a row added after mount into the submitted list, scoped global without the user picking one", async () => { + const onFinish = renderSection({ env_vars: [] }); + + await userEvent.click(screen.getByText("Add Variable")); + await userEvent.type(screen.getByPlaceholderText("e.g. DB_PROTOCOL"), "DB_PROTOCOL"); + await userEvent.type(screen.getByPlaceholderText("e.g. postgresql"), "postgresql"); + await userEvent.click(screen.getByText("Submit")); + + expect(onFinish).toHaveBeenCalledWith( + expect.objectContaining({ + env_vars: [expect.objectContaining({ name: "DB_PROTOCOL", value: "postgresql", scope: "global" })], + }), + ); + }); + + it("rejects a variable name that starts with a digit", async () => { + renderSection({ env_vars: [{ name: "", value: "", scope: "global", description: "" }] }); + + await userEvent.type(screen.getByPlaceholderText("e.g. DB_PROTOCOL"), "9LIVES"); + + expect(await screen.findByText("Use letters, digits, underscores; cannot start with a digit.")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.tsx index fbaacc40263..539a06910c1 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.tsx @@ -1,6 +1,16 @@ import React from "react"; -import { Form, Input, Select, Button, Tooltip, Typography } from "antd"; +import { Input, Select, Button, Tooltip, Typography } from "antd"; import { InfoCircleOutlined, MinusCircleOutlined, PlusOutlined } from "@ant-design/icons"; +import { useFieldArray, useFormContext, useWatch } from "react-hook-form"; + +import { + MountedFormField, + useMountedName, + type MountedFormValues, +} from "@/components/common_components/MountedFormField"; +import { antdRequired } from "@/components/common_components/antdFormRules"; +import { matchesPattern, selectControl, textControl } from "./mcpFieldRules"; +import { listControl } from "./mcpFormStore"; const { Text } = Typography; @@ -9,6 +19,8 @@ const SCOPE_OPTIONS = [ { value: "user", label: "Per-user" }, ]; +const VARIABLE_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; + /** * Form section for admin-configured MCP environment variables. * @@ -20,6 +32,10 @@ const SCOPE_OPTIONS = [ * The parent form reads the ``env_vars`` field from the form values. */ const EnvVarsSection: React.FC = () => { + const { control } = useFormContext(); + const { fields, append, remove } = useFieldArray({ control: listControl(control), name: "env_vars" }); + useMountedName("env_vars"); + return (
@@ -48,60 +64,52 @@ const EnvVarsSection: React.FC = () => { - - {(fields, { add, remove }) => ( -
- {fields.length > 0 && ( -
-
Variable Name
-
Value / Description
-
Scope
-
-
- )} - {fields.map(({ key, name, ...restField }) => ( -
- - - -
- -
- - + )} + +
+ +
+ + {(control) => - - - Hint - - - } - placeholder="e.g. Your DB username" - styles={{ input: { color: "#9ca3af" } }} - /> -
+ + {(control) => ( + + + + Hint + + + } + placeholder="e.g. Your DB username" + styles={{ input: { color: "#9ca3af" } }} + /> + )} + ); } return ( - - - + + {(control) => } + ); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/IdJagFormFields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/IdJagFormFields.tsx index e8730a5b974..9a6f4ab571f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/IdJagFormFields.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/IdJagFormFields.tsx @@ -1,7 +1,11 @@ import React from "react"; -import { Form, Input, Select, Tooltip } from "antd"; +import { Input, Select, Tooltip } from "antd"; import { InfoCircleOutlined } from "@ant-design/icons"; +import { MountedFormField } from "@/components/common_components/MountedFormField"; +import { antdRequired } from "@/components/common_components/antdFormRules"; +import { requiredUnlessSiblingSet, selectControl, textControl } from "./mcpFieldRules"; + interface IdJagFormFieldsProps { isEditing?: boolean; } @@ -17,12 +21,16 @@ const FieldLabel: React.FC<{ label: string; tooltip: string }> = ({ label, toolt ); +const PRIVATE_KEY_PATH = ["credentials", "client_private_key"] as const; + const IdJagFormFields: React.FC = ({ isEditing = false }) => { const placeholderSuffix = isEditing ? " (leave blank to keep existing)" : ""; + const requiredWhenCreating = (message: string) => + isEditing ? undefined : { validate: { required: antdRequired(message) } }; return ( <> - = ({ isEditing = false }) /> } name="token_exchange_endpoint" - rules={[{ required: !isEditing, message: "The org token endpoint is required for ID-JAG" }]} + required={!isEditing} + rules={requiredWhenCreating("The org token endpoint is required for ID-JAG")} > - - - ( + + )} + + = ({ isEditing = false }) /> } name={["credentials", "id_jag_resource_token_endpoint"]} - rules={[{ required: !isEditing, message: "The resource token endpoint is required for ID-JAG" }]} + required={!isEditing} + rules={requiredWhenCreating("The resource token endpoint is required for ID-JAG")} > - - - ( + + )} + + } name={["credentials", "client_id"]} - rules={[{ required: !isEditing, message: "Client ID is required for ID-JAG" }]} + required={!isEditing} + rules={requiredWhenCreating("Client ID is required for ID-JAG")} > - - - ( + + )} + + = ({ isEditing = false }) /> } name={["credentials", "client_secret"]} - dependencies={[["credentials", "client_private_key"]]} - rules={[ - ({ getFieldValue }) => ({ - validator: (_, value) => { - if (isEditing || value || getFieldValue(["credentials", "client_private_key"])) { - return Promise.resolve(); + rules={ + isEditing + ? undefined + : { + deps: ["credentials.client_private_key"], + validate: { + secretOrPrivateKey: requiredUnlessSiblingSet( + PRIVATE_KEY_PATH, + "Provide either a client secret or a client private key", + ), + }, } - return Promise.reject(new Error("Provide either a client secret or a client private key")); - }, - }), - ]} + } > - - - ( + + )} + + } - name={["credentials", "client_private_key"]} + name={PRIVATE_KEY_PATH} > - - - ( + + )} + + = ({ isEditing = false }) } name={["credentials", "client_private_key_id"]} > - - - } + + = ({ isEditing = false }) } name={["credentials", "client_assertion_signing_alg"]} > - - - } + + = ({ isEditing = false }) } name="audience" > - - - ( + + )} + + = ({ isEditing = false }) } name={["credentials", "id_jag_resource"]} > - - - ( + + )} + + = ({ isEditing = false }) } name="subject_token_type" > - - - ( + + )} + + } name={["credentials", "scopes"]} > - + )} + ); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.test.tsx index ee6aee86a4d..7d36872cee8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.test.tsx @@ -1,10 +1,10 @@ import React from "react"; -import { render, screen } from "@testing-library/react"; +import { screen } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { describe, it, expect } from "vitest"; -import { Form } from "antd"; import MCPPermissionManagement from "./MCPPermissionManagement"; +import { renderInMcpForm } from "./McpFormTestHarness"; const defaultProps = { availableAccessGroups: [], @@ -12,6 +12,7 @@ const defaultProps = { searchValue: "", setSearchValue: () => {}, getAccessGroupOptions: () => [], + mountedAuthType: undefined, }; describe("MCPPermissionManagement", () => { @@ -24,22 +25,8 @@ describe("MCPPermissionManagement", () => { return user; }; - const renderWithForm = (props = {}) => { - const Wrapper: React.FC = ({ children }) => { - const [form] = Form.useForm(); - return ( -
- {children} -
- ); - }; - - return render( - - - , - ); - }; + const renderWithForm = (props = {}) => + renderInMcpForm(, { allow_all_keys: false }); it("should default allow_all_keys switch to unchecked for new servers", async () => { renderWithForm(); @@ -51,27 +38,15 @@ describe("MCPPermissionManagement", () => { expect(toggle).not.toBeChecked(); }); - const renderWithInitialValues = (initialValues: Record, props = {}) => { - const Wrapper: React.FC = ({ children }) => { - const [form] = Form.useForm(); - return ( -
- {/* In the real app auth_type is registered by the parent form; the - component only watches it. Register a hidden field here so - Form.useWatch("auth_type") resolves the initial value. */} - - {children} -
- ); - }; - return render( - - - , + const renderWithInitialValues = (initialValues: Record, props = {}) => + renderInMcpForm( + , + initialValues, ); - }; it("shows only the oauth2 PKCE-delegation toggle for oauth2 servers", async () => { renderWithInitialValues({ allow_all_keys: false, auth_type: "oauth2" }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.tsx index aae13d4b467..3711140562f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.tsx @@ -1,7 +1,17 @@ import React, { useEffect } from "react"; -import { Alert, Form, Select, Tooltip, Collapse, Input, Space, Button, Switch } from "antd"; +import { Alert, Select, Tooltip, Collapse, Input, Space, Button, Switch } from "antd"; import { InfoCircleOutlined, MinusCircleOutlined, PlusOutlined } from "@ant-design/icons"; +import { useFieldArray, useFormContext, useWatch } from "react-hook-form"; import { MCPServer, AUTH_TYPE } from "@/components/mcp_tools/types"; +import { + MountedFormField, + useMountedName, + type MountedFormValues, +} from "@/components/common_components/MountedFormField"; +import { antdRequired } from "@/components/common_components/antdFormRules"; +import { Field, FieldLabel } from "@/components/shared/form/field"; +import { invertedSwitchControl, selectControl, switchControl, textControl } from "./mcpFieldRules"; +import { listControl } from "./mcpFormStore"; const { Panel } = Collapse; interface MCPPermissionManagementProps { @@ -13,20 +23,79 @@ interface MCPPermissionManagementProps { value: string; label: React.ReactNode; }>; + /** + * The auth type as seen through the gate that mounts the auth_type field. + * Callers pass undefined whenever that field is unmounted, because both + * toggles below are mounted from this value and the payload only carries + * what is mounted. + */ + mountedAuthType: string | null | undefined; } +const StaticHeadersFieldArray: React.FC = () => { + const { control } = useFormContext(); + const { fields, append, remove } = useFieldArray({ control: listControl(control), name: "static_headers" }); + useMountedName("static_headers"); + + return ( +
+ {fields.map((item, index) => ( + + + {(headerControl) => ( + + )} + + + {(valueControl) => ( + + )} + + remove(index)} + className="text-gray-500 hover:text-red-500 cursor-pointer" + /> + + ))} + +
+ ); +}; + const MCPPermissionManagement: React.FC = ({ availableAccessGroups, mcpServer, searchValue, setSearchValue, getAccessGroupOptions, + mountedAuthType, }) => { - const form = Form.useFormInstance(); - const watchedAuthType = Form.useWatch("auth_type", form); - const isOAuth2 = watchedAuthType === AUTH_TYPE.OAUTH2; - const isNoneAuth = watchedAuthType === AUTH_TYPE.NONE || watchedAuthType == null; - const watchedExtraHeaders = Form.useWatch("extra_headers", form); + const { setValue } = useFormContext(); + const isOAuth2 = mountedAuthType === AUTH_TYPE.OAUTH2; + const isNoneAuth = mountedAuthType === AUTH_TYPE.NONE || mountedAuthType == null; + const watchedExtraHeaders = useWatch({ name: "extra_headers" }); const hasAuthorizationHeader = Array.isArray(watchedExtraHeaders) && watchedExtraHeaders.some((h) => typeof h === "string" && h.toLowerCase() === "authorization"); @@ -39,8 +108,8 @@ const MCPPermissionManagement: React.FC = ({ // Kept as separate flags so neither silently implies the other and existing // oauth2 servers can't regress into pass-through behavior. const canEnableOAuthPassthrough = isNoneAuth && hasAuthorizationHeader; - const watchedDelegateAuth = Form.useWatch("delegate_auth_to_upstream", form); - const watchedPublicInternet = Form.useWatch("available_on_public_internet", form); + const watchedDelegateAuth = useWatch({ name: "delegate_auth_to_upstream" }); + const watchedPublicInternet = useWatch({ name: "available_on_public_internet" }); const showInternalDelegatePkceWarning = isOAuth2 && watchedDelegateAuth === true && watchedPublicInternet === false; // Set initial values when mcpServer changes @@ -51,10 +120,10 @@ const MCPPermissionManagement: React.FC = ({ header, value: value != null ? String(value) : "", })); - form.setFieldValue("static_headers", staticHeaders); + setValue("static_headers", staticHeaders); } if (Array.isArray(mcpServer.env_vars) && mcpServer.env_vars.length > 0) { - form.setFieldValue( + setValue( "env_vars", mcpServer.env_vars.map((entry) => ({ name: entry.name, @@ -65,41 +134,41 @@ const MCPPermissionManagement: React.FC = ({ ); } if (typeof mcpServer.allow_all_keys === "boolean") { - form.setFieldValue("allow_all_keys", mcpServer.allow_all_keys); + setValue("allow_all_keys", mcpServer.allow_all_keys); } if (typeof mcpServer.available_on_public_internet === "boolean") { - form.setFieldValue("available_on_public_internet", mcpServer.available_on_public_internet); + setValue("available_on_public_internet", mcpServer.available_on_public_internet); } if (typeof mcpServer.delegate_auth_to_upstream === "boolean") { - form.setFieldValue("delegate_auth_to_upstream", mcpServer.delegate_auth_to_upstream); + setValue("delegate_auth_to_upstream", mcpServer.delegate_auth_to_upstream); } if (typeof mcpServer.oauth_passthrough === "boolean") { - form.setFieldValue("oauth_passthrough", mcpServer.oauth_passthrough); + setValue("oauth_passthrough", mcpServer.oauth_passthrough); } } else { - form.setFieldValue("allow_all_keys", false); - form.setFieldValue("available_on_public_internet", true); - form.setFieldValue("delegate_auth_to_upstream", false); - form.setFieldValue("oauth_passthrough", false); + setValue("allow_all_keys", false); + setValue("available_on_public_internet", true); + setValue("delegate_auth_to_upstream", false); + setValue("oauth_passthrough", false); } - }, [mcpServer, form]); + }, [mcpServer, setValue]); // delegate_auth_to_upstream is only honored server-side for oauth2 servers. // Force it back to false whenever the user switches away from oauth2 so a // stale toggle value doesn't get persisted unexpectedly. useEffect(() => { if (!isOAuth2) { - form.setFieldValue("delegate_auth_to_upstream", false); + setValue("delegate_auth_to_upstream", false); } - }, [isOAuth2, form]); + }, [isOAuth2, setValue]); // oauth_passthrough is only honored for auth_type=none servers that forward // Authorization upstream. Force it back to false otherwise. useEffect(() => { if (!canEnableOAuthPassthrough) { - form.setFieldValue("oauth_passthrough", false); + setValue("oauth_passthrough", false); } - }, [canEnableOAuthPassthrough, form]); + }, [canEnableOAuthPassthrough, setValue]); return ( @@ -130,14 +199,9 @@ const MCPPermissionManagement: React.FC = ({ Enable if this server should be "public" to all keys.

- - - + + {(control) => } +
@@ -152,16 +216,9 @@ const MCPPermissionManagement: React.FC = ({ Turn on to restrict access to callers within your internal network only.

- ({ checked: !value })} - getValueFromEvent={(checked: boolean) => !checked} - initialValue={true} - className="mb-0" - > - - + + {(control) => } +
{isOAuth2 && ( @@ -177,14 +234,13 @@ const MCPPermissionManagement: React.FC = ({ Bypass LiteLLM auth so clients authenticate directly with the upstream OAuth MCP server.

- - - + {(control) => } +
)} @@ -202,14 +258,13 @@ const MCPPermissionManagement: React.FC = ({ upstream MCP server.

- - - + {(control) => } +
)} @@ -223,7 +278,7 @@ const MCPPermissionManagement: React.FC = ({ /> )} - MCP Access Groups @@ -235,21 +290,24 @@ const MCPPermissionManagement: React.FC = ({ name="mcp_access_groups" className="mb-4" > - (option?.value ?? "").toLowerCase().includes(input.toLowerCase())} + onSearch={(value) => setSearchValue(value)} + tokenSeparators={[","]} + options={getAccessGroupOptions()} + maxTagCount="responsive" + allowClear + /> + )} + - Extra Headers @@ -265,70 +323,34 @@ const MCPPermissionManagement: React.FC = ({ } name="extra_headers" > - 0 + ? `Currently: ${mcpServer.extra_headers.join(", ")}` + : "Enter header names (e.g., Authorization, X-Custom-Header)" + } + className="rounded-lg" + size="large" + tokenSeparators={[","]} + allowClear + /> + )} + - + Static Headers - } - required={false} - > - - {(fields, { add, remove }) => ( -
- {fields.map(({ key, name, ...restField }) => ( - - - - - - - - remove(name)} - className="text-gray-500 hover:text-red-500 cursor-pointer" - /> - - ))} - -
- )} -
-
+ + +
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/McpFormTestHarness.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/McpFormTestHarness.tsx new file mode 100644 index 00000000000..0529c9c7c61 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/McpFormTestHarness.tsx @@ -0,0 +1,37 @@ +import * as React from "react"; +import { render, type RenderResult } from "@testing-library/react"; +import { FormProvider, useForm } from "react-hook-form"; + +import { + MountedFormProvider, + projectMountedValues, + useMountRegistry, + type MountedFormValues, +} from "@/components/common_components/MountedFormField"; + +export const McpFormHarness: React.FC<{ + defaultValues?: MountedFormValues; + onFinish?: (values: MountedFormValues) => void; + children: React.ReactNode; +}> = ({ defaultValues, onFinish, children }) => { + const form = useForm({ mode: "onChange", defaultValues }); + const registry = useMountRegistry(); + return ( + + +
{ + event.preventDefault(); + onFinish?.(projectMountedValues(registry, form.getValues)); + }} + > + {children} + +
+
+
+ ); +}; + +export const renderInMcpForm = (ui: React.ReactNode, defaultValues: MountedFormValues = {}): RenderResult => + render({ui}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.test.tsx index 48964490339..21bb2801e8c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.test.tsx @@ -1,24 +1,8 @@ import React from "react"; import { describe, it, expect, vi, beforeEach } from "vitest"; import { render, screen, waitFor, act, fireEvent } from "@testing-library/react"; -import { Form } from "antd"; import OAuthFormFields from "./OAuthFormFields"; - -// ── helpers ────────────────────────────────────────────────────────────────── - -/** Minimal Ant Form wrapper so Form.Item registers correctly. */ -const WithForm: React.FC<{ children: React.ReactNode; onFinish?: (values: any) => void }> = ({ - children, - onFinish, -}) => { - const [form] = Form.useForm(); - return ( -
- {children} - -
- ); -}; +import { McpFormHarness as WithForm } from "./McpFormTestHarness"; // ── tests ───────────────────────────────────────────────────────────────────── diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.tsx index cbe6ac18d22..545efb910c4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.tsx @@ -1,10 +1,13 @@ import React from "react"; -import { Form, Input as AntdInput, InputNumber, Select, Tooltip } from "antd"; +import { Input as AntdInput, InputNumber, Select, Tooltip } from "antd"; import { InfoCircleOutlined } from "@ant-design/icons"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { OAUTH_FLOW } from "@/components/mcp_tools/types"; +import { MountedFormField } from "@/components/common_components/MountedFormField"; +import { antdRequired } from "@/components/common_components/antdFormRules"; import TokenEndpointAuthMethodField from "./TokenEndpointAuthMethodField"; +import { numberControl, parsesAsJson, selectControl, textControl } from "./mcpFieldRules"; interface OAuthFlowStatus { startOAuthFlow: () => void; @@ -41,12 +44,14 @@ const FieldLabel: React.FC<{ label: string; tooltip: string }> = ({ label, toolt ); const UpstreamResourceField: React.FC = () => ( - } name={["credentials", "upstream_resource"]} > - - + {(control) => ( + + )} + ); const OAuthFormFields: React.FC = ({ @@ -57,11 +62,12 @@ const OAuthFormFields: React.FC = ({ docsUrl, }) => { const placeholderSuffix = isEditing ? " (leave blank to keep existing)" : ""; - const requiredWhenCreating = (message: string) => (isEditing ? [] : [{ required: true, message }]); + const requiredWhenCreating = (message: string) => + isEditing ? undefined : { validate: { required: antdRequired(message) } }; return ( <> - = ({ /> } name="oauth_flow_type" - {...(initialFlowType ? { initialValue: initialFlowType } : {})} + {...(initialFlowType ? { defaultValue: initialFlowType } : {})} > - - + {(control) => ( + + )} + {isM2M ? ( <> - } name={["credentials", "client_id"]} + required={!isEditing} rules={requiredWhenCreating("Client ID is required for M2M OAuth")} > - - - ( + + )} + + } name={["credentials", "client_secret"]} + required={!isEditing} rules={requiredWhenCreating("Client Secret is required for M2M OAuth")} > - - - ( + + )} + + } name="token_url" + required={!isEditing} rules={requiredWhenCreating("Token URL is required for M2M OAuth")} > - - + {(control) => ( + + )} + - = ({ } name={["credentials", "scopes"]} > - + )} + ) : ( <> - = ({ } name={["credentials", "client_id"]} > - - - ( + + )} + + = ({ } name={["credentials", "client_secret"]} > - - - ( + + )} + + = ({ } name={["credentials", "scopes"]} > - + )} + - = ({ } name="issuer" > - - - ( + + )} + + = ({ } name="authorization_url" > - - - ( + + )} + + } name="token_url" > - - + {(control) => ( + + )} + - = ({ } name="registration_url" > - - - ( + + )} + + = ({ /> } name="token_validation_json" - rules={[ - { - validator: (_: any, value: string) => { - if (!value || value.trim() === "") return Promise.resolve(); - try { - JSON.parse(value); - return Promise.resolve(); - } catch { - return Promise.reject(new Error("Must be valid JSON")); - } - }, - }, - ]} + rules={{ validate: { json: parsesAsJson("Must be valid JSON") } }} > - - - ( + + )} + + = ({ } name="token_storage_ttl_seconds" > - - + {(control) => ( + + )} + {oauthFlow && (

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenAPIFormSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenAPIFormSection.tsx index 073780b359f..78c8bbe73a9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenAPIFormSection.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenAPIFormSection.tsx @@ -1,12 +1,15 @@ import React, { useState } from "react"; -import { Form, Input, Tooltip } from "antd"; +import { Input, Tooltip } from "antd"; import { InfoCircleOutlined } from "@ant-design/icons"; -import { FormInstance } from "antd/es/form"; import { AUTH_TYPE, OAUTH_FLOW } from "@/components/mcp_tools/types"; +import { MountedFormField } from "@/components/common_components/MountedFormField"; +import { antdRequired } from "@/components/common_components/antdFormRules"; import OpenAPIQuickPicker, { OpenAPIRegistryEntry, OpenAPIKeyTool } from "./OpenAPIQuickPicker"; +import { McpForm, resetFields, setFieldsValue } from "./mcpFormStore"; +import { textControl } from "./mcpFieldRules"; interface OpenAPIFormSectionProps { - form: FormInstance; + form: McpForm; accessToken: string | null; /** Called when a preset is selected so the parent can sync its formValues state. */ onValuesChange: (updates: Record) => void; @@ -47,13 +50,11 @@ const OpenAPIFormSection: React.FC = ({ updates.oauth_flow_type = OAUTH_FLOW.INTERACTIVE; updates.authorization_url = entry.oauth.authorization_url; updates.token_url = entry.oauth.token_url; - form.setFieldsValue(updates); + setFieldsValue(form, updates); onOAuthDocsUrlChange?.(entry.oauth.docs_url ?? null); } else { - // resetFields is required to visually clear Ant Design form fields — - // setFieldsValue with undefined silently skips undefined keys. - form.resetFields(["auth_type", "authorization_url", "token_url"]); - form.setFieldsValue(updates); + resetFields(form, ["auth_type", "authorization_url", "token_url"]); + setFieldsValue(form, updates); onOAuthDocsUrlChange?.(null); } onValuesChange(updates); @@ -63,7 +64,7 @@ const OpenAPIFormSection: React.FC = ({ <> - OpenAPI Spec URL @@ -73,20 +74,25 @@ const OpenAPIFormSection: React.FC = ({ } name="spec_path" - rules={[{ required: true, message: "Please enter an OpenAPI spec URL" }]} + required + rules={{ validate: { required: antdRequired("Please enter an OpenAPI spec URL") } }} > - { - // Clear the preset selection when the user manually edits the spec URL - // so stale suggested tools from a previous preset don't persist. - setSelectedPreset(null); - onKeyToolsChange?.([]); - onOAuthDocsUrlChange?.(null); - }} - /> - + {(control) => ( + { + control.onChange(event); + // Clear the preset selection when the user manually edits the spec URL + // so stale suggested tools from a previous preset don't persist. + setSelectedPreset(null); + onKeyToolsChange?.([]); + onOAuthDocsUrlChange?.(null); + }} + /> + )} + ); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenApiByokFields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenApiByokFields.tsx index 2ac4279e20a..83c841439f0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenApiByokFields.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenApiByokFields.tsx @@ -1,91 +1,101 @@ import React from "react"; -import { Form, Input, Select, Switch, Tooltip } from "antd"; +import { Input, Select, Switch, Tooltip } from "antd"; import { InfoCircleOutlined } from "@ant-design/icons"; +import { useWatch } from "react-hook-form"; -const OpenApiByokFields: React.FC = () => ( - <> - - BYOK (Bring Your Own Key) - - - - - } - name="is_byok" - valuePropName="checked" - > - - +import { MountedFormField } from "@/components/common_components/MountedFormField"; +import { selectControl, switchControl, textControl } from "./mcpFieldRules"; - prev.is_byok !== cur.is_byok || prev.auth_type !== cur.auth_type}> - {({ getFieldValue }) => - getFieldValue("is_byok") ? ( - <> - {/* Auth format hint */} - {getFieldValue("auth_type") && getFieldValue("auth_type") !== "none" && ( -

- - - User keys will be sent as:{" "} - - {getFieldValue("auth_type") === "bearer_token" && "Authorization: Bearer {key}"} - {getFieldValue("auth_type") === "token" && "Authorization: token {key}"} - {getFieldValue("auth_type") === "api_key" && "x-api-key: {key}"} - {getFieldValue("auth_type") === "basic" && "Authorization: Basic {key}"} - {getFieldValue("auth_type") === "authorization" && "Authorization: {key}"} - - {!getFieldValue("auth_type") && "Set Authentication Type below to specify the format."} - -
- )} - {!getFieldValue("auth_type") && ( -
- - - Set the Authentication Type below to specify how user keys are sent (e.g., Bearer - Token, API Key header). - -
- )} - - Access Description - - - - - } - name="byok_description" - > +const AUTH_HEADER_FORMATS: Readonly> = { + bearer_token: "Authorization: Bearer {key}", + token: "Authorization: token {key}", + api_key: "x-api-key: {key}", + basic: "Authorization: Basic {key}", + authorization: "Authorization: {key}", +}; + +const OpenApiByokFields: React.FC = () => { + const isByok = Boolean(useWatch({ name: "is_byok" })); + const authType = useWatch({ name: "auth_type" }) as string | undefined; + const hasAuthType = Boolean(authType) && authType !== "none"; + + return ( + <> + + BYOK (Bring Your Own Key) + + + + + } + name="is_byok" + > + {(control) => } + + + {isByok && ( + <> + {hasAuthType && ( +
+ + + User keys will be sent as:{" "} + + {authType === undefined ? "" : AUTH_HEADER_FORMATS[authType]} + + +
+ )} + {!authType && ( +
+ + + Set the Authentication Type below to specify how user keys are sent (e.g., Bearer + Token, API Key header). + +
+ )} + + Access Description + + + + + } + name="byok_description" + > + {(control) => ( -
- - ) : null - } - - -); + + API Key Help URL + + + + + } + name="byok_api_key_help_url" + > + {(control) => } + + + )} + + ); +}; export default OpenApiByokFields; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/PassthroughAuthorizeSection.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/PassthroughAuthorizeSection.test.tsx index 0a09ef3f856..a7577ccb781 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/PassthroughAuthorizeSection.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/PassthroughAuthorizeSection.test.tsx @@ -1,13 +1,10 @@ import React from "react"; import { describe, it, expect } from "vitest"; import { render, screen } from "@testing-library/react"; -import { Form } from "antd"; import PassthroughAuthorizeSection from "./PassthroughAuthorizeSection"; +import { McpFormHarness } from "./McpFormTestHarness"; -const WithForm: React.FC<{ children: React.ReactNode }> = ({ children }) => { - const [form] = Form.useForm(); - return
{children}
; -}; +const WithForm = McpFormHarness; const noopFlow = { startOAuthFlow: () => {}, status: "idle", error: null, tokenResponse: null }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/PassthroughAuthorizeSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/PassthroughAuthorizeSection.tsx index dc10f0f1392..375e1dfe515 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/PassthroughAuthorizeSection.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/PassthroughAuthorizeSection.tsx @@ -1,6 +1,8 @@ import React from "react"; -import { Button, Checkbox, Form, Input } from "antd"; +import { Button, Checkbox, Input } from "antd"; import DcrBridgeToggle from "./DcrBridgeToggle"; +import { MountedFormField } from "@/components/common_components/MountedFormField"; +import { textControl } from "./mcpFieldRules"; import { credentialAuthClass, isClientForwardedTokenMode } from "@/components/mcp_tools/types"; interface PassthroughOAuthFlow { @@ -81,27 +83,33 @@ export default function PassthroughAuthorizeSection({ and may not be valid. Update the client ID, or clear it to use dynamic client registration.

)} - OAuth Client ID (optional)} name={["credentials", "client_id"]} - extra={clientIdExtra} + help={clientIdExtra} > - - - ( + + )} + + OAuth Client Secret (optional)} name={["credentials", "client_secret"]} > - - + {(control) => ( + + )} + {isEditing && onRemoveStoredAppChange && ( onRemoveStoredAppChange(e.target.checked)}> diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/StdioConfiguration.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/StdioConfiguration.tsx index 476a5b61683..bb410aa19dc 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/StdioConfiguration.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/StdioConfiguration.tsx @@ -1,7 +1,11 @@ import React from "react"; -import { Form, Input, Tooltip } from "antd"; +import { Input, Tooltip } from "antd"; import { InfoCircleOutlined } from "@ant-design/icons"; +import { MountedFormField } from "@/components/common_components/MountedFormField"; +import { antdRequired } from "@/components/common_components/antdFormRules"; +import { parsesAsJson, textControl } from "./mcpFieldRules"; + interface StdioConfigurationProps { isVisible: boolean; /** @@ -11,37 +15,7 @@ interface StdioConfigurationProps { required?: boolean; } -const StdioConfiguration: React.FC = ({ isVisible, required = true }) => { - if (!isVisible) return null; - - return ( - - Stdio Configuration (JSON) - - - - - } - name="stdio_config" - rules={[ - ...(required ? [{ required: true, message: "Please enter stdio configuration" }] : []), - { - validator: (_, value) => { - if (!value) return Promise.resolve(); - try { - JSON.parse(value); - return Promise.resolve(); - } catch { - return Promise.reject("Please enter valid JSON"); - } - }, - }, - ]} - > - = ({ isVisible, requ } } } -}`} - rows={12} - className="rounded-lg border-gray-300 focus:border-blue-500 focus:ring-blue-500 font-mono text-sm" - /> - +}`; + +const StdioConfiguration: React.FC = ({ isVisible, required = true }) => { + if (!isVisible) return null; + + return ( + + Stdio Configuration (JSON) + + + + + } + name="stdio_config" + required={required} + rules={{ + validate: { + ...(required ? { required: antdRequired("Please enter stdio configuration") } : {}), + json: parsesAsJson("Please enter valid JSON"), + }, + }} + > + {(control) => ( + + )} + ); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenEndpointAuthMethodField.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenEndpointAuthMethodField.tsx index c97ce96bfb1..38fa3573079 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenEndpointAuthMethodField.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenEndpointAuthMethodField.tsx @@ -1,7 +1,10 @@ import React from "react"; -import { Form, Select, Tooltip } from "antd"; +import { Select, Tooltip } from "antd"; import { InfoCircleOutlined } from "@ant-design/icons"; +import { MountedFormField } from "@/components/common_components/MountedFormField"; +import { selectControl } from "./mcpFieldRules"; + const TOKEN_ENDPOINT_AUTH_METHOD_OPTIONS = [ { value: "client_secret_basic", label: "Client Secret Basic" }, { value: "client_secret_post", label: "Client Secret Post" }, @@ -12,7 +15,7 @@ interface TokenEndpointAuthMethodFieldProps { } const TokenEndpointAuthMethodField: React.FC = ({ isEditing = false }) => ( - Token Endpoint Auth Method (optional) @@ -23,16 +26,19 @@ const TokenEndpointAuthMethodField: React.FC } name={["credentials", "token_endpoint_auth_method"]} > - + )} + ); export default TokenEndpointAuthMethodField; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenExchangeFormFields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenExchangeFormFields.tsx index 9e1e1a85743..ba213b20655 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenExchangeFormFields.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenExchangeFormFields.tsx @@ -1,6 +1,11 @@ import React from "react"; -import { Form, Input, Select, Tooltip } from "antd"; +import { Input, Select, Tooltip } from "antd"; import { InfoCircleOutlined } from "@ant-design/icons"; +import { useWatch } from "react-hook-form"; + +import { MountedFormField } from "@/components/common_components/MountedFormField"; +import { antdRequired } from "@/components/common_components/antdFormRules"; +import { selectControl, textControl } from "./mcpFieldRules"; interface TokenExchangeFormFieldsProps { isEditing?: boolean; @@ -19,10 +24,13 @@ const FieldLabel: React.FC<{ label: string; tooltip: string }> = ({ label, toolt const TokenExchangeFormFields: React.FC = ({ isEditing = false }) => { const placeholderSuffix = isEditing ? " (leave blank to keep existing)" : ""; + const isEntraObo = useWatch({ name: "token_exchange_profile" }) === "entra_obo"; + const requiredWhenCreating = (message: string) => + isEditing ? undefined : { validate: { required: antdRequired(message) } }; return ( <> - = ({ isEdi /> } name="token_exchange_profile" - {...(isEditing ? {} : { initialValue: "rfc8693" })} + {...(isEditing ? {} : { defaultValue: "rfc8693" })} > - - - ( + + )} + + = ({ isEdi } name="token_exchange_endpoint" > - - - ( + + )} + + = ({ isEdi /> } name={["credentials", "client_id"]} - rules={[{ required: !isEditing, message: "Client ID is required for token exchange" }]} + required={!isEditing} + rules={requiredWhenCreating("Client ID is required for token exchange")} > - - - ( + + )} + + = ({ isEdi /> } name={["credentials", "client_secret"]} - rules={[{ required: !isEditing, message: "Client Secret is required for token exchange" }]} + required={!isEditing} + rules={requiredWhenCreating("Client Secret is required for token exchange")} > - - - prev.token_exchange_profile !== cur.token_exchange_profile}> - {({ getFieldValue }) => { - const isEntraObo = getFieldValue("token_exchange_profile") === "entra_obo"; - return ( - <> - {!isEntraObo && ( - <> - - } - name="audience" - > - - - - } - name="subject_token_type" - > - - - - )} - /.default)." - : "Optional scopes to request during the token exchange." - } - /> - } - name={["credentials", "scopes"]} - rules={ - isEntraObo - ? [ - { - required: true, - message: "Microsoft Entra OBO requires a scope, e.g. api:///.default", - }, - ] - : [] - } - > - + )} + + + } + name="subject_token_type" + > + {(control) => ( + + )} + + + )} + /.default)." + : "Optional scopes to request during the token exchange." + } + /> + } + name={["credentials", "scopes"]} + required={isEntraObo} + rules={ + isEntraObo + ? { + validate: { + required: antdRequired("Microsoft Entra OBO requires a scope, e.g. api:///.default"), + }, + } + : undefined + } + > + {(control) => ( + - - validateMCPServerName(value), - }, - ]} - > - setAliasManuallyEdited(true)} - className="rounded-lg border-gray-300 focus:border-blue-500 focus:ring-blue-500" - /> - - - - - - - - - - {/* URL field - only for HTTP/SSE */} - {isMCPTransport && ( - validateMCPServerUrl(value) }, - ]} - > - - - )} - - {/* OpenAPI Spec URL - only for OpenAPI transport */} - {isOpenAPITransport && ( - - OpenAPI Spec URL - - - - - } - name="spec_path" - rules={[{ required: true, message: "Please enter an OpenAPI spec URL" }]} - > - - - )} - - - Max Concurrent Requests (optional) - - - - - } - name="max_concurrent_requests" - > - - - - {/* Authentication - for HTTP, SSE, and OpenAPI */} - {!isStdioTransport && ( - <> - - - - - - - )} - - {isStdioTransport && ( -
-

- Configure the stdio transport used to launch the MCP server process. You can either fill in the fields - below or paste a JSON configuration. -

- - - - - - - - - - AWS Service Name - - - - - } - name={["credentials", "aws_service_name"]} - > - - - - AWS Access Key ID - - - - - } - name={["credentials", "aws_access_key_id"]} - rules={[]} - > - - - - AWS Secret Access Key - - - - - } - name={["credentials", "aws_secret_access_key"]} - rules={[]} - > - - - - AWS Session Token - - - - - } - name={["credentials", "aws_session_token"]} - > - - - - AWS Role ARN - - - - - } - name={["credentials", "aws_role_name"]} - > - - - - AWS Session Name - - - - - } - name={["credentials", "aws_session_name"]} - > - - - - )} - - {/* Environment Variables Section */} -
- -
- - {/* Permission Management / Access Control Section */} -
- -
- - {/* Tool Configuration Section */} -
- + +
{ + event.preventDefault(); + void submitForm(); }} - allowedTools={allowedTools} - existingAllowedTools={existingAllowedTools} - hasToolAllowlistInteraction={hasToolAllowlistInteraction} - isEditMode - onAllowedToolsChange={setAllowedTools} - onToolAllowlistInteraction={() => setHasToolAllowlistInteraction(true)} - toolNameToDisplayName={toolNameToDisplayName} - toolNameToDescription={toolNameToDescription} - onToolNameToDisplayNameChange={setToolNameToDisplayName} - onToolNameToDescriptionChange={setToolNameToDescription} - externalTools={tools} - externalIsLoading={isLoadingTools} - externalError={toolsError} - externalCanFetch={true} - /> -
+ > + validateMCPServerName(value) }) }} + > + {(control) => ( + + )} + + validateMCPServerName(value) }) }} + > + {(control) => ( + { + control.onChange(event); + setAliasManuallyEdited(true); + }} + className="rounded-lg border-gray-300 focus:border-blue-500 focus:ring-blue-500" + /> + )} + + + {(control) => ( + + )} + + + + {(control) => ( + + )} + -
- Cancel - -
- + {/* URL field - only for HTTP/SSE */} + {isMCPTransport && ( + validateMCPServerUrl(value) }), + }, + }} + > + {(control) => ( + + )} + + )} + + {/* OpenAPI Spec URL - only for OpenAPI transport */} + {isOpenAPITransport && ( + + OpenAPI Spec URL + + + + + } + name="spec_path" + required + rules={{ validate: { required: antdRequired("Please enter an OpenAPI spec URL") } }} + > + {(control) => ( + + )} + + )} + + + Max Concurrent Requests (optional) + + + + + } + name="max_concurrent_requests" + > + {(control) => ( + + )} + + + {/* Authentication - for HTTP, SSE, and OpenAPI */} + {!isStdioTransport && ( + <> + + {(control) => ( + + )} + + + + + )} + + {isStdioTransport && ( +
+

+ Configure the stdio transport used to launch the MCP server process. You can either fill in the + fields below or paste a JSON configuration. +

+ + + {(control) => ( + + )} + + + + {(control) => ( + + )} + + + AWS Service Name + + + + + } + name={["credentials", "aws_service_name"]} + > + {(control) => ( + + )} + + + AWS Access Key ID + + + + + } + name={["credentials", "aws_access_key_id"]} + > + {(control) => ( + + )} + + + AWS Secret Access Key + + + + + } + name={["credentials", "aws_secret_access_key"]} + > + {(control) => ( + + )} + + + AWS Session Token + + + + + } + name={["credentials", "aws_session_token"]} + > + {(control) => ( + + )} + + + AWS Role ARN + + + + + } + name={["credentials", "aws_role_name"]} + > + {(control) => ( + + )} + + + AWS Session Name + + + + + } + name={["credentials", "aws_session_name"]} + > + {(control) => ( + + )} + + + )} + + {/* Environment Variables Section */} +
+ +
+ + {/* Permission Management / Access Control Section */} +
+ +
+ + {/* Tool Configuration Section */} +
+ setHasToolAllowlistInteraction(true)} + toolNameToDisplayName={toolNameToDisplayName} + toolNameToDescription={toolNameToDescription} + onToolNameToDisplayNameChange={setToolNameToDisplayName} + onToolNameToDescriptionChange={setToolNameToDescription} + externalTools={tools} + externalIsLoading={isLoadingTools} + externalError={toolsError} + externalCanFetch={true} + /> +
+ +
+ Cancel + +
+ + + @@ -1460,7 +1285,7 @@ const MCPServerEdit: React.FC = ({
Cancel - +
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mountedServerFields.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mountedServerFields.test.ts new file mode 100644 index 00000000000..dd9c8db6d30 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mountedServerFields.test.ts @@ -0,0 +1,497 @@ +import { describe, expect, it } from "vitest"; +import { + mountedCreateFieldNames, + mountedEditFieldNames, + projectMountedCreateValues, + projectMountedEditValues, +} from "./mountedServerFields"; + +const editRoot = (values: Record) => mountedEditFieldNames(values).root; +const editCreds = (values: Record) => mountedEditFieldNames(values).credentials; +const createRoot = (values: Record) => mountedCreateFieldNames(values).root; +const createCreds = (values: Record) => mountedCreateFieldNames(values).credentials; + +const HTTP_NONE = { transport: "http", auth_type: "none" }; + +describe("edit root: transport gates", () => { + it("mounts url for http but not spec_path or the stdio group", () => { + const root = editRoot(HTTP_NONE); + expect(root).toContain("url"); + expect(root).not.toContain("spec_path"); + expect(root).not.toContain("command"); + expect(root).not.toContain("stdio_config"); + }); + + it("mounts url for sse", () => { + expect(editRoot({ transport: "sse", auth_type: "none" })).toContain("url"); + }); + + it("mounts spec_path and not url for openapi", () => { + const root = editRoot({ transport: "openapi", auth_type: "none" }); + expect(root).toContain("spec_path"); + expect(root).not.toContain("url"); + }); + + it("swaps the whole auth subtree for the stdio group on stdio", () => { + const root = editRoot({ transport: "stdio", auth_type: "oauth2" }); + expect(root).toStrictEqual([ + "server_name", + "alias", + "description", + "transport", + "max_concurrent_requests", + "command", + "args", + "env_json", + "stdio_config", + "env_vars", + "allow_all_keys", + "available_on_public_internet", + "mcp_access_groups", + "extra_headers", + "static_headers", + ]); + }); + + it("drops every credential on stdio even when auth_type is stored as oauth2", () => { + expect(editCreds({ transport: "stdio", auth_type: "oauth2" })).toStrictEqual([]); + }); +}); + +describe("edit root: auth_type gates", () => { + it("mounts credentials.auth_value only for the four value-bearing auth types", () => { + for (const authType of ["api_key", "bearer_token", "token", "basic"]) { + expect(editCreds({ transport: "http", auth_type: authType })).toStrictEqual(["auth_value"]); + } + expect(editCreds(HTTP_NONE)).toStrictEqual([]); + }); + + it("swaps the oauth2 endpoint set on the M2M flow", () => { + const m2m = editRoot({ transport: "http", auth_type: "oauth2", oauth_flow_type: "m2m" }); + const interactive = editRoot({ transport: "http", auth_type: "oauth2", oauth_flow_type: "interactive" }); + expect(m2m).toContain("token_url"); + expect(m2m).not.toContain("issuer"); + expect(m2m).not.toContain("registration_url"); + expect(interactive).toContain("issuer"); + expect(interactive).toContain("registration_url"); + }); + + it("mounts token_validation_json ONLY on the interactive oauth2 branch", () => { + expect(editRoot({ transport: "http", auth_type: "oauth2", oauth_flow_type: "interactive" })).toContain( + "token_validation_json", + ); + expect(editRoot({ transport: "http", auth_type: "oauth2", oauth_flow_type: "m2m" })).not.toContain( + "token_validation_json", + ); + expect(editRoot({ transport: "http", auth_type: "oauth2_token_exchange" })).not.toContain("token_validation_json"); + expect(editRoot(HTTP_NONE)).not.toContain("token_validation_json"); + }); + + it("mounts token_storage_ttl_seconds only on the interactive oauth2 branch", () => { + expect(editRoot({ transport: "http", auth_type: "oauth2", oauth_flow_type: "interactive" })).toContain( + "token_storage_ttl_seconds", + ); + expect(editRoot({ transport: "http", auth_type: "oauth2", oauth_flow_type: "m2m" })).not.toContain( + "token_storage_ttl_seconds", + ); + }); + + it("gates audience and subject_token_type on the entra_obo token-exchange profile", () => { + const rfc = editRoot({ transport: "http", auth_type: "oauth2_token_exchange", token_exchange_profile: "rfc8693" }); + const entra = editRoot({ + transport: "http", + auth_type: "oauth2_token_exchange", + token_exchange_profile: "entra_obo", + }); + expect(rfc).toContain("audience"); + expect(rfc).toContain("subject_token_type"); + expect(entra).not.toContain("audience"); + expect(entra).not.toContain("subject_token_type"); + expect(entra).toContain("token_exchange_profile"); + }); + + it("mounts the seven aws credentials only for aws_sigv4", () => { + expect(sorted(editCreds({ transport: "http", auth_type: "aws_sigv4" }))).toStrictEqual( + sorted([ + "aws_region_name", + "aws_service_name", + "aws_access_key_id", + "aws_secret_access_key", + "aws_session_token", + "aws_role_name", + "aws_session_name", + ]), + ); + expect(editCreds(HTTP_NONE)).not.toContain("aws_region_name"); + }); + + it("mounts the id-jag credential set only for oauth2_id_jag", () => { + const creds = editCreds({ transport: "http", auth_type: "oauth2_id_jag" }); + expect(creds).toContain("id_jag_resource_token_endpoint"); + expect(creds).toContain("client_private_key"); + expect(creds).toContain("client_assertion_signing_alg"); + expect(editCreds(HTTP_NONE)).not.toContain("id_jag_resource_token_endpoint"); + }); +}); + +describe("edit root: children that gate by early return null", () => { + it("mounts dcr_bridge and the declared-app credentials only for the client-forwarded modes", () => { + for (const authType of ["true_passthrough", "oauth_delegate"]) { + expect(editRoot({ transport: "http", auth_type: authType })).toContain("dcr_bridge"); + expect(sorted(editCreds({ transport: "http", auth_type: authType }))).toStrictEqual( + sorted(["client_id", "client_secret"]), + ); + } + expect(editRoot(HTTP_NONE)).not.toContain("dcr_bridge"); + expect(editRoot({ transport: "http", auth_type: "oauth2" })).not.toContain("dcr_bridge"); + }); + + it("unmounts dcr_bridge with its parent section on stdio", () => { + expect(editRoot({ transport: "stdio", auth_type: "true_passthrough" })).not.toContain("dcr_bridge"); + }); +}); + +describe("edit root: permission-section gates", () => { + it("mounts delegate_auth_to_upstream only for oauth2", () => { + expect(editRoot({ transport: "http", auth_type: "oauth2" })).toContain("delegate_auth_to_upstream"); + expect(editRoot(HTTP_NONE)).not.toContain("delegate_auth_to_upstream"); + expect(editRoot({ transport: "http", auth_type: "api_key" })).not.toContain("delegate_auth_to_upstream"); + }); + + it("mounts oauth_passthrough only for none-auth WITH an Authorization extra header", () => { + expect(editRoot({ ...HTTP_NONE, extra_headers: ["Authorization"] })).toContain("oauth_passthrough"); + expect(editRoot({ ...HTTP_NONE, extra_headers: ["authorization"] })).toContain("oauth_passthrough"); + expect(editRoot({ ...HTTP_NONE, extra_headers: ["X-Other"] })).not.toContain("oauth_passthrough"); + expect(editRoot(HTTP_NONE)).not.toContain("oauth_passthrough"); + expect(editRoot({ transport: "http", auth_type: "oauth2", extra_headers: ["Authorization"] })).not.toContain( + "oauth_passthrough", + ); + }); + + it("treats an absent auth_type as none-auth for the oauth_passthrough gate", () => { + expect(editRoot({ transport: "http", extra_headers: ["Authorization"] })).toContain("oauth_passthrough"); + }); +}); + +describe("create root: where it diverges from edit", () => { + it("mounts source_url, which the edit root has no binding for", () => { + expect(createRoot({ transport: "http", auth_type: "none" })).toContain("source_url"); + expect(editRoot(HTTP_NONE)).not.toContain("source_url"); + }); + + it("gates url on an allow-list, so a blank transport mounts NEITHER url nor auth_type", () => { + const blank = createRoot({ transport: "" }); + expect(blank).not.toContain("url"); + expect(blank).not.toContain("auth_type"); + expect(editRoot({ transport: "" })).toContain("url"); + expect(editRoot({ transport: "" })).toContain("auth_type"); + }); + + it("mounts stdio_config on stdio but never the edit root's command/args/env_json", () => { + const root = createRoot({ transport: "stdio" }); + expect(root).toContain("stdio_config"); + expect(root).not.toContain("command"); + expect(root).not.toContain("args"); + expect(root).not.toContain("env_json"); + }); + + it("mounts the byok fields only for openapi with is_byok on", () => { + expect(createRoot({ transport: "openapi", auth_type: "none" })).toContain("is_byok"); + expect(createRoot({ transport: "openapi", auth_type: "none" })).not.toContain("byok_description"); + const on = createRoot({ transport: "openapi", auth_type: "none", is_byok: true }); + expect(on).toContain("byok_description"); + expect(on).toContain("byok_api_key_help_url"); + expect(createRoot({ transport: "http", auth_type: "none", is_byok: true })).not.toContain("byok_description"); + }); + + it("drops every credential while the transport is unset", () => { + expect(createCreds({ transport: "", auth_type: "aws_sigv4" })).toStrictEqual([]); + expect(createCreds({ transport: "http", auth_type: "aws_sigv4" })).toContain("aws_region_name"); + }); +}); + +const ALWAYS = ["server_name", "alias", "description", "transport", "max_concurrent_requests"]; +const PERMS = [ + "allow_all_keys", + "available_on_public_internet", + "mcp_access_groups", + "extra_headers", + "static_headers", +]; +const sorted = (xs: readonly string[]) => [...xs].sort(); + +const expectEditSets = ( + values: Record, + expected: { root: readonly string[]; credentials: readonly string[] }, +) => { + expect(sorted(editRoot(values))).toStrictEqual(sorted(expected.root)); + expect(sorted(editCreds(values))).toStrictEqual(sorted(expected.credentials)); +}; + +const expectCreateSets = ( + values: Record, + expected: { root: readonly string[]; credentials: readonly string[] }, +) => { + expect(sorted(createRoot(values))).toStrictEqual(sorted(expected.root)); + expect(sorted(createCreds(values))).toStrictEqual(sorted(expected.credentials)); +}; + +describe("edit root: exact mounted set per auth configuration", () => { + it("http + none", () => { + expectEditSets(HTTP_NONE, { root: [...ALWAYS, "url", "auth_type", "env_vars", ...PERMS], credentials: [] }); + }); + + it("http + api_key", () => { + expectEditSets( + { transport: "http", auth_type: "api_key" }, + { root: [...ALWAYS, "url", "auth_type", "env_vars", ...PERMS], credentials: ["auth_value"] }, + ); + }); + + it("http + oauth2 M2M", () => { + expectEditSets( + { transport: "http", auth_type: "oauth2", oauth_flow_type: "m2m" }, + { + root: [ + ...ALWAYS, + "url", + "auth_type", + "oauth_flow_type", + "token_url", + "env_vars", + ...PERMS, + "delegate_auth_to_upstream", + ], + credentials: ["client_id", "client_secret", "token_endpoint_auth_method", "scopes", "upstream_resource"], + }, + ); + }); + + it("http + oauth2 interactive", () => { + expectEditSets( + { transport: "http", auth_type: "oauth2", oauth_flow_type: "interactive" }, + { + root: [ + ...ALWAYS, + "url", + "auth_type", + "oauth_flow_type", + "issuer", + "authorization_url", + "token_url", + "registration_url", + "token_validation_json", + "token_storage_ttl_seconds", + "env_vars", + ...PERMS, + "delegate_auth_to_upstream", + ], + credentials: ["client_id", "client_secret", "scopes", "upstream_resource", "token_endpoint_auth_method"], + }, + ); + }); + + it("http + token exchange, rfc8693", () => { + expectEditSets( + { transport: "http", auth_type: "oauth2_token_exchange", token_exchange_profile: "rfc8693" }, + { + root: [ + ...ALWAYS, + "url", + "auth_type", + "token_exchange_profile", + "token_exchange_endpoint", + "audience", + "subject_token_type", + "env_vars", + ...PERMS, + ], + credentials: ["client_id", "client_secret", "scopes"], + }, + ); + }); + + it("http + token exchange, entra_obo keeps the endpoint while dropping audience and subject_token_type", () => { + expectEditSets( + { transport: "http", auth_type: "oauth2_token_exchange", token_exchange_profile: "entra_obo" }, + { + root: [ + ...ALWAYS, + "url", + "auth_type", + "token_exchange_profile", + "token_exchange_endpoint", + "env_vars", + ...PERMS, + ], + credentials: ["client_id", "client_secret", "scopes"], + }, + ); + }); + + it("http + id-jag", () => { + expectEditSets( + { transport: "http", auth_type: "oauth2_id_jag" }, + { + root: [ + ...ALWAYS, + "url", + "auth_type", + "token_exchange_endpoint", + "audience", + "subject_token_type", + "env_vars", + ...PERMS, + ], + credentials: [ + "id_jag_resource_token_endpoint", + "client_id", + "client_secret", + "client_private_key", + "client_private_key_id", + "client_assertion_signing_alg", + "id_jag_resource", + "scopes", + ], + }, + ); + }); + + it("http + true_passthrough", () => { + expectEditSets( + { transport: "http", auth_type: "true_passthrough" }, + { + root: [...ALWAYS, "url", "auth_type", "dcr_bridge", "env_vars", ...PERMS], + credentials: ["client_id", "client_secret"], + }, + ); + }); + + it("openapi + none", () => { + expectEditSets( + { transport: "openapi", auth_type: "none" }, + { root: [...ALWAYS, "spec_path", "auth_type", "env_vars", ...PERMS], credentials: [] }, + ); + }); +}); + +describe("create root: exact mounted set per configuration", () => { + it("http + none", () => { + expectCreateSets( + { transport: "http", auth_type: "none" }, + { root: [...ALWAYS, "source_url", "url", "auth_type", "env_vars", ...PERMS], credentials: [] }, + ); + }); + + it("openapi with byok on", () => { + expectCreateSets( + { transport: "openapi", auth_type: "none", is_byok: true }, + { + root: [ + ...ALWAYS, + "source_url", + "spec_path", + "is_byok", + "byok_description", + "byok_api_key_help_url", + "auth_type", + "env_vars", + ...PERMS, + ], + credentials: [], + }, + ); + }); + + it("stdio", () => { + expectCreateSets( + { transport: "stdio" }, + { root: [...ALWAYS, "source_url", "stdio_config", "env_vars", ...PERMS], credentials: [] }, + ); + }); + + it("transport still unset", () => { + expectCreateSets({ transport: "" }, { root: [...ALWAYS, "source_url", "env_vars", ...PERMS], credentials: [] }); + }); + + it("http + oauth2 interactive", () => { + expectCreateSets( + { transport: "http", auth_type: "oauth2", oauth_flow_type: "interactive" }, + { + root: [ + ...ALWAYS, + "source_url", + "url", + "auth_type", + "oauth_flow_type", + "issuer", + "authorization_url", + "token_url", + "registration_url", + "token_validation_json", + "token_storage_ttl_seconds", + "env_vars", + ...PERMS, + "delegate_auth_to_upstream", + ], + credentials: ["client_id", "client_secret", "scopes", "upstream_resource", "token_endpoint_auth_method"], + }, + ); + }); + + it("http + oauth_delegate mounts dcr_bridge and the declared app", () => { + expectCreateSets( + { transport: "http", auth_type: "oauth_delegate" }, + { + root: [...ALWAYS, "source_url", "url", "auth_type", "dcr_bridge", "env_vars", ...PERMS], + credentials: ["client_id", "client_secret"], + }, + ); + }); +}); + +describe("projection shape", () => { + it("EMITS a mounted-but-unset field as a key holding undefined, matching antd onFinish", () => { + const projected = projectMountedEditValues({ transport: "http", auth_type: "none", server_name: "s" }); + expect("description" in projected).toBe(true); + expect(projected.description).toBeUndefined(); + expect(Object.keys(projected)).toContain("max_concurrent_requests"); + }); + + it("emits mounted-but-unset CREDENTIAL keys as undefined rather than omitting them", () => { + const projected = projectMountedEditValues({ transport: "http", auth_type: "api_key" }); + expect(Object.keys(projected.credentials as object)).toStrictEqual(["auth_value"]); + expect((projected.credentials as Record).auth_value).toBeUndefined(); + }); + + it("omits the credentials key entirely when no credential field is mounted", () => { + expect("credentials" in projectMountedEditValues(HTTP_NONE)).toBe(false); + }); + + it("drops an unmounted field even when the store still holds a value for it", () => { + const storeWithStaleHttpValues = { + transport: "stdio", + auth_type: "oauth2", + url: "https://kept-in-store.example", + issuer: "https://kept-in-store.example", + command: "npx", + }; + const projected = projectMountedEditValues(storeWithStaleHttpValues); + expect("url" in projected).toBe(false); + expect("issuer" in projected).toBe(false); + expect(projected.command).toBe("npx"); + }); + + it("passes list rows through whole, since a list field is projected as one key and not per mounted sub-field", () => { + const row = { name: "N", value: "V", scope: "user", description: "D" }; + const projected = projectMountedEditValues({ ...HTTP_NONE, env_vars: [row] }); + expect(projected.env_vars).toStrictEqual([row]); + }); + + it("keeps static_headers rows whole", () => { + const rows = [{ header: "X-A", value: "1" }]; + expect( + projectMountedCreateValues({ transport: "http", auth_type: "none", static_headers: rows }).static_headers, + ).toStrictEqual(rows); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mountedServerFields.ts b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mountedServerFields.ts new file mode 100644 index 00000000000..22f0146afc9 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mountedServerFields.ts @@ -0,0 +1,195 @@ +import { AUTH_TYPE, OAUTH_FLOW, TRANSPORT, isClientForwardedTokenMode } from "@/components/mcp_tools/types"; +import { AUTH_TYPES_REQUIRING_AUTH_VALUE } from "./createServerPayload"; + +export interface MountedFieldNames { + readonly root: readonly string[]; + readonly credentials: readonly string[]; +} + +const ENTRA_OBO_PROFILE = "entra_obo"; + +const ALWAYS_MOUNTED_ROOT = ["server_name", "alias", "description", "transport", "max_concurrent_requests"] as const; + +const PERMISSION_SECTION_ROOT = [ + "allow_all_keys", + "available_on_public_internet", + "mcp_access_groups", + "extra_headers", + "static_headers", +] as const; + +const OAUTH_M2M_CREDENTIALS = [ + "client_id", + "client_secret", + "token_endpoint_auth_method", + "scopes", + "upstream_resource", +] as const; + +const OAUTH_INTERACTIVE_CREDENTIALS = [ + "client_id", + "client_secret", + "scopes", + "upstream_resource", + "token_endpoint_auth_method", +] as const; + +const OAUTH_INTERACTIVE_ROOT = [ + "issuer", + "authorization_url", + "token_url", + "registration_url", + "token_validation_json", + "token_storage_ttl_seconds", +] as const; + +const ID_JAG_CREDENTIALS = [ + "id_jag_resource_token_endpoint", + "client_id", + "client_secret", + "client_private_key", + "client_private_key_id", + "client_assertion_signing_alg", + "id_jag_resource", + "scopes", +] as const; + +const AWS_SIGV4_CREDENTIALS = [ + "aws_region_name", + "aws_service_name", + "aws_access_key_id", + "aws_secret_access_key", + "aws_session_token", + "aws_role_name", + "aws_session_name", +] as const; + +const hasAuthorizationExtraHeader = (extraHeaders: unknown): boolean => + Array.isArray(extraHeaders) && extraHeaders.some((h) => typeof h === "string" && h.toLowerCase() === "authorization"); + +interface AuthSubtreeGates { + readonly authType: string | undefined; + readonly oauthFlowType: string | undefined; + readonly tokenExchangeProfile: string | undefined; +} + +const authSubtreeRoot = ({ authType, oauthFlowType, tokenExchangeProfile }: AuthSubtreeGates): readonly string[] => { + if (authType === AUTH_TYPE.OAUTH2) { + return oauthFlowType === OAUTH_FLOW.M2M + ? ["oauth_flow_type", "token_url"] + : ["oauth_flow_type", ...OAUTH_INTERACTIVE_ROOT]; + } + if (authType === AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE) { + return tokenExchangeProfile === ENTRA_OBO_PROFILE + ? ["token_exchange_profile", "token_exchange_endpoint"] + : ["token_exchange_profile", "token_exchange_endpoint", "audience", "subject_token_type"]; + } + if (authType === AUTH_TYPE.OAUTH2_ID_JAG) { + return ["token_exchange_endpoint", "audience", "subject_token_type"]; + } + return []; +}; + +const authSubtreeCredentials = ({ authType, oauthFlowType }: AuthSubtreeGates): readonly string[] => { + const authValue = AUTH_TYPES_REQUIRING_AUTH_VALUE.includes(authType as string) ? ["auth_value"] : []; + const clientForwarded = isClientForwardedTokenMode(authType) ? ["client_id", "client_secret"] : []; + if (authType === AUTH_TYPE.OAUTH2) { + return [ + ...authValue, + ...(oauthFlowType === OAUTH_FLOW.M2M ? OAUTH_M2M_CREDENTIALS : OAUTH_INTERACTIVE_CREDENTIALS), + ]; + } + if (authType === AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE) { + return [...authValue, "client_id", "client_secret", "scopes"]; + } + if (authType === AUTH_TYPE.OAUTH2_ID_JAG) { + return [...authValue, ...ID_JAG_CREDENTIALS]; + } + if (authType === AUTH_TYPE.AWS_SIGV4) { + return [...authValue, ...AWS_SIGV4_CREDENTIALS]; + } + return [...authValue, ...clientForwarded]; +}; + +const permissionSectionRoot = (authType: string | undefined, extraHeaders: unknown): readonly string[] => { + const isNoneAuth = authType === AUTH_TYPE.NONE || authType == null; + return [ + ...PERMISSION_SECTION_ROOT, + ...(authType === AUTH_TYPE.OAUTH2 ? ["delegate_auth_to_upstream"] : []), + ...(isNoneAuth && hasAuthorizationExtraHeader(extraHeaders) ? ["oauth_passthrough"] : []), + ]; +}; + +const dedupe = (names: readonly string[]): readonly string[] => Array.from(new Set(names)); + +export const mountedEditFieldNames = (values: Record): MountedFieldNames => { + const transport = values.transport as string | undefined; + const isStdio = transport === "stdio"; + const isOpenApi = transport === TRANSPORT.OPENAPI; + const isMcp = !isStdio && !isOpenApi; + const gates: AuthSubtreeGates = { + authType: isStdio ? undefined : (values.auth_type as string | undefined), + oauthFlowType: values.oauth_flow_type as string | undefined, + tokenExchangeProfile: values.token_exchange_profile as string | undefined, + }; + + return { + root: dedupe([ + ...ALWAYS_MOUNTED_ROOT, + ...(isMcp ? ["url"] : []), + ...(isOpenApi ? ["spec_path"] : []), + ...(isStdio ? ["command", "args", "env_json", "stdio_config"] : ["auth_type"]), + ...(isStdio ? [] : authSubtreeRoot(gates)), + ...(!isStdio && isClientForwardedTokenMode(gates.authType) ? ["dcr_bridge"] : []), + "env_vars", + ...permissionSectionRoot(gates.authType, values.extra_headers), + ]), + credentials: isStdio ? [] : dedupe(authSubtreeCredentials(gates)), + }; +}; + +export const mountedCreateFieldNames = (values: Record): MountedFieldNames => { + const transport = values.transport as string | undefined; + const isStdio = transport === "stdio"; + const isOpenApi = transport === TRANSPORT.OPENAPI; + const authSectionMounted = !isStdio && transport !== "" && transport !== undefined; + const gates: AuthSubtreeGates = { + authType: authSectionMounted ? (values.auth_type as string | undefined) : undefined, + oauthFlowType: values.oauth_flow_type as string | undefined, + tokenExchangeProfile: values.token_exchange_profile as string | undefined, + }; + + return { + root: dedupe([ + ...ALWAYS_MOUNTED_ROOT, + "source_url", + ...(transport === "http" || transport === "sse" ? ["url"] : []), + ...(isOpenApi ? ["spec_path", "is_byok"] : []), + ...(isOpenApi && values.is_byok ? ["byok_description", "byok_api_key_help_url"] : []), + ...(authSectionMounted ? ["auth_type"] : []), + ...(authSectionMounted ? authSubtreeRoot(gates) : []), + ...(authSectionMounted && isClientForwardedTokenMode(gates.authType) ? ["dcr_bridge"] : []), + ...(isStdio ? ["stdio_config"] : []), + "env_vars", + ...permissionSectionRoot(gates.authType, values.extra_headers), + ]), + credentials: authSectionMounted ? dedupe(authSubtreeCredentials(gates)) : [], + }; +}; + +const pickEmitting = (source: Record | undefined, names: readonly string[]): Record => + Object.fromEntries(names.map((name) => [name, source?.[name]])); + +const projectWith = + (namesOf: (values: Record) => MountedFieldNames) => + (values: Record): Record => { + const names = namesOf(values); + const credentials = values.credentials as Record | undefined; + return { + ...pickEmitting(values, names.root), + ...(names.credentials.length > 0 ? { credentials: pickEmitting(credentials, names.credentials) } : {}), + }; + }; + +export const projectMountedEditValues = projectWith(mountedEditFieldNames); +export const projectMountedCreateValues = projectWith(mountedCreateFieldNames); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/testUtils.ts b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/testUtils.ts index 7911c3eb800..c1692f436d7 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/testUtils.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/testUtils.ts @@ -4,6 +4,7 @@ import { expect } from "vitest"; export async function selectAntOption(labelText: string, optionText: string) { const label = screen.getByText(labelText); const select = + label.closest("[data-slot='field']")?.querySelector(".ant-select") ?? label.closest(".ant-form-item")?.querySelector(".ant-select") ?? label.closest(".ant-collapse-item")?.querySelector(".ant-select") ?? label.closest("div")?.querySelector(".ant-select") ?? diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.test.tsx index 282cc2722db..8b34d61ebad 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.test.tsx @@ -3,11 +3,6 @@ import { render } from "@testing-library/react"; import { describe, expect, it, vi } from "vitest"; import PriceDataManagementTab from "./PriceDataManagementTab"; -// Deliberately do NOT mock @tremor/react. These tab components render standalone -// (inside antd Tabs / directly as a route page), no longer inside a Tremor -// . A Tremor root renders nothing without that context, so -// this asserts the component's content is visible on its own — reverting the root -// back to makes the title disappear and fails this test. vi.mock("@/components/price_data_reload", () => ({ default: () =>
reload
})); vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: () => ({ accessToken: "sk-test" }) })); vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({ @@ -15,7 +10,7 @@ vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({ })); describe("PriceDataManagementTab", () => { - it("renders its content standalone, without a Tremor TabGroup ancestor", () => { + it("renders its content standalone, without a tab-panel ancestor", () => { const { getByText } = render(); expect(getByText("Price Data Management")).toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx new file mode 100644 index 00000000000..b762f006261 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx @@ -0,0 +1,424 @@ +import { renderWithProviders, screen, waitFor } from "../../../../../tests/test-utils"; +import userEvent, { PointerEventsCheckLevel } from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import AddModelPanel from "./AddModelPanel"; + +const modelCreateCall = vi.fn(); +const mockPtuEnabled = vi.fn(); +const mockAuthorized = vi.fn(); + +vi.mock("@/components/networking", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + modelCreateCall: (accessToken: string, model: unknown) => modelCreateCall(accessToken, model), + modelAvailableCall: vi.fn().mockResolvedValue({ data: [{ id: "group-a" }] }), + }; +}); + +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: () => mockAuthorized() })); + +vi.mock("@/app/(dashboard)/hooks/uiSettings/usePtuCostAttributionEnabled", () => ({ + usePtuCostAttributionEnabled: () => mockPtuEnabled(), +})); + +vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({ useModelCostMap: () => ({ data: {} }) })); + +vi.mock("@/app/(dashboard)/hooks/credentials/useCredentials", () => ({ + useCredentials: () => ({ data: { credentials: [] } }), +})); + +vi.mock("@/app/(dashboard)/hooks/teams/useTeams", () => ({ + useTeams: () => ({ data: [] }), + useInfiniteTeams: () => ({ + data: { pages: [{ teams: [], total: 0, page: 1, page_size: 20, total_pages: 1 }] }, + fetchNextPage: vi.fn(), + hasNextPage: false, + isFetchingNextPage: false, + isLoading: false, + }), +})); + +vi.mock("@/app/(dashboard)/hooks/guardrails/useGuardrails", () => ({ + useGuardrails: () => ({ data: { guardrails: [{ guardrail_name: "g-1" }] }, isLoading: false, error: null }), +})); + +vi.mock("@/app/(dashboard)/hooks/tags/useTags", () => ({ + useTags: () => ({ data: {}, isLoading: false, error: null }), +})); + +vi.mock("@/app/(dashboard)/hooks/providers/useProviderFields", () => ({ + useProviderFields: () => ({ + data: [ + { + provider: "OpenAI", + provider_display_name: "OpenAI", + litellm_provider: "openai", + default_model_placeholder: "gpt-4o", + credential_fields: [ + { key: "api_key", label: "API Key", field_type: "password", required: false }, + { key: "api_base", label: "API Base", field_type: "text", required: false }, + ], + }, + ], + isLoading: false, + error: null, + }), +})); + +vi.mock("@/components/vector_store_management/VectorStoreSelector", () => ({ + default: () =>
, +})); + +const lastCreatedModel = () => modelCreateCall.mock.calls.at(-1)?.[1]; + +const PROXY_ADMIN = { + token: "t", + accessToken: "test-access-token", + userId: "user-1", + userEmail: "a@b.c", + userRole: "proxy_admin", + premiumUser: true, + disabledPersonalKeyCreation: false, + showSSOBanner: false, +}; + +const alwaysMounted = { + api_key: undefined, + api_base: undefined, + custom_llm_provider: "openai", + litellm_credential_name: null, + model: "gpt-4o", +}; + +const advancedOpenExtras = { + guardrails: undefined, + tags: undefined, + use_in_pass_through: undefined, + vector_store_ids: undefined, +}; + +const baseModelInfo = { access_groups: undefined, mode: undefined }; + +const { api_base: _omitted, ...ALWAYS_MOUNTED_WITHOUT_API_BASE } = alwaysMounted; + +const setup = async () => { + const user = userEvent.setup({ pointerEventsCheck: PointerEventsCheckLevel.Never }); + renderWithProviders(); + await screen.findByText("Provider"); + + const openAdvanced = async () => { + await user.click(screen.getByText("Advanced Settings")); + await screen.findByText("Tags"); + }; + + const closeAdvanced = async () => { + await user.click(screen.getByText("Advanced Settings")); + await waitFor(() => expect(screen.queryByText("Tags")).not.toBeInTheDocument()); + }; + + const fillRequired = async (modelName = "gpt-4o") => { + await user.click(screen.getByRole("combobox", { name: /provider/i })); + await user.click(await screen.findByText("OpenAI")); + await user.type(await screen.findByPlaceholderText("gpt-3.5-turbo"), modelName); + }; + + const submit = async () => { + await user.click(screen.getByTestId("add-model-btn")); + await waitFor(() => expect(modelCreateCall).toHaveBeenCalled()); + }; + + const submitExpectingRejection = async (message: string) => { + await user.click(screen.getByTestId("add-model-btn")); + await screen.findByText(message); + }; + + return { user, openAdvanced, closeAdvanced, fillRequired, submit, submitExpectingRejection }; +}; + +describe("AddModelPanel submit payload contract", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockPtuEnabled.mockReturnValue(false); + mockAuthorized.mockReturnValue(PROXY_ADMIN); + }); + + it("sends only the always-mounted fields while Advanced Settings stays closed", async () => { + const { fillRequired, submit } = await setup(); + await fillRequired(); + await submit(); + + expect(lastCreatedModel()).toStrictEqual({ + model_name: "gpt-4o", + litellm_params: { ...alwaysMounted }, + model_info: { ...baseModelInfo }, + }); + }); + + it("registers four more keys as undefined once Advanced Settings opens", async () => { + const { openAdvanced, fillRequired, submit } = await setup(); + await fillRequired(); + await openAdvanced(); + await submit(); + + expect(lastCreatedModel()).toStrictEqual({ + model_name: "gpt-4o", + litellm_params: { ...alwaysMounted, ...advancedOpenExtras }, + model_info: { ...baseModelInfo }, + }); + }); + + it("merges typed LiteLLM Params into litellm_params", async () => { + const { user, openAdvanced, fillRequired, submit } = await setup(); + await fillRequired(); + await openAdvanced(); + await user.type(screen.getByLabelText("LiteLLM Params"), '{{"rpm": 7}'); + await submit(); + + expect(lastCreatedModel()).toStrictEqual({ + model_name: "gpt-4o", + litellm_params: { ...alwaysMounted, ...advancedOpenExtras, rpm: 7 }, + model_info: { ...baseModelInfo }, + }); + }); + + it("drops a collapsed section's keys and the value typed into it", async () => { + const { user, openAdvanced, closeAdvanced, fillRequired, submit } = await setup(); + await fillRequired(); + await openAdvanced(); + await user.type(screen.getByLabelText("LiteLLM Params"), '{{"rpm": 7}'); + await closeAdvanced(); + await submit(); + + expect(lastCreatedModel()).toStrictEqual({ + model_name: "gpt-4o", + litellm_params: { ...alwaysMounted }, + model_info: { ...baseModelInfo }, + }); + }); + + it("restores the typed value when the section is expanded again", async () => { + const { user, openAdvanced, closeAdvanced, fillRequired, submit } = await setup(); + await fillRequired(); + await openAdvanced(); + await user.type(screen.getByLabelText("LiteLLM Params"), '{{"rpm": 7}'); + await closeAdvanced(); + await openAdvanced(); + expect(screen.getByLabelText("LiteLLM Params")).toHaveValue('{"rpm": 7}'); + + await submit(); + + expect(lastCreatedModel()).toStrictEqual({ + model_name: "gpt-4o", + litellm_params: { ...alwaysMounted, ...advancedOpenExtras, rpm: 7 }, + model_info: { ...baseModelInfo }, + }); + }); + + it("converts per-million pricing to per-token and falls back to input cost for cache reads", async () => { + const { user, openAdvanced, fillRequired, submit } = await setup(); + await fillRequired(); + await openAdvanced(); + await user.click(screen.getByLabelText("Custom Pricing")); + await user.type(await screen.findByLabelText("Input Cost (per 1M tokens)"), "3"); + await user.type(screen.getByLabelText("Output Cost (per 1M tokens)"), "9"); + await submit(); + + expect(lastCreatedModel()).toStrictEqual({ + model_name: "gpt-4o", + litellm_params: { + ...alwaysMounted, + ...advancedOpenExtras, + input_cost_per_token: 0.000003, + output_cost_per_token: 0.000009, + cache_read_input_token_cost: 0.000003, + }, + model_info: { ...baseModelInfo }, + }); + }); + + it("sends the seeded injection point when cache control is switched on", async () => { + const { user, openAdvanced, fillRequired, submit } = await setup(); + await fillRequired(); + await openAdvanced(); + await user.click(screen.getByLabelText("Cache Control Injection Points")); + await screen.findByText("Add Injection Point"); + await submit(); + + expect(lastCreatedModel()).toStrictEqual({ + model_name: "gpt-4o", + litellm_params: { + ...alwaysMounted, + ...advancedOpenExtras, + cache_control_injection_points: [{ location: "message" }], + }, + model_info: { ...baseModelInfo }, + }); + }); + + it("carries a role picked inside the injection point editor, with the index kept a string", async () => { + const { user, openAdvanced, fillRequired, submit } = await setup(); + await fillRequired(); + await openAdvanced(); + await user.click(screen.getByLabelText("Cache Control Injection Points")); + await screen.findByText("Add Injection Point"); + await user.click(screen.getByText("Select a role")); + await user.click(await screen.findByText("System")); + await user.type(screen.getByPlaceholderText("Optional"), "3"); + await submit(); + + expect(lastCreatedModel()).toStrictEqual({ + model_name: "gpt-4o", + litellm_params: { + ...alwaysMounted, + ...advancedOpenExtras, + cache_control_injection_points: [{ location: "message", role: "system", index: "3" }], + }, + model_info: { ...baseModelInfo }, + }); + }); + + it("mounts team_id only once the Team-BYOK switch is on", async () => { + const { user, fillRequired, submit } = await setup(); + await fillRequired(); + await user.click(screen.getByRole("switch", { name: "Team-BYOK Model" })); + await screen.findByText("Select Team"); + await submit(); + + expect(lastCreatedModel()).toStrictEqual({ + model_name: "gpt-4o", + litellm_params: { ...alwaysMounted }, + model_info: { ...baseModelInfo, team_id: undefined }, + }); + }); +}); + +describe("AddModelPanel empty-string skip", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockPtuEnabled.mockReturnValue(false); + mockAuthorized.mockReturnValue(PROXY_ADMIN); + }); + + it("sends a typed api_base, so the binding behind the next case is known to be live", async () => { + const { user, fillRequired, submit } = await setup(); + await fillRequired(); + await user.type(screen.getByLabelText("API Base"), "https://example.test"); + await submit(); + + expect(lastCreatedModel().litellm_params).toStrictEqual({ + ...alwaysMounted, + api_base: "https://example.test", + }); + }); + + it("omits api_base entirely once it is cleared, rather than sending an empty string", async () => { + const { user, fillRequired, submit } = await setup(); + await fillRequired(); + const apiBase = screen.getByLabelText("API Base"); + await user.type(apiBase, "https://example.test"); + await user.clear(apiBase); + await submit(); + + const params = lastCreatedModel().litellm_params; + expect(params).not.toHaveProperty("api_base"); + expect(params).toStrictEqual(ALWAYS_MOUNTED_WITHOUT_API_BASE); + }); +}); + +describe("AddModelPanel validation gates", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockPtuEnabled.mockReturnValue(true); + mockAuthorized.mockReturnValue(PROXY_ADMIN); + }); + + it("blocks the submit when a PTU count carries no effective-from date", async () => { + const { user, openAdvanced, fillRequired, submitExpectingRejection } = await setup(); + await fillRequired(); + await openAdvanced(); + await user.type(screen.getByLabelText("PTU Count"), "15"); + await user.type(screen.getByLabelText("Calculated Cost per PTU / Hour (USD)"), "2"); + await submitExpectingRejection("PTU Effective From is required when PTU Count is set"); + + expect(modelCreateCall).not.toHaveBeenCalled(); + }); + + it("hides the PTU fields entirely when the capability is off", async () => { + mockPtuEnabled.mockReturnValue(false); + const { openAdvanced, fillRequired } = await setup(); + await fillRequired(); + await openAdvanced(); + + expect(screen.queryByLabelText("PTU Count")).not.toBeInTheDocument(); + }); + + it("requires a model before anything is sent", async () => { + const { user, submitExpectingRejection } = await setup(); + await user.click(screen.getByRole("combobox", { name: /provider/i })); + await user.click(await screen.findByText("OpenAI")); + await submitExpectingRejection("Please enter at least one model."); + + expect(modelCreateCall).not.toHaveBeenCalled(); + }); + + it("blocks the submit when LiteLLM Params is not valid JSON", async () => { + mockPtuEnabled.mockReturnValue(false); + const { user, openAdvanced, fillRequired, submitExpectingRejection } = await setup(); + await fillRequired(); + await openAdvanced(); + await user.type(screen.getByLabelText("LiteLLM Params"), "rpm: 7"); + await submitExpectingRejection("Please enter valid JSON"); + + expect(modelCreateCall).not.toHaveBeenCalled(); + }); +}); + +describe("AddModelPanel behaviours the removed Advanced Settings form instance never drove", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockPtuEnabled.mockReturnValue(false); + mockAuthorized.mockReturnValue(PROXY_ADMIN); + }); + + it("leaves LiteLLM Params untouched when pass through routes is switched on", async () => { + const { user, openAdvanced, fillRequired, submit } = await setup(); + await fillRequired(); + await openAdvanced(); + await user.click(screen.getByLabelText("Use in pass through routes")); + expect(screen.getByLabelText("LiteLLM Params")).toHaveValue(""); + + await submit(); + + expect(lastCreatedModel()).toStrictEqual({ + model_name: "gpt-4o", + litellm_params: { ...alwaysMounted, ...advancedOpenExtras, use_in_pass_through: true }, + model_info: { ...baseModelInfo }, + }); + }); + + it("keeps a typed cost when custom pricing is switched off and back on", async () => { + const { user, openAdvanced, fillRequired, submit } = await setup(); + await fillRequired(); + await openAdvanced(); + await user.click(screen.getByLabelText("Custom Pricing")); + await user.type(await screen.findByLabelText("Input Cost (per 1M tokens)"), "3"); + await user.click(screen.getByLabelText("Custom Pricing")); + await waitFor(() => expect(screen.queryByLabelText("Input Cost (per 1M tokens)")).not.toBeInTheDocument()); + await user.click(screen.getByLabelText("Custom Pricing")); + expect(await screen.findByLabelText("Input Cost (per 1M tokens)")).toHaveValue("3"); + + await submit(); + + expect(lastCreatedModel()).toStrictEqual({ + model_name: "gpt-4o", + litellm_params: { + ...alwaysMounted, + ...advancedOpenExtras, + input_cost_per_token: 0.000003, + cache_read_input_token_cost: 0.000003, + }, + model_info: { ...baseModelInfo }, + }); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx index 35be19531d5..59d4f95c038 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx @@ -1,21 +1,28 @@ "use client"; -import { Form } from "antd"; import { useState } from "react"; +import { useForm } from "react-hook-form"; import { useQueryClient } from "@tanstack/react-query"; import AddModelForm from "@/components/add_model/AddModelForm"; import { handleAddModelSubmit } from "@/components/add_model/handle_add_model_submit"; +import { + projectMountedValues, + useMountRegistry, + type MountedFormValues, +} from "@/components/common_components/MountedFormField"; import { Providers, getPlaceholder, getProviderModels } from "@/components/provider_info_helpers"; -import { toast } from "@/lib/toast"; import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap"; import { useCredentials } from "@/app/(dashboard)/hooks/credentials/useCredentials"; import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { vertexCredentialsUploadProps } from "@/app/(dashboard)/models-and-endpoints/vertexCredentialsUpload"; +const INITIAL_VALUES: MountedFormValues = { litellm_credential_name: null }; + export default function AddModelPanel() { const { accessToken } = useAuthorized(); - const [form] = Form.useForm(); + const form = useForm({ mode: "onChange", defaultValues: INITIAL_VALUES }); + const registry = useMountRegistry(); const queryClient = useQueryClient(); const { data: modelCostMapData } = useModelCostMap(); const { data: credentialsResponse } = useCredentials(); @@ -26,28 +33,36 @@ export default function AddModelPanel() { const refresh = () => queryClient.invalidateQueries({ queryKey: ["models", "list"] }); - const handleOk = async () => { - try { - const values = await form.validateFields(); - await handleAddModelSubmit(values, accessToken, form, refresh); - } catch (error: any) { - const errorMessages = - error.errorFields?.map((field: any) => `${field.name.join(".")}: ${field.errors.join(", ")}`).join(" | ") || - "Unknown validation error"; - toast.fromError(`Please fill in the following required fields: ${errorMessages}`); + const mountedValues = () => projectMountedValues(registry, form.getValues); + + const handleOk = async (): Promise => { + const isValid = await form.trigger(registry.mountedNames() as string[]); + if (!isValid) { + return false; } + await handleAddModelSubmit( + mountedValues(), + accessToken, + { resetFields: () => form.reset(INITIAL_VALUES) }, + refresh, + ); + return true; }; return ( setProviderModels(getProviderModels(provider, modelCostMapData))} getPlaceholder={getPlaceholder} - uploadProps={vertexCredentialsUploadProps(form)} + uploadProps={vertexCredentialsUploadProps({ + setFieldsValue: (values) => form.setValue("vertex_credentials", values.vertex_credentials), + })} showAdvancedSettings={showAdvancedSettings} setShowAdvancedSettings={setShowAdvancedSettings} teams={teams ?? null} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel.tsx index 6112e6bffbd..7251da7c3c3 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/LlmCredentialsPanel.tsx @@ -1,10 +1,17 @@ "use client"; -import { Form } from "antd"; +import { useForm } from "react-hook-form"; import CredentialsPanel from "@/components/model_add/CredentialsPanel"; +import type { MountedFormValues } from "@/components/common_components/MountedFormField"; import { vertexCredentialsUploadProps } from "@/app/(dashboard)/models-and-endpoints/vertexCredentialsUpload"; export default function LlmCredentialsPanel() { - const [form] = Form.useForm(); - return ; + const form = useForm(); + return ( + form.setValue("vertex_credentials", values.vertex_credentials), + })} + /> + ); } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/impact_popover.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/impact_popover.test.tsx index 66f03577e20..69c471a7c3d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/impact_popover.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/impact_popover.test.tsx @@ -30,16 +30,6 @@ vi.mock("@heroicons/react/outline", () => ({ }, })); -vi.mock("@tremor/react", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - Icon: React.forwardRef(({ icon: _icon, ...props }, ref) => ( -