scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-13 23:11:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 101

Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_lit4354_resolved_guardrail_hook

Browse source
This commit is contained in:
Yucheng Zhu 2026-07-11 12:51:08 -07:00
commit aa2b640b0d
81 changed files with 5871 additions and 1201 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
.github/workflows/test-linting.yml vendored
View file

@ -48,7 +48,7 @@ jobs:
- name: Install dependencies
run: |
uv sync --frozen --group proxy-dev
uv sync --frozen --group proxy-dev --group e2e-dev
# basedpyright resolves Prisma's generated client (litellm/proxy/schema.prisma)
# only after `prisma generate` writes prisma/client.py et al. Without this the
@ -107,6 +107,16 @@ jobs:
run: |
(uv run --no-sync basedpyright --outputjson || true) | uv run --no-sync python scripts/type_check_gate.py --base "$BASE_SHA"
- name: Check tests/e2e basedpyright (zero errors)
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
if git diff --name-only --diff-filter=ACMRD "$BASE_SHA"...HEAD -- 'tests/e2e/**/*.py' | grep -q .; then
uv run --no-sync basedpyright tests/e2e
else
echo "No changed tests/e2e Python files; skipping."
fi
- name: Check for circular imports
run: |
cd litellm

13
Makefile
View file

@ -5,7 +5,7 @@
test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \
test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
info lint lint-dev lint-checks format \
lint-basedpyright lint-basedpyright-budget-update lint-type-discipline lint-type-discipline-budget-update \
lint-basedpyright lint-e2e-basedpyright lint-basedpyright-budget-update lint-type-discipline lint-type-discipline-budget-update \
lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-gate \
install-dev install-proxy-dev install-test-deps install-hooks \
install-helm-unittest check-circular-imports check-import-safety pre-commit \
@ -27,6 +27,7 @@ help:
@echo " make lint - Run all linting (Ruff, basedpyright, format check, circular imports, import safety)"
@echo " make lint-ruff - Run Ruff linting only"
@echo " make lint-basedpyright - Run basedpyright strict, gated by per-rule error counts"
@echo " make lint-e2e-basedpyright - Run basedpyright over tests/e2e (zero errors allowed)"
@echo " make lint-basedpyright-budget-update - Ratchet basedpyright limits down by what this branch fixed"
@echo " make lint-format - Check ruff format formatting (matches CI)"
@echo " make lint-ruff-budget - Gate the codebase total of each strict ruff rule against its limit"
@ -54,6 +55,7 @@ UV := uv
UV_RUN := $(UV) run --no-sync
LINT_DEP_INSTALL ?= install-dev
LINT_E2E_DEP_INSTALL ?= lint-install
LINT_DEP_BASE ?= lint-fetch-base
LINT_JOBS := $(shell sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 4)
LINT_OUTPUT_SYNC := $(if $(filter output-sync,$(.FEATURES)),--output-sync=target,)
@ -111,7 +113,7 @@ lint-fetch-base:
# CI's). --inexact tops up the venv instead of pruning the proxy extras gen:api and the
# running proxy need.
lint-install:
$(UV) sync --inexact --frozen --group proxy-dev
$(UV) sync --inexact --frozen --group proxy-dev --group e2e-dev
$(UV_RUN) python scripts/prisma_generate_if_needed.py
# Diff-scoped format check, identical to test-linting.yml's "Check ruff format" step:
@ -164,6 +166,9 @@ lint-ruff-FULL-dev: install-dev
lint-basedpyright: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --base origin/litellm_internal_staging
lint-e2e-basedpyright: $(LINT_E2E_DEP_INSTALL)
$(UV_RUN) basedpyright tests/e2e
# Type-discipline budget (mutable collections / casts / type guards / kwargs /
# unexplained suppressions), the test-linting.yml step `make lint` used to omit.
lint-type-discipline: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
@ -208,9 +213,9 @@ check-import-safety: $(LINT_DEP_INSTALL)
# base fetch) runs once up front; the checks themselves are independent, so a sub-make
# fans them out with -j and the fast ones finish under basedpyright's shadow.
lint: lint-install lint-fetch-base
$(MAKE) -j $(LINT_JOBS) $(LINT_OUTPUT_SYNC) LINT_DEP_INSTALL= LINT_DEP_BASE= lint-checks
$(MAKE) -j $(LINT_JOBS) $(LINT_OUTPUT_SYNC) LINT_DEP_INSTALL= LINT_E2E_DEP_INSTALL= LINT_DEP_BASE= lint-checks
lint-checks: lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-basedpyright check-circular-imports check-import-safety
lint-checks: lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-basedpyright lint-e2e-basedpyright check-circular-imports check-import-safety
# Faster linting for local development (only checks changed code)
lint-dev: lint-format-changed check-circular-imports check-import-safety

10
basedpyright-code-budget.json
View file

@ -57,7 +57,7 @@
"limit": 5900
},
"reportMissingTypeArgument": {
"limit": 15918
"limit": 15903
},
"reportMissingTypeStubs": {
"limit": 41
@ -105,13 +105,13 @@
"limit": 113
},
"reportUnknownMemberType": {
"limit": 40541
"limit": 40539
},
"reportUnknownParameterType": {
"limit": 20418
"limit": 20403
},
"reportUnknownVariableType": {
"limit": 32151
"limit": 32141
},
"reportUnnecessaryCast": {
"limit": 177
@ -123,7 +123,7 @@
"limit": 7
},
"reportUnnecessaryIsInstance": {
"limit": 1212
"limit": 1209
},
"reportUntypedBaseClass": {
"limit": 165

229
litellm/litellm_core_utils/fallback_generalizations.py
View file

@ -3,55 +3,69 @@ Declarative fallback generalizations for unknown / newly-released models.
The ``fallback_generalizations`` block in ``model_prices_and_context_window.json``
holds an ordered list of rules. Each rule pairs a single case-insensitive regex
with the metadata to apply when a model name has no exact entry in the cost map.
The metadata is a partial cost-map entry: ``litellm_provider`` drives provider
routing, and the remaining fields (``mode``, ``supports_*``, context window,
pricing, ...) drive ``get_model_info`` / ``supports_*``.
with a ``model_info`` dict, and the structure of ``model_info`` decides which of
two kinds the rule is.
Precedence: rules are evaluated in file order and the first match wins. Callers
with extra constraints (model-info resolution checks the provider) use
``match_all_fallback_generalizations`` to skip inapplicable earlier rules instead
of discarding the model name. Rules are consulted only after exact and
case-insensitive lookups miss, so an exact entry always takes precedence over a
rule.
A ROUTING rule carries exactly one ``model_info`` key, ``litellm_provider``. It is
consumed only by ``get_llm_provider`` bare-id inference: the first routing rule
whose regex matches decides the provider. Routing rules never contribute to model
info.
A CAPABILITY rule carries any ``model_info`` keys except ``litellm_provider``
(``mode``, ``supports_*``, context window, pricing, ...). It is consumed by
``get_model_info`` fallback resolution: the ``model_info`` of ALL capability rules
whose regex matches is unioned in file order, with later rules overriding earlier
ones on key conflicts, and the caller backfills ``litellm_provider`` with the
provider it requested. If no capability rule matches, model-info resolution misses
as if no rules existed.
LEGACY-SCHEMA SHIM (temporary, until the new-schema JSON reaches main): released
proxies fetch this JSON remotely from main, whose block still ships the old schema
where a rule mixes ``litellm_provider`` with capability keys and may inherit a
parent's ``model_info`` via ``extends``. Such a legacy rule is tolerated rather
than skipped: ``extends`` is resolved once at install time (single level, against
raw parents), and the resolved rule acts as BOTH kinds, a routing rule (its
``litellm_provider`` participates in first-hit inference) and a capability rule
(its full ``model_info``, provider included, participates in the union). New-schema
rules never mix the two and never use ``extends``. A rule whose
``litellm_provider`` is not a string is invalid and is warned about and skipped
(a warning rather than a crash, for the same remote-fetch reason).
Rules are only consulted after exact and case-insensitive lookups miss, so an
exact cost-map entry always takes precedence over any rule.
Patterns are matched case-insensitively with ``re.search`` and are not implicitly
anchored: a rule must include ``^`` and ``$`` (as the shipped rules do) to bind to
the whole model name, otherwise it matches as a substring. Keeping anchoring in the
regex makes the rule the single, self-contained source of truth for what it matches.
A rule may set ``extends`` to the ``name`` of another rule to inherit that rule's
``model_info``; the rule's own ``model_info`` overrides the inherited keys, so a
narrow rule (for example a version-gated capability flag) carries only its delta
instead of duplicating the parent's pricing block. Inheritance is resolved once,
at install time, against each rule's raw (unresolved) ``model_info``; it is a
single level (a parent that itself extends is not chained).
anchored: a rule must include ``^`` and ``$`` to bind to the whole model name,
otherwise it matches as a substring. Keeping anchoring in the regex makes the rule
the single, self-contained source of truth for what it matches.
Any other keys on a rule (for example a free-text ``description`` documenting what
the regex matches) are ignored by the engine and exist only for the reader.
The compiled-regex list is built once and cached. ``match_fallback_generalization``
is O(number of rules); callers must only invoke it on a cache miss.
Rules are compiled and classified once, at install time. The match functions are
O(number of rules); callers must only invoke them on a cache miss.
"""
import re
from typing import Optional
from dataclasses import dataclass
from typing import Optional, Union
from litellm._logging import verbose_logger
NAME_FIELD = "name"
PATTERN_FIELD = "pattern"
MODEL_INFO_FIELD = "model_info"
EXTENDS_FIELD = "extends"
PROVIDER_KEY = "litellm_provider"
LEGACY_EXTENDS_FIELD = "extends"
def _resolve_extends(rules: list) -> list:
"""Expand ``extends`` inheritance so each rule's ``model_info`` is self-contained.
def _resolve_legacy_extends(rules: list) -> list:
"""Expand legacy ``extends`` inheritance so each rule's ``model_info`` is self-contained.
A rule with ``extends: <name>`` is rewritten with ``model_info`` set to the parent's
``model_info`` overlaid by its own. Resolution is single-level and uses each rule's
raw ``model_info`` as the parent source. Non-dict rules and dangling parents are
passed through unchanged.
Compatibility shim for the old remote schema: single level, resolved against each
parent's raw ``model_info``, with the child's own keys winning on conflict. Non-dict
rules and dangling parents pass through unchanged; new-schema rules carry no
``extends`` and are untouched.
"""
base_by_name = {
rule[NAME_FIELD]: rule[MODEL_INFO_FIELD]
@ -61,93 +75,138 @@ def _resolve_extends(rules: list) -> list:
and isinstance(rule.get(MODEL_INFO_FIELD), dict)
}
def resolved(rule: dict) -> dict:
parent_name = rule.get(EXTENDS_FIELD)
def resolved(rule: object) -> object:
if not isinstance(rule, dict):
return rule
parent_name = rule.get(LEGACY_EXTENDS_FIELD)
own_info = rule.get(MODEL_INFO_FIELD)
parent_info = base_by_name.get(parent_name) if isinstance(parent_name, str) else None
if parent_info is None or not isinstance(own_info, dict):
return rule
return {**rule, MODEL_INFO_FIELD: {**parent_info, **own_info}}
return [resolved(rule) if isinstance(rule, dict) else rule for rule in rules]
return [resolved(rule) for rule in rules]
@dataclass(frozen=True, slots=True)
class _RoutingRule:
pattern: re.Pattern
provider: str
@dataclass(frozen=True, slots=True)
class _CapabilityRule:
pattern: re.Pattern
model_info: dict
_CompiledRule = Union[_RoutingRule, _CapabilityRule]
def _compile_rule(rule: object) -> tuple[_CompiledRule, ...]:
if not isinstance(rule, dict):
return ()
pattern = rule.get(PATTERN_FIELD)
model_info = rule.get(MODEL_INFO_FIELD)
if not isinstance(pattern, str) or not isinstance(model_info, dict):
verbose_logger.warning(
"LiteLLM: skipping malformed fallback generalization rule %s (needs string '%s' and dict '%s').",
rule.get(NAME_FIELD, pattern),
PATTERN_FIELD,
MODEL_INFO_FIELD,
)
return ()
try:
compiled = re.compile(pattern, re.IGNORECASE)
except re.error as e:
verbose_logger.warning(
"LiteLLM: skipping fallback generalization rule with invalid regex %r: %s",
pattern,
e,
)
return ()
if PROVIDER_KEY not in model_info:
return (_CapabilityRule(pattern=compiled, model_info=model_info),)
provider = model_info[PROVIDER_KEY]
if not isinstance(provider, str):
verbose_logger.warning(
"LiteLLM: skipping invalid fallback generalization rule %s: '%s' in '%s' must be a string.",
rule.get(NAME_FIELD, pattern),
PROVIDER_KEY,
MODEL_INFO_FIELD,
)
return ()
if len(model_info) == 1:
return (_RoutingRule(pattern=compiled, provider=provider),)
return (
_RoutingRule(pattern=compiled, provider=provider),
_CapabilityRule(pattern=compiled, model_info=model_info),
)
class _FallbackGeneralizations:
"""Holds the active rule list and its lazily-compiled regex cache."""
"""Holds the raw rule list and its install-time-compiled routing and capability rules."""
def __init__(self) -> None:
self.rules: list[dict] = []
self._compiled: Optional[list[tuple[re.Pattern, dict]]] = None
self.rules: list = []
self.routing_rules: tuple = ()
self.capability_rules: tuple = ()
def set_rules(self, rules: Optional[list[dict]]) -> None:
self.rules = rules if isinstance(rules, list) else []
self._compiled = None
def set_rules(self, rules: Optional[list]) -> None:
installed = rules if isinstance(rules, list) else []
compiled = tuple(kind for rule in _resolve_legacy_extends(installed) for kind in _compile_rule(rule))
self.rules = installed
self.routing_rules = tuple(rule for rule in compiled if isinstance(rule, _RoutingRule))
self.capability_rules = tuple(rule for rule in compiled if isinstance(rule, _CapabilityRule))
def _compile(self) -> list[tuple[re.Pattern, dict]]:
compiled: list[tuple[re.Pattern, dict]] = []
for rule in self.rules:
if not isinstance(rule, dict):
continue
pattern = rule.get(PATTERN_FIELD)
model_info = rule.get(MODEL_INFO_FIELD)
if not isinstance(pattern, str) or not isinstance(model_info, dict):
verbose_logger.warning(
"LiteLLM: skipping malformed fallback generalization rule %s (needs string '%s' and dict '%s').",
rule.get("name", pattern),
PATTERN_FIELD,
MODEL_INFO_FIELD,
)
continue
try:
compiled.append((re.compile(pattern, re.IGNORECASE), model_info))
except re.error as e:
verbose_logger.warning(
"LiteLLM: skipping fallback generalization rule with invalid regex %r: %s",
pattern,
e,
)
return compiled
def matches(self, model: str) -> list[dict]:
def match_routing(self, model: str) -> Optional[str]:
if not model:
return []
if self._compiled is None:
self._compiled = self._compile()
return [dict(model_info) for pattern, model_info in self._compiled if pattern.search(model) is not None]
return None
return next(
(rule.provider for rule in self.routing_rules if rule.pattern.search(model) is not None),
None,
)
def match(self, model: str) -> Optional[dict]:
return next(iter(self.matches(model)), None)
def match_capabilities(self, model: str) -> Optional[dict]:
if not model:
return None
matched = tuple(rule.model_info for rule in self.capability_rules if rule.pattern.search(model) is not None)
if not matched:
return None
return {key: value for model_info in matched for key, value in model_info.items()}
_registry = _FallbackGeneralizations()
def set_fallback_generalizations(rules: Optional[list[dict]]) -> None:
"""Install the active rule list and invalidate the compiled-regex cache.
def set_fallback_generalizations(rules: Optional[list]) -> None:
"""Install the active rule list, compiling and classifying each rule.
``extends`` inheritance is resolved here, once, before the rules are stored.
Called once when the model cost map is loaded (and again on any reload).
Legacy ``extends`` inheritance is resolved here, once, before classification;
a legacy rule mixing ``litellm_provider`` with capability keys installs as both
kinds. Malformed and invalid-regex rules are warned about and skipped. Called
once when the model cost map is loaded (and again on any reload).
"""
_registry.set_rules(_resolve_extends(rules) if isinstance(rules, list) else rules)
_registry.set_rules(rules)
def get_fallback_generalization_rules() -> list[dict]:
def get_fallback_generalization_rules() -> list:
"""Return the raw rule list (read-only view for callers/tests)."""
return _registry.rules
def match_fallback_generalization(model: str) -> Optional[dict]:
"""Return the ``model_info`` of the first rule whose regex matches ``model``.
def match_routing_generalization(model: str) -> Optional[str]:
"""Return the provider of the first routing rule whose regex matches ``model``.
O(number of rules). Only call this once exact lookups have missed.
"""
return _registry.match(model)
return _registry.match_routing(model)
def match_all_fallback_generalizations(model: str) -> list[dict]:
"""Return the ``model_info`` of every rule whose regex matches ``model``, in rule order.
def match_capability_generalizations(model: str) -> Optional[dict]:
"""Return the union of the ``model_info`` of every capability rule matching ``model``.
Lets a caller with extra constraints (e.g. a provider match) skip an
inapplicable earlier rule instead of discarding the whole candidate.
Later rules override earlier ones on key conflicts. Returns ``None`` when no
capability rule matches. O(number of rules); only call once exact lookups have missed.
"""
return _registry.matches(model)
return _registry.match_capabilities(model)

8
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -4,7 +4,7 @@ from urllib.parse import urlparse
import litellm
from litellm.constants import REPLICATE_MODEL_NAME_WITH_ID_LENGTH
from litellm.litellm_core_utils.fallback_generalizations import (
match_fallback_generalization,
match_routing_generalization,
)
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
from litellm.secret_managers.main import get_secret, get_secret_str
@ -474,12 +474,10 @@ def get_llm_provider(
custom_llm_provider = "sap"
# Last resort for an otherwise-unknown model: a declarative
# fallback-generalization rule (e.g. routes future claude-* to anthropic).
# fallback-generalization routing rule (e.g. routes future claude-* to anthropic).
# Exact provider matches above always win; this only runs on a miss.
if not custom_llm_provider:
generalization = match_fallback_generalization(model)
if generalization is not None:
custom_llm_provider = generalization.get("litellm_provider") or None
custom_llm_provider = match_routing_generalization(model)
if not custom_llm_provider:
if litellm.suppress_debug_info is False:

54
litellm/model_prices_and_context_window_backup.json
View file

@ -1359,6 +1359,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1393,6 +1394,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1427,6 +1429,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1461,6 +1464,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1661,6 +1665,7 @@
"jp.anthropic.claude-opus-4-8": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
"cache_read_input_token_cost": 5.5e-07,
@ -1743,6 +1748,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1777,6 +1783,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1811,6 +1818,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1845,6 +1853,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1879,6 +1888,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1913,6 +1923,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -45039,31 +45050,26 @@
"fallback_generalizations": {
"rules": [
{
"name": "bedrock-anthropic-claude-mid-conversation-system",
"pattern": "anthropic\\.claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)",
"description": "Bedrock Invoke ids for Claude 4.8 or higher: anthropic.claude-<family> with minor 4.8 through 4.99, any 5.x or later major-minor, or a bare 5+ major, which also admits new families such as fable. These models accept mid-conversation role system messages in place (verified live on Opus 4.8, Sonnet 5 and Fable 5), so unmapped future Bedrock Claudes keep the cache-preserving in-place handling instead of the hoist-all default. Listed first so bare-id provider inference, which takes the first pattern hit, resolves these Bedrock ids to bedrock; model-info resolution skips provider-mismatched rules either way.",
"extends": "anthropic-claude",
"name": "bedrock-claude-ids",
"pattern": "anthropic\\.claude-",
"description": "Any Bedrock-syntax Claude id: the dotted anthropic.claude- segment appears in bare (anthropic.claude-...), region-prefixed (us./eu./au./jp./apac.) and global.-prefixed ids, for every version. Routes these to bedrock before the bare-id Anthropic rule is consulted.",
"model_info": {
"litellm_provider": "bedrock",
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true
"litellm_provider": "bedrock"
}
},
{
"name": "anthropic-claude-adaptive-thinking",
"pattern": "(?:opus|sonnet|haiku)[-._](?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d{1,})[-._]\\d{1,2}(?!\\d))",
"description": "Claude opus/sonnet/haiku at version 4.6 or higher: 4.6 through 4.99, then any 5.x, 6.x or later major. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new families with no code change.",
"extends": "anthropic-claude",
"name": "anthropic-claude-ids",
"pattern": "^claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?$",
"description": "A bare Claude family-major id with an optional minor and an optional 8-digit date suffix, anchored to the whole name, so claude-newfamily-5 routes like claude-newfamily-5-1 does. Routes an unmapped Claude id that carries no provider namespace to the Anthropic API.",
"model_info": {
"supports_adaptive_thinking": true
"litellm_provider": "anthropic"
}
},
{
"name": "anthropic-claude",
"pattern": "^claude-[a-z]+-\\d+[-.]\\d+(?:-\\d{8})?$",
"description": "Any Claude family-major-minor id, optionally with an 8-digit date suffix, anchored to the whole name. Version-neutral fallback that gives an unmapped Claude provider routing and baseline capabilities; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.",
"name": "claude-family-baseline",
"pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?",
"description": "Any Claude family-major id with an optional minor and an optional 8-digit date suffix, under any provider namespace (bare, bedrock-dotted, vertex, databricks, ...), so bare majors like claude-newfamily-5 get the same baseline as claude-newfamily-5-1. Carries the model-family facts every Claude shares; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.",
"model_info": {
"litellm_provider": "anthropic",
"mode": "chat",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
@ -45079,6 +45085,22 @@
"supports_pdf_input": true,
"supports_system_messages": true
}
},
{
"name": "claude-adaptive-thinking",
"pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)",
"description": "Claude at version 4.6 or higher, in any id shape that contains claude-<family>-: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new versions and new families with no code change.",
"model_info": {
"supports_adaptive_thinking": true
}
},
{
"name": "claude-mid-conversation-system",
"pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)",
"description": "Claude at version 4.8 or higher, in any id shape that contains claude-<family>-: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.",
"model_info": {
"supports_mid_conversation_system": true
}
}
]
}

93
litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
View file

@ -820,6 +820,22 @@ class _PersistedDcrCredentials(BaseModel):
client_id: Optional[str] = None
client_secret: Optional[str] = None
token_endpoint_auth_method: Optional[str] = None
redirect_uris: Optional[list[str]] = None
def _redirect_uri_not_registered(credentials: _PersistedDcrCredentials, current_redirect_uri: str) -> bool:
"""Whether a persisted DCR client is positively known NOT to cover the current callback.
A DCR client is bound to the redirect_uris it was registered with; if the proxy's
resolved public origin has since changed, every authorize built for it will be
rejected by the IdP. Clients persisted before ``redirect_uris`` was recorded (and
admin-configured clients, which never get a recording) return False so they are
grandfathered rather than re-registered, because re-minting a client_id orphans
every user's refresh tokens for that server."""
recorded = credentials.redirect_uris
if not recorded:
return False
return current_redirect_uri not in recorded
def _get_persisted_dcr_credentials(credentials: object) -> Optional[_PersistedDcrCredentials]:
@ -886,11 +902,23 @@ async def _get_persisted_mcp_server_with_dcr_client_id(
return persisted_mcp_server, credentials
async def _reuse_persisted_dcr_client_if_available(mcp_server: MCPServer) -> bool:
async def _reuse_persisted_dcr_client_if_available(
mcp_server: MCPServer, current_redirect_uri: Optional[str] = None
) -> bool:
persisted = await _get_persisted_mcp_server_with_dcr_client_id(mcp_server)
if persisted is None:
return False
persisted_mcp_server, credentials = persisted
if current_redirect_uri is not None and _redirect_uri_not_registered(credentials, current_redirect_uri):
verbose_logger.debug(
"register_client_with_server: not reusing persisted DCR client for server_id=%s; its registered "
"redirect_uris=%s do not include the current callback %s. The operator-facing warning for this "
"re-registration event is emitted once by _persisted_dcr_redirect_uri_is_stale.",
mcp_server.server_id,
credentials.redirect_uris,
current_redirect_uri,
)
return False
if not _apply_persisted_dcr_credentials(mcp_server, credentials):
return False
@ -909,11 +937,36 @@ async def _reuse_persisted_dcr_client_if_available(mcp_server: MCPServer) -> boo
return bool(mcp_server.client_id)
async def _persisted_dcr_redirect_uri_is_stale(mcp_server: MCPServer, current_redirect_uri: str) -> bool:
"""Whether the server's persisted DCR client is bound to redirect_uris that no longer
cover the current proxy callback, meaning authorize is guaranteed to fail IdP-side.
Consulted when the in-memory server already carries a hydrated client_id, which
otherwise short-circuits registration before any redirect check can run. Servers
without a persisted DCR recording (admin-configured client_id, or registered before
redirect_uris were recorded) are never reported stale."""
persisted = await _get_persisted_mcp_server_with_dcr_client_id(mcp_server)
if persisted is None:
return False
_, credentials = persisted
if not _redirect_uri_not_registered(credentials, current_redirect_uri):
return False
verbose_logger.warning(
"register_client_with_server: persisted DCR client for server_id=%s is registered with redirect_uris=%s "
"which do not include the current callback %s (proxy origin changed); registering a replacement client. "
"Users previously signed in to this server will need to re-authenticate.",
mcp_server.server_id,
credentials.redirect_uris,
current_redirect_uri,
)
return True
DcrRegistrationPersistenceResult = Literal["persisted", "reused", "skipped", "failed"]
async def _persist_dcr_client_registration(
mcp_server: MCPServer, registration_response: object
mcp_server: MCPServer, registration_response: object, current_redirect_uri: str
) -> DcrRegistrationPersistenceResult:
"""Persist the dynamically registered OAuth client (RFC 7591) onto the MCP server row.
@ -929,6 +982,13 @@ async def _persist_dcr_client_registration(
client identity for these servers. Persisting here would stamp ``oauth2_flow`` and a
``client_id`` onto a server whose mode promises the gateway stores nothing, making a
fresh pass-through server read as gateway-authorized.
``redirect_uris`` records what the client is bound to so a later origin change can be
detected as a positive mismatch and trigger re-registration instead of stranding the
server on IdP-side redirect_uri rejections. ``client_secret`` and
``token_endpoint_auth_method`` are written explicitly (None when absent) because
``update_mcp_server`` merges credential blobs: a re-registered public client must not
inherit the previous client's secret or auth method.
"""
if mcp_server.is_true_passthrough or mcp_server.is_oauth_delegate:
return "skipped"
@ -944,17 +1004,16 @@ async def _persist_dcr_client_registration(
)
return "failed"
if await _reuse_persisted_dcr_client_if_available(mcp_server):
if await _reuse_persisted_dcr_client_if_available(mcp_server, current_redirect_uri=current_redirect_uri):
return "reused"
credentials: MCPCredentials = {
"client_id": registration.client_id,
**({"client_secret": registration.client_secret} if registration.client_secret is not None else {}),
**(
{"token_endpoint_auth_method": "client_secret_basic"}
if registration.token_endpoint_auth_method == "client_secret_basic"
else {}
"client_secret": registration.client_secret,
"token_endpoint_auth_method": (
"client_secret_basic" if registration.token_endpoint_auth_method == "client_secret_basic" else None
),
"redirect_uris": [current_redirect_uri],
}
from litellm.proxy._experimental.mcp_server.db import update_mcp_server # noqa: PLC0415
@ -1017,16 +1076,24 @@ async def register_client_with_server(
):
_raise_if_not_oauth2(mcp_server)
request_base_url = get_request_base_url(request)
current_redirect_uri = f"{request_base_url}/callback"
dummy_return = {
"client_id": fallback_client_id or mcp_server.server_name,
"client_secret": "dummy",
"redirect_uris": [f"{request_base_url}/callback"],
"redirect_uris": [current_redirect_uri],
}
if mcp_server.client_id:
if mcp_server.client_id and not (
persist_credentials
and mcp_server.registration_url
and await _persisted_dcr_redirect_uri_is_stale(mcp_server, current_redirect_uri)
):
return dummy_return
if await _reuse_persisted_dcr_client_if_available(mcp_server):
if await _reuse_persisted_dcr_client_if_available(
mcp_server,
current_redirect_uri=current_redirect_uri if persist_credentials else None,
):
return dummy_return
if mcp_server.authorization_url is None:
@ -1044,7 +1111,7 @@ async def register_client_with_server(
register_data = {
"client_name": client_name,
"redirect_uris": client_redirect_uris if bridge_relay else [f"{request_base_url}/callback"],
"redirect_uris": client_redirect_uris if bridge_relay else [current_redirect_uri],
"grant_types": grant_types or (["authorization_code", "refresh_token"] if bridge_relay else []),
"response_types": response_types or (["code"] if bridge_relay else []),
"token_endpoint_auth_method": token_endpoint_auth_method or ("none" if bridge_relay else ""),
@ -1072,7 +1139,7 @@ async def register_client_with_server(
token_response = response.json()
if persist_credentials and not bridge_relay:
persistence_result = await _persist_dcr_client_registration(mcp_server, token_response)
persistence_result = await _persist_dcr_client_registration(mcp_server, token_response, current_redirect_uri)
if persistence_result == "reused":
return dummy_return

2
litellm/proxy/_experimental/mcp_server/oauth_utils.py
View file

@ -129,7 +129,7 @@ def get_request_base_url(request: Request) -> str:
if x_forwarded_port and ":" not in netloc:
netloc = f"{netloc}:{x_forwarded_port}"
return urlunparse((scheme, netloc, parsed.path, "", "", ""))
return urlunparse((scheme, _strip_default_port(scheme, netloc), parsed.path, "", "", ""))
def validate_loopback_redirect_uri(redirect_uri: str) -> None:

155
litellm/proxy/hooks/parallel_request_limiter_v3.py
View file

@ -244,6 +244,10 @@ TPM_RESERVED_SCOPES_KEY = "_litellm_tpm_reserved_scopes"
# does not double-refund.
TPM_RESERVATION_RELEASED_KEY = "_litellm_tpm_reservation_released"
RATE_LIMIT_DESCRIPTORS_KEY = "_litellm_rate_limit_descriptors"
# Pre-call RateLimitResponse stashed here so streaming success logging can
# mirror ``x-ratelimit-*`` headers into the SLP. Streaming exits
# common_request_processing before ``async_post_call_success_hook`` runs.
RATE_LIMIT_RESPONSE_KEY = "_litellm_proxy_rate_limit_response"
# Stash keys live ONLY in metadata channels — never at the top level of the
# request body. Top-level keys are forwarded as body params to upstream
# providers, which reject unknown fields with 400/429 errors.
@ -253,6 +257,7 @@ _LITELLM_STASH_KEYS: Tuple[str, ...] = (
TPM_RESERVED_SCOPES_KEY,
TPM_RESERVATION_RELEASED_KEY,
RATE_LIMIT_DESCRIPTORS_KEY,
RATE_LIMIT_RESPONSE_KEY,
)
@ -2037,6 +2042,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
else:
# add descriptors to request headers
data["litellm_proxy_rate_limit_response"] = response
# Mirror into metadata so streaming success logging can find
# it via ``kwargs["litellm_params"]["metadata"]``.
self._stash_value_in_metadata_channels(
data=data,
key=RATE_LIMIT_RESPONSE_KEY,
value=response,
)
# ----------------------------------------------------------------
# TPM token reservation
@ -2133,6 +2145,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
stored_response.setdefault("statuses", []).extend(tpm_response["statuses"])
elif tpm_response["statuses"]:
data["litellm_proxy_rate_limit_response"] = tpm_response
# Keep the metadata stash in sync when this is the
# first snapshot written.
self._stash_value_in_metadata_channels(
data=data,
key=RATE_LIMIT_RESPONSE_KEY,
value=tpm_response,
)
verbose_proxy_logger.debug(f"TPM tokens reserved: {estimated_tokens} for model {requested_model}")
@ -2318,6 +2337,23 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
return "total" # default to total
return specified_rate_limit_type
@staticmethod
def _merge_ratelimit_statuses_into_additional_headers(
additional_headers: Dict[str, Any],
statuses: List[RateLimitStatus],
) -> Dict[str, Any]:
"""
Return ``additional_headers`` extended with
``x-ratelimit-{descriptor_key}-{remaining|limit}-{rate_limit_type}``
entries. Non-mutating so callers pick their own target dict.
"""
merged: Dict[str, Any] = dict(additional_headers)
for status in statuses:
prefix = f"x-ratelimit-{status['descriptor_key']}"
merged[f"{prefix}-remaining-{status['rate_limit_type']}"] = status["limit_remaining"]
merged[f"{prefix}-limit-{status['rate_limit_type']}"] = status["current_limit"]
return merged
@staticmethod
def _stash_value_in_metadata_channels(
data: Dict[str, Any],
@ -2698,6 +2734,112 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
except Exception as e:
verbose_proxy_logger.exception(f"Error in rate limit success event: {str(e)}")
async def async_logging_hook(
self,
kwargs: dict,
result: Any,
call_type: str,
) -> Tuple[dict, Any]:
"""
Mirror the pre-call rate-limit snapshot into the SLP so streaming
success callbacks see the same ``x-ratelimit-*`` headers the
non-streaming path writes via ``async_post_call_success_hook``.
Runs in the earlier of the two callback loops inside
``async_success_handler`` so downstream callbacks see the values
regardless of registration order. Idempotent for non-streaming.
"""
self._mirror_ratelimit_response_into_logging_payload(
kwargs=kwargs,
response_obj=result,
)
return kwargs, result
def _mirror_ratelimit_response_into_logging_payload(
self,
kwargs: Any,
response_obj: Any,
) -> None:
"""
Copy the stashed ``RateLimitResponse`` into the SLP's
``hidden_params.additional_headers`` and the response object's
``_hidden_params.additional_headers`` (when the latter is a dict).
"""
if not isinstance(kwargs, dict):
return
standard_logging_object = kwargs.get("standard_logging_object")
standard_logging_metadata: Optional[Dict[str, Any]] = None
if isinstance(standard_logging_object, dict):
slp_metadata = standard_logging_object.get("metadata")
if isinstance(slp_metadata, dict):
standard_logging_metadata = slp_metadata
statuses = self._narrow_ratelimit_statuses(
self._lookup_stashed_value(
kwargs=kwargs,
standard_logging_metadata=standard_logging_metadata,
key=RATE_LIMIT_RESPONSE_KEY,
)
)
if not statuses:
return
if isinstance(standard_logging_object, dict):
hidden_params = standard_logging_object.get("hidden_params")
if not isinstance(hidden_params, dict):
hidden_params = {}
existing = hidden_params.get("additional_headers")
hidden_params["additional_headers"] = self._merge_ratelimit_statuses_into_additional_headers(
additional_headers=existing if isinstance(existing, dict) else {},
statuses=statuses,
)
standard_logging_object["hidden_params"] = hidden_params
response_hidden = getattr(response_obj, "_hidden_params", None)
if isinstance(response_hidden, dict):
existing = response_hidden.get("additional_headers")
response_hidden["additional_headers"] = self._merge_ratelimit_statuses_into_additional_headers(
additional_headers=existing if isinstance(existing, dict) else {},
statuses=statuses,
)
@staticmethod
def _narrow_ratelimit_statuses(stashed: Any) -> List[RateLimitStatus]:
"""
Narrow a stashed ``RateLimitResponse``-shaped dict to a typed
``statuses`` list. Entries missing any header-write field are dropped;
an empty list means "nothing to mirror".
"""
if not isinstance(stashed, dict):
return []
raw_statuses = stashed.get("statuses")
if not isinstance(raw_statuses, list):
return []
narrowed: List[RateLimitStatus] = []
for entry in raw_statuses:
if not isinstance(entry, dict):
continue
descriptor_key = entry.get("descriptor_key")
rate_limit_type = entry.get("rate_limit_type")
current_limit = entry.get("current_limit")
limit_remaining = entry.get("limit_remaining")
if (
isinstance(descriptor_key, str)
and rate_limit_type in ("requests", "tokens", "max_parallel_requests")
and isinstance(current_limit, int)
and isinstance(limit_remaining, int)
):
narrowed.append(
RateLimitStatus(
code=entry.get("code", "OK") if isinstance(entry.get("code"), str) else "OK",
current_limit=current_limit,
limit_remaining=limit_remaining,
rate_limit_type=rate_limit_type,
descriptor_key=descriptor_key,
)
)
return narrowed
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
"""
On failure: decrement max_parallel_requests and refund the upfront
@ -2838,15 +2980,10 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
if isinstance(_hidden_params, BaseModel):
_hidden_params = _hidden_params.model_dump()
_additional_headers = _hidden_params.get("additional_headers", {}) or {}
# Add rate limit headers
for status in litellm_proxy_rate_limit_response["statuses"]:
prefix = f"x-ratelimit-{status['descriptor_key']}"
_additional_headers[f"{prefix}-remaining-{status['rate_limit_type']}"] = status[
"limit_remaining"
]
_additional_headers[f"{prefix}-limit-{status['rate_limit_type']}"] = status["current_limit"]
_additional_headers = self._merge_ratelimit_statuses_into_additional_headers(
additional_headers=_hidden_params.get("additional_headers", {}) or {},
statuses=litellm_proxy_rate_limit_response["statuses"],
)
setattr(
response,

9
litellm/types/mcp.py
View file

@ -165,6 +165,15 @@ class MCPCredentials(TypedDict, total=False):
sends HTTP Basic; defaults to "client_secret_post" when unset.
"""
redirect_uris: Optional[List[str]]
"""
The redirect URIs a dynamically registered (RFC 7591) OAuth client was bound to at
registration time. Lets a later registration detect that the proxy's public origin no
longer matches the registered callback and re-register instead of reusing a client the
IdP will reject. Absent for admin-configured clients and for clients registered before
this field existed. Not a secret; stored unencrypted.
"""
token_exchange_profile: Optional[str]
"""
Token exchange wire dialect: "rfc8693" (default, the standard token-exchange grant) or

59
litellm/utils.py
View file

@ -61,7 +61,7 @@ from litellm._lazy_imports import (
)
from litellm._uuid import uuid
from litellm.litellm_core_utils.fallback_generalizations import (
match_all_fallback_generalizations,
match_capability_generalizations,
)
from litellm.constants import (
DEFAULT_CHAT_COMPLETION_PARAM_VALUES,
@ -2651,6 +2651,26 @@ def _resolve_builtin_model_cost_entry(key: str, provider: str) -> Optional[Dict[
return None
def _get_builtin_model_info_for_registration(model: str) -> Optional[ModelInfo]:
"""Resolve ``model`` to its built-in cost-map entry for registration merging.
Returns ``None`` when the lookup raises or when it resolved via a
fallback-generalization capability rule, detected as the resolved key missing
``litellm.model_cost`` while matching a capability rule. A rule-derived entry
carries no pricing, so treating it as a hit would skip the built-in
cache-pricing inheritance for prefix-mangled keys.
"""
try:
info = get_model_info(model=model)
except Exception:
return None
if info["key"] in litellm.model_cost:
return info
if match_capability_generalizations(info["key"]) is None:
return info
return None
def register_model(model_cost: Union[str, dict]):
"""
Register new / Override existing models (and their pricing) to specific providers.
@ -2691,10 +2711,11 @@ def register_model(model_cost: Union[str, dict]):
existing_model = litellm.model_cost.get(key, {})
model_cost_key = key
else:
try:
existing_model = cast(dict, get_model_info(model=key))
builtin_model_info = _get_builtin_model_info_for_registration(model=_key_str)
if builtin_model_info is not None:
existing_model = cast(dict, builtin_model_info)
model_cost_key = existing_model["key"]
except Exception:
else:
existing_model = {}
model_cost_key = key
builtin_entry = _resolve_builtin_model_cost_entry(key=_key_str, provider=provider)
@ -5043,25 +5064,35 @@ def _get_model_info_from_generalization(
potential_model_names: PotentialModelNamesAndCustomLLMProvider,
custom_llm_provider: Optional[str],
) -> Optional[tuple[str, dict]]:
"""Resolve an unmapped model via a declarative fallback-generalization rule.
"""Resolve an unmapped model via the declarative capability generalization rules.
Tries the same name candidates as the exact lookups, in the same order, and
returns ``(matched_name, model_info)`` for the first matching rule that also
satisfies the provider constraint; a rule scoped to another provider is
skipped in favor of later rules rather than discarding the candidate.
O(number of rules); only call after the exact lookups have missed.
returns ``(matched_name, model_info)`` for the first candidate matched by at
least one capability rule, with ``litellm_provider`` backfilled from the
provider the caller requested. Rules lose to exact entries: if ANY candidate is
an exact ``litellm.model_cost`` key (necessarily provider-mismatched, or the
exact lookups would have returned it), the model is known rather than unmapped,
and resolving it from rules would hand an unpriced rule-derived entry to
callers whose fallback ladder (e.g. the cost calculator's model-name variants)
still had a priced exact name to try. O(number of rules); only call after the
exact lookups have missed.
"""
candidates = [
candidates = (
potential_model_names["combined_model_name"],
model,
potential_model_names["split_model"],
potential_model_names["combined_stripped_model_name"],
potential_model_names["stripped_model_name"],
]
)
if any(_get_model_cost_key(candidate) is not None for candidate in candidates):
return None
for candidate in candidates:
for generalized_info in match_all_fallback_generalizations(candidate):
if _check_provider_match(model_info=generalized_info, custom_llm_provider=custom_llm_provider):
return candidate, generalized_info
generalized_info = match_capability_generalizations(candidate)
if generalized_info is None:
continue
if custom_llm_provider is None:
return candidate, generalized_info
return candidate, {**generalized_info, "litellm_provider": custom_llm_provider}
return None

54
model_prices_and_context_window.json
View file

@ -1359,6 +1359,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1393,6 +1394,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1427,6 +1429,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1461,6 +1464,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1661,6 +1665,7 @@
"jp.anthropic.claude-opus-4-8": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
"cache_read_input_token_cost": 5.5e-07,
@ -1743,6 +1748,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1777,6 +1783,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1811,6 +1818,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1845,6 +1853,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1879,6 +1888,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1913,6 +1923,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -45272,31 +45283,26 @@
"fallback_generalizations": {
"rules": [
{
"name": "bedrock-anthropic-claude-mid-conversation-system",
"pattern": "anthropic\\.claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)",
"description": "Bedrock Invoke ids for Claude 4.8 or higher: anthropic.claude-<family> with minor 4.8 through 4.99, any 5.x or later major-minor, or a bare 5+ major, which also admits new families such as fable. These models accept mid-conversation role system messages in place (verified live on Opus 4.8, Sonnet 5 and Fable 5), so unmapped future Bedrock Claudes keep the cache-preserving in-place handling instead of the hoist-all default. Listed first so bare-id provider inference, which takes the first pattern hit, resolves these Bedrock ids to bedrock; model-info resolution skips provider-mismatched rules either way.",
"extends": "anthropic-claude",
"name": "bedrock-claude-ids",
"pattern": "anthropic\\.claude-",
"description": "Any Bedrock-syntax Claude id: the dotted anthropic.claude- segment appears in bare (anthropic.claude-...), region-prefixed (us./eu./au./jp./apac.) and global.-prefixed ids, for every version. Routes these to bedrock before the bare-id Anthropic rule is consulted.",
"model_info": {
"litellm_provider": "bedrock",
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true
"litellm_provider": "bedrock"
}
},
{
"name": "anthropic-claude-adaptive-thinking",
"pattern": "(?:opus|sonnet|haiku)[-._](?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d{1,})[-._]\\d{1,2}(?!\\d))",
"description": "Claude opus/sonnet/haiku at version 4.6 or higher: 4.6 through 4.99, then any 5.x, 6.x or later major. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new families with no code change.",
"extends": "anthropic-claude",
"name": "anthropic-claude-ids",
"pattern": "^claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?$",
"description": "A bare Claude family-major id with an optional minor and an optional 8-digit date suffix, anchored to the whole name, so claude-newfamily-5 routes like claude-newfamily-5-1 does. Routes an unmapped Claude id that carries no provider namespace to the Anthropic API.",
"model_info": {
"supports_adaptive_thinking": true
"litellm_provider": "anthropic"
}
},
{
"name": "anthropic-claude",
"pattern": "^claude-[a-z]+-\\d+[-.]\\d+(?:-\\d{8})?$",
"description": "Any Claude family-major-minor id, optionally with an 8-digit date suffix, anchored to the whole name. Version-neutral fallback that gives an unmapped Claude provider routing and baseline capabilities; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.",
"name": "claude-family-baseline",
"pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?",
"description": "Any Claude family-major id with an optional minor and an optional 8-digit date suffix, under any provider namespace (bare, bedrock-dotted, vertex, databricks, ...), so bare majors like claude-newfamily-5 get the same baseline as claude-newfamily-5-1. Carries the model-family facts every Claude shares; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.",
"model_info": {
"litellm_provider": "anthropic",
"mode": "chat",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
@ -45312,6 +45318,22 @@
"supports_pdf_input": true,
"supports_system_messages": true
}
},
{
"name": "claude-adaptive-thinking",
"pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)",
"description": "Claude at version 4.6 or higher, in any id shape that contains claude-<family>-: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new versions and new families with no code change.",
"model_info": {
"supports_adaptive_thinking": true
}
},
{
"name": "claude-mid-conversation-system",
"pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)",
"description": "Claude at version 4.8 or higher, in any id shape that contains claude-<family>-: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.",
"model_info": {
"supports_mid_conversation_system": true
}
}
]
}

4
pyproject.toml
View file

@ -184,6 +184,10 @@ dev = [
"vcrpy==8.2.1",
"pytest-recording==0.13.4",
]
e2e-dev = [
"playwright==1.61.0",
"websockets>=15.0.1,<16.0",
]
proxy-dev = [
"prisma==0.11.0",
"hypercorn==0.17.3",

7
scripts/pre_commit_lint.sh
View file

@ -4,6 +4,7 @@
# before `git commit`; it inspects your staged files and runs only the matching
# gating CI checks, so a clean run means a green CI lint:
# - litellm/ Python staged -> `make lint` (test-linting.yml's lint job)
# - tests/e2e Python staged -> `make lint-e2e-basedpyright` (test-linting.yml's e2e type-check step)
# - dashboard staged -> prettier + eslint + lint budgets (test-litellm-ui-build.yml's frontend-lint)
# - proxy/types staged -> regenerate dashboard API types and fail on drift (check-ui-api-types.yml)
#
@ -25,6 +26,7 @@ staged_match() { printf '%s\n' "$staged" | grep -E "$1" || true; }
# scripts-only commit can't turn it red; scope the trigger there to skip the slow
# make lint when it couldn't catch anything.
litellm_py_files=$(staged_match '^litellm/.*\.py$')
e2e_py_files=$(staged_match '^tests/e2e/.*\.py$')
# ruff format (and CI's format step) skip enterprise; the rest of make lint covers it.
fmt_files=$(printf '%s\n' "$litellm_py_files" | grep -v '^litellm/enterprise/' || true)
# check-ui-api-types.yml triggers on any file under litellm/proxy or litellm/types
@ -107,6 +109,11 @@ if [ -n "$litellm_py_files" ]; then
fi
fi
if [ -n "$e2e_py_files" ] && [ -z "$litellm_py_files" ]; then
echo "pre-commit: type-checking tests/e2e (make lint-e2e-basedpyright)"
make lint-e2e-basedpyright || { echo "✗ tests/e2e basedpyright failed. Fix the errors above, then re-run make pre-commit." >&2; status=1; }
fi
if [ -n "$ui_prettier_files" ] || [ -n "$ui_eslint_files" ]; then
echo "pre-commit: linting dashboard (prettier + eslint + lint budgets)"
lint_dashboard || { echo "✗ Dashboard lint failed. See above; format with: (cd ui/litellm-dashboard && npm run format)." >&2; status=1; }

2
tests/e2e/CLAUDE.md
View file

@ -57,7 +57,7 @@ Mark live tests with `@pytest.mark.e2e` (on the class or the module). Pure cover
## Typing
The harness is fully typed and new code must not add `Any` or widen the basedpyright budgets. When a response field is untyped, model it in `models.py` (just the fields you read) and let pydantic validate it, rather than threading a `dict` or `Any` through the test
The harness is fully typed with no error budget: `make lint-e2e-basedpyright` must report zero basedpyright errors, and CI enforces that on any PR touching `tests/e2e/**/*.py`. When a response field is untyped, model it in `models.py` (just the fields you read) and let pydantic validate it, rather than threading a `dict` or `Any` through the test
## Coverage registry

6
tests/e2e/CONTRIBUTING.md
View file

@ -34,10 +34,10 @@ The suites run against a live proxy, so bring one up first. `docker-compose.yml`
uv run pytest tests/e2e/llm_translation/ -v
```
The browser tests in the `management/` suite drive the dashboard the proxy serves at `/ui` through playwright, an optional dependency behind `importorskip` (the suite's API tests run without it). Install it once into your environment along with its browser:
The browser tests in the `management/` suite drive the dashboard the proxy serves at `/ui` through playwright, an optional dependency behind `importorskip` (the suite's API tests run without it). It lives in the `e2e-dev` dependency group; install it along with its browser:
```bash
uv pip install playwright
uv sync --inexact --group e2e-dev
uv run playwright install chromium
```
@ -140,7 +140,7 @@ Mark live tests with `@pytest.mark.e2e` (on the class or the module). Pure cover
Before you push
1. Run basedpyright over your changes; the harness is fully typed and new code must not add `Any` or widen the budgets
1. Run `make lint-e2e-basedpyright` (or `make pre-commit` with your changes staged); the harness is fully typed and the gate allows zero basedpyright errors, enforced in CI on any PR touching `tests/e2e/**/*.py`
2. Add the models your test needs to the inline config in `docker-compose.yml`

2
tests/e2e/budgets/budget_client.py
View file

@ -342,7 +342,7 @@ class BudgetClient:
case _:
time.sleep(_TEAM_READY_SLEEP_SECONDS)
assert last is not None
_ = unwrap(last)
raise AssertionError(last)
def add_team_member(self, team_id: str, user_id: str, *, max_budget_in_team: float | None = None) -> None:
last_body = ""

6
tests/e2e/budgets/test_budget_fallback_e2e.py
View file

@ -4,7 +4,6 @@ blocking the caller with a `budget_exceeded` error. Coverage for the
budget_fallbacks feature in litellm/proxy/hooks/model_max_budget_limiter.py.
"""
import json
import time
import pytest
@ -12,6 +11,7 @@ import pytest
from budget_client import BudgetClient, model_budget
from e2e_config import unique_marker
from lifecycle import ResourceManager
from models import AnthropicMessagesResponse
pytestmark = pytest.mark.e2e
@ -42,8 +42,8 @@ def test_budget_fallback_reroutes_anthropic_messages_to_openai(
"budget_fallbacks must reroute transparently, never surface a "
f"block; status={result.status_code} body={result.body[:300]}"
)
served_by = json.loads(result.body)["model"]
if FALLBACK_MODEL in served_by:
served_by = AnthropicMessagesResponse.model_validate_json(result.body).model
if served_by is not None and FALLBACK_MODEL in served_by:
break
time.sleep(1)
assert served_by is not None and FALLBACK_MODEL in served_by, (

14
tests/e2e/budgets/test_spend_counter_reseed_e2e.py
View file

@ -23,6 +23,7 @@ from threading import Barrier
from typing import TYPE_CHECKING
import pytest
from pydantic import TypeAdapter, ValidationError
from budget_client import BudgetClient
from e2e_config import unique_marker
@ -42,6 +43,8 @@ BURST = 6
# expires the counter; this waits out both.
COLD_WAIT_SECONDS = 80
_JSON_FLOAT: TypeAdapter[float] = TypeAdapter(float)
def _redis() -> "redis.Redis[str] | RedisCluster[str]":
"""The proxy's Redis. The deployed runner sets REDIS_HOST to the serverless
@ -75,12 +78,11 @@ def _parse_counter(raw: object) -> float | None:
try:
return float(text)
except ValueError:
try:
import json
return float(json.loads(text))
except Exception:
return None
pass
try:
return _JSON_FLOAT.validate_json(text)
except ValidationError:
return None
def _spend_counter(rds: "redis.Redis[str] | RedisCluster[str]", key: str) -> float | None:

48
tests/e2e/conftest.py
View file

@ -1,9 +1,9 @@
"""Shared fixtures for all live e2e suites under tests/e2e/.
Design rule: skip on environment, fail on behavior. Live tests (marked `e2e`)
skip when no proxy answers; once a request reaches the proxy, behavior is
asserted. Pure unit coverage of the harness itself carries no `e2e` marker and
runs regardless of whether a proxy is up.
Design rule: hard failures only. Live tests (marked `e2e`) fail when no proxy
answers or when credentials/env are missing; they never skip. Pure unit coverage
of the harness itself carries no `e2e` marker and runs regardless of whether a
proxy is up.
Lifecycle: the `resources` fixture maps the init -> run -> teardown contract
(lifecycle.E2ECase) onto pytest - setup is init(), the test body is run(), and
@ -40,7 +40,7 @@ def pytest_configure(config: pytest.Config) -> None:
def _liveness_reason(label: str, base_url: str) -> str | None:
"""None if `base_url` answers its liveness probe, else a skip reason."""
"""None if `base_url` answers its liveness probe, else a failure reason."""
try:
resp = requests.get(f"{base_url}/health/liveliness", timeout=5)
except requests.RequestException as exc:
@ -51,10 +51,10 @@ def _liveness_reason(label: str, base_url: str) -> str | None:
@functools.lru_cache(maxsize=1)
def _proxy_skip_reason() -> str | None:
"""Probe the proxy once per session. None if it answers, else a skip reason. In
a split deployment the management/admin control plane is a separate service, so
require it too (when it differs) - else its tests would fail rather than skip."""
def _proxy_fail_reason() -> str | None:
"""Probe the proxy once per session. None if it answers, else a failure reason.
In a split deployment the management/admin control plane is a separate service,
so require it too when it differs."""
reason = _liveness_reason("proxy", PROXY_BASE_URL)
if reason is not None:
return reason
@ -64,19 +64,19 @@ def _proxy_skip_reason() -> str | None:
def pytest_runtest_setup(item: pytest.Item) -> None:
"""Skip `e2e`-marked tests unless a proxy answers its liveness probe. Unmarked
tests (unit coverage of the harness) don't touch the proxy, so they run even
when none is up."""
"""Hard-fail `e2e`-marked tests unless a proxy answers its liveness probe.
Unmarked tests (unit coverage of the harness) don't touch the proxy, so they
run even when none is up. Never skip for a missing proxy."""
if item.get_closest_marker("e2e") is None:
return
reason = _proxy_skip_reason()
reason = _proxy_fail_reason()
if reason is not None:
pytest.skip(reason)
pytest.fail(reason)
def pytest_runtest_call(item: pytest.Item) -> None:
"""Mark that an e2e test body actually ran (not skipped at setup). Skipped
sessions never reach this hook, so the session-finish cleanup can use it as a
"""Mark that an e2e test body actually ran (setup passed). Sessions that fail
setup never reach this hook, so the session-finish cleanup can use it as a
guard before truncating the spend-log DB. Tests under `tests/e2e/` without the
`e2e` marker (pure unit coverage for the harness itself) never hit the proxy,
so they must not arm the destructive DB truncate."""
@ -87,12 +87,12 @@ def pytest_runtest_call(item: pytest.Item) -> None:
def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None:
"""Once the whole e2e session is done (all suites), truncate the spend logs so
the DB doesn't accumulate test rows. Skipped sessions (no live proxy, no test
actually executed) leave the DB alone so a `DATABASE_URL` pointing at a shared
instance is never wiped without an e2e run. Best-effort: a cleanup failure (no
DB reachable) must not fail the run. The spend_tracking dir goes on sys.path
only for this import and is removed after, so a broader `pytest tests/` run is
not left with a mutated path."""
the DB doesn't accumulate test rows. Sessions where no e2e test body ran leave
the DB alone so a `DATABASE_URL` pointing at a shared instance is never wiped
without an e2e run. Best-effort: a cleanup failure (no DB reachable) must not
fail the run. The spend_tracking dir goes on sys.path only for this import and
is removed after, so a broader `pytest tests/` run is not left with a mutated
path."""
if not session.stash.get(_E2E_TEST_RAN, False):
return
spend_dir = str(Path(__file__).parent / "spend_tracking")
@ -102,7 +102,7 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None:
reset_spend_logs()
except Exception as exc: # noqa: BLE001 - cleanup is best-effort
print(f"spend-log cleanup skipped: {exc}")
print(f"spend-log cleanup best-effort failed: {exc}")
finally:
if spend_dir in sys.path:
sys.path.remove(spend_dir)
@ -112,7 +112,7 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None:
remediate(session)
except Exception as exc: # noqa: BLE001 - remediation is best-effort
print(f"devin remediation skipped: {exc}")
print(f"devin remediation best-effort failed: {exc}")
@pytest.fixture

19
tests/e2e/coverage_registry/collector.py
View file

@ -17,8 +17,10 @@ import sys
from argparse import ArgumentParser
from dataclasses import dataclass
from pathlib import Path
from typing import Literal
import pytest
from pydantic import BaseModel
from .registry import load_registry
from .schema import MODULE_ORDER, Cell, Tier, dashboard_module, loki_module_label
@ -35,12 +37,13 @@ class _CoversSink:
self.collection_errors: tuple[str, ...] = ()
def pytest_collection_finish(self, session: pytest.Session) -> None:
self.covered_ids = frozenset(
arg
marker_args: tuple[tuple[object, ...], ...] = tuple(
marker.args
for item in session.items
for marker in item.iter_markers(name="covers")
for arg in marker.args
if isinstance(arg, str)
)
self.covered_ids = frozenset(
arg for args in marker_args for arg in args if isinstance(arg, str)
)
def pytest_collectreport(self, report: pytest.CollectReport) -> None:
@ -257,6 +260,12 @@ def render_loki(report: CoverageReport) -> str:
return "\n".join(lines)
class _CliArgs(BaseModel):
format: Literal["text", "json", "prometheus", "loki"]
strict: bool
fail_on_collection_errors: bool
def main() -> int:
parser = ArgumentParser()
parser.add_argument(
@ -275,7 +284,7 @@ def main() -> int:
action="store_true",
help="Exit non-zero if pytest collection errors are found.",
)
args = parser.parse_args()
args = _CliArgs.model_validate(vars(parser.parse_args()))
cells = load_registry()
covered, errors = collect_covered_ids()
report = compute_coverage(cells, covered, errors)

15
tests/e2e/coverage_registry/registry.py
View file

@ -6,20 +6,23 @@ from collections import Counter
from pathlib import Path
import yaml
from pydantic import TypeAdapter
from .schema import CELL_ADAPTER, Cell
from .schema import Cell
REGISTRY_DIR = Path(__file__).resolve().parent
_CELLS_ADAPTER: TypeAdapter[tuple[Cell, ...]] = TypeAdapter(tuple[Cell, ...])
def _load_cells(path: Path) -> tuple[Cell, ...]:
return _CELLS_ADAPTER.validate_python(yaml.safe_load(path.read_text()) or ())
def load_registry(registry_dir: Path = REGISTRY_DIR) -> tuple[Cell, ...]:
"""Every cell across every `*.yaml`, validated. Raises on a schema violation or
a duplicate id, since either would corrupt the coverage denominator."""
cells = tuple(
CELL_ADAPTER.validate_python(row)
for path in sorted(registry_dir.glob("*.yaml"))
for row in (yaml.safe_load(path.read_text()) or ())
)
cells = tuple(cell for path in sorted(registry_dir.glob("*.yaml")) for cell in _load_cells(path))
duplicates = sorted(cid for cid, n in Counter(c.id for c in cells).items() if n > 1)
if duplicates:
raise ValueError(f"duplicate cell ids in registry: {duplicates}")

23
tests/e2e/e2e_http.py
View file

@ -107,13 +107,15 @@ class ProbeResult(BaseModel):
class StreamingResponse(BaseModel):
"""Raw outcome for calls whose body is provider-native or streamed: status, the
x-litellm-call-id header (== SpendLogs.request_id), the content-type (which
tells streaming `text/event-stream` from non-streaming `application/json`), and
the body. Used by passthrough and streaming, where one validated JSON model
does not fit."""
x-litellm-call-id header, the x-litellm-response-cost header (StandardLogging
response_cost), the content-type (which tells streaming `text/event-stream` from
non-streaming `application/json`), and the body. SpendLogs.request_id is the
completion body id, not call_id. Used by passthrough and streaming, where one
validated JSON model does not fit."""
status_code: int
call_id: str | None = None # x-litellm-call-id header
response_cost: float | None = None # x-litellm-response-cost header
content_type: str | None = None
body: str
chunks: int = 0 # streamed events (0 for non-streaming)
@ -260,13 +262,25 @@ def probe(
return ProbeResult(status_code=resp.status_code, body=resp.text)
def _parse_response_cost(resp: requests.Response) -> float | None:
raw = _hdr(resp, "x-litellm-response-cost")
if raw is None or raw == "":
return None
try:
return float(raw)
except ValueError:
return None
def _streaming_outcome(resp: requests.Response, stream: bool) -> StreamingResponse:
call_id = _hdr(resp, "x-litellm-call-id")
response_cost = _parse_response_cost(resp)
content_type = _hdr(resp, "content-type")
if not stream or not (200 <= resp.status_code < 300):
return StreamingResponse(
status_code=resp.status_code,
call_id=call_id,
response_cost=response_cost,
content_type=content_type,
body=resp.text,
)
@ -275,6 +289,7 @@ def _streaming_outcome(resp: requests.Response, stream: bool) -> StreamingRespon
return StreamingResponse(
status_code=resp.status_code,
call_id=call_id,
response_cost=response_cost,
content_type=content_type,
body="<streamed>",
chunks=chunks,

17
tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md
View file

@ -17,14 +17,15 @@ transcript, and that `response.done` carries normalized usage.
normalized `response.function_call_arguments.done` with valid JSON arguments and
a matching `function_call` output item, the test sends a `function_call_output`
back, and the follow-up response incorporates the result (the temperature 72
appears).
appears). That raw-websocket tool path is the source of truth for tool calling.
`test_realtime_pipecat_e2e` is a realism layer that drives the same providers
through pipecat's GA `OpenAIRealtimeLLMService` (base_url pointed at the proxy)
rather than speaking the protocol by hand. Its assertions are coarse (the tool
callback fired, assistant text was produced); the raw-websocket suite is the
source of truth. It skips unless `pipecat-ai` is installed
(`uv pip install "pipecat-ai[openai]"`).
`test_pipecat_tool_smoke` is a realism layer through pipecat for openai, azure,
and gemini only (not vertex_ai: native-audio live is flaky under pipecat tool
calling while raw-ws tools pass; see pipecat-ai/pipecat#2544). Assertions are
coarse; raw-ws remains authoritative. Requires `pipecat-ai`.
Pipecat audio coverage lives in `test_realtime_pipecat_audio_e2e.py` (VAD / audio
I/O).
## Provisioning
@ -43,7 +44,7 @@ at call time. The provider table below is the source of truth; edit `PROVIDERS`
| gemini | `gemini-realtime` | `gemini/gemini-3.1-flash-live-preview` |
| vertex_ai | `vertex-realtime` | `vertex_ai/gemini-live-2.5-flash-preview-native-audio-09-2025` |
Bedrock and xai (`xai/grok-4-1-fast`) are supported by the proxy but
Bedrock and xai (`xai/grok-4-1-fast-non-reasoning`) are supported by the proxy but
kept commented out in `PROVIDERS` until they pass end-to-end here; re-enable them by
uncommenting their entry.

8
tests/e2e/llm_translation/realtime/realtime_client.py
View file

@ -14,7 +14,7 @@ import time
from collections.abc import Generator, Mapping
from contextlib import contextmanager
from dataclasses import dataclass
from typing import Any, TypeVar
from typing import TypeVar
from urllib.parse import urlencode
from pydantic import BaseModel, ConfigDict
@ -28,7 +28,7 @@ from models import LiteLLMParamsBody
_M = TypeVar("_M", bound=BaseModel)
def _ws_base_url() -> str:
def ws_base_url() -> str:
for scheme, ws_scheme in (("https://", "wss://"), ("http://", "ws://")):
if PROXY_BASE_URL.startswith(scheme):
return ws_scheme + PROXY_BASE_URL[len(scheme) :]
@ -36,7 +36,7 @@ def _ws_base_url() -> str:
def realtime_ws_url(model: str) -> str:
return f"{_ws_base_url()}/v1/realtime?{urlencode({'model': model})}"
return f"{ws_base_url()}/v1/realtime?{urlencode({'model': model})}"
@dataclass(frozen=True, slots=True)
@ -217,7 +217,7 @@ class OutputItemDone(BaseModel):
class ResponsePayload(BaseModel):
model_config = ConfigDict(extra="allow")
usage: dict[str, Any] | None = None
usage: dict[str, object] | None = None
output: list[OutputItem] | None = None

6
tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py
View file

@ -30,7 +30,7 @@ import pytest
from realtime_client import (
PROVIDERS,
RealtimeProvider,
_ws_base_url,
ws_base_url,
realtime_model,
)
@ -154,7 +154,7 @@ async def _run_pipeline(
llm = LiteLLMRealtimeLLMService(
api_key=key,
base_url=f"{_ws_base_url()}/v1/realtime",
base_url=f"{ws_base_url()}/v1/realtime",
settings=OpenAIRealtimeLLMService.Settings(
model=model,
system_instruction=(
@ -276,7 +276,7 @@ async def _run_audio_input_pipeline(
llm = LiteLLMRealtimeLLMService(
api_key=key,
base_url=f"{_ws_base_url()}/v1/realtime",
base_url=f"{ws_base_url()}/v1/realtime",
settings=OpenAIRealtimeLLMService.Settings(
model=model,
system_instruction=(

11
tests/e2e/llm_translation/realtime/test_realtime_pipecat_e2e.py
View file

@ -28,7 +28,7 @@ import pytest
from realtime_client import (
PROVIDERS,
RealtimeProvider,
_ws_base_url,
ws_base_url,
realtime_model,
)
@ -60,7 +60,12 @@ from pipecat.services.llm_service import FunctionCallParams # noqa: E402
from pipecat_service import LiteLLMRealtimeLLMService # noqa: E402
PROVIDER_PARAMS = [pytest.param(p, id=p.id) for p in PROVIDERS]
# Vertex native-audio live is flaky through pipecat tool calling (upstream
# pipecat-ai/pipecat#2544); raw-ws tool_call_round_trip[vertex_ai] is the
# source of truth for that provider. Keep openai/azure/gemini here.
PROVIDER_PARAMS = [
pytest.param(p, id=p.id) for p in PROVIDERS if p.id != "vertex_ai"
]
WEATHER_TOOL = ToolsSchema(
standard_tools=[
@ -94,7 +99,7 @@ async def _run_pipeline(key: str, model: str) -> tuple[bool, bool]:
await params.result_callback({"city": "Paris", "temperature_f": 72})
llm = LiteLLMRealtimeLLMService(
api_key=key, base_url=f"{_ws_base_url()}/v1/realtime", model=model
api_key=key, base_url=f"{ws_base_url()}/v1/realtime", model=model
)
llm.register_function("get_weather", get_weather)

5
tests/e2e/llm_translation/test_cache_control.py
View file

@ -111,12 +111,11 @@ def _assert_cache_read_on_second_call(
first = unwrap(_cache_chat(client, key, model, prefix))
assert first.choices, f"{model}: first cache-priming call returned no choices: {first}"
read_tokens = 0
deadline = time.monotonic() + 30.0
while time.monotonic() < deadline:
while True:
second = unwrap(_cache_chat(client, key, model, prefix))
read_tokens = _cached_read_tokens(second.usage)
if read_tokens > 0:
if read_tokens > 0 or time.monotonic() >= deadline:
break
time.sleep(3.0)

32
tests/e2e/logging/conftest.py
View file

@ -1,37 +1,43 @@
"""Fixtures for the Datadog logging suite.
"""Fixtures for the logging e2e suite.
These tests drive the Datadog batch-send path (#25663) directly against the real
Datadog logs intake with synthetic events - no LLM calls, no proxy, no log
read-back - so they need only the shipping credentials DD_API_KEY + DD_SITE
(DD_SERVICE is an optional tag). No Datadog Application key is required, and they
skip when the shipping credentials are absent from the environment.
Missing proxy, provider keys, or integration credentials are hard failures.
Never pytest.skip from this suite for environment gaps.
"""
from __future__ import annotations
import os
import pytest
from logging_client import LoggingClient, build_logging_client
from logging_client import LangfuseCreds, LoggingClient, build_logging_client, load_langfuse_creds
def pytest_configure(config: pytest.Config) -> None:
config.addinivalue_line(
"markers",
"covers: registry cell a test covers, e.g. logging.datadog.success.writes_object",
"covers: registry cell a test covers, e.g. logging.langfuse.success.logs_spend",
)
@pytest.fixture(scope="session")
def client() -> LoggingClient:
"""The logging suite's client: holds the shared Gateway so `resources` /
`scoped_key` clean up keys, and adds `/metrics` scraping."""
`scoped_key` clean up keys and teams, and adds `/metrics` scraping plus
Langfuse read-back."""
return build_logging_client()
@pytest.fixture
def datadog_creds() -> None:
"""Gate the suite on the Datadog shipping credentials. The DataDogLogger is built
inside each async test, not here, because its __init__ schedules a periodic-flush
task via asyncio.create_task and so needs a running event loop."""
"""Require Datadog shipping credentials. Hard-fail when absent; never skip."""
if not (os.getenv("DD_API_KEY") and os.getenv("DD_SITE")):
pytest.skip("set DD_API_KEY and DD_SITE to run the Datadog logging suite")
pytest.fail(
"Datadog e2e requires DD_API_KEY and DD_SITE; missing credentials is a hard failure, not a skip"
)
@pytest.fixture(scope="session")
def langfuse_creds() -> LangfuseCreds:
"""Require real Langfuse cloud credentials for team callback + trace poll."""
return load_langfuse_creds()

547
tests/e2e/logging/logging_client.py
View file

@ -1,48 +1,571 @@
"""Client for the logging e2e suite: drive traffic and scrape the proxy's
Prometheus ``/metrics`` endpoint.
"""Client for the logging e2e suite: team/key/org-scoped Langfuse OTEL callbacks,
chat (including tools), Prometheus scrape, and Langfuse observation read-back.
Holds the shared Gateway so the ``resources`` fixture cleans up keys it creates.
``/metrics`` is exposed as plaintext (not a typed JSON body), so scraping goes
through ``transport.probe`` and returns the raw exposition text for a Prometheus
parser to read.
Holds the shared Gateway so the ``resources`` fixture cleans up keys, teams,
users, orgs, and models it creates. External Langfuse reads go through
``e2e_http`` (the only module allowed to call ``requests.*``).
Uses the ``langfuse_otel`` callback (OTLP to ``{host}/api/public/otel``), not
the classic ``langfuse`` SDK callback. OTEL generations land as name
``litellm_request``; correlate by unique prompt marker and ``user_api_key_alias``
in metadata. Spend is on ``calculatedTotalCost`` (StandardLogging response_cost).
"""
from __future__ import annotations
import base64
import json
import os
import time
from dataclasses import dataclass
from typing import Literal
import pytest
from pydantic import BaseModel, ConfigDict, Field
from e2e_config import POLL_INTERVAL, POLL_TIMEOUT
from e2e_gateway import Gateway, build_gateway
from e2e_http import NoBody, unwrap
from models import ChatBody, ChatMessage, ChatResponse, KeyGenerateBody
from e2e_http import (
URL,
AuthHeaders,
NoBody,
StreamingResponse,
Success,
get,
unwrap,
)
from models import (
ChatBody,
ChatMessage,
ChatResponse,
ChatTool,
ChatToolFunction,
KeyGenerateBody,
KeyLoggingCallback,
KeyLoggingCallbackVars,
KeyMetadata,
LiteLLMParamsBody,
OrgDeleteBody,
OrgNewBody,
OrgNewResponse,
SpendLogRow,
TeamDeleteBody,
TeamNewBody,
TeamNewResponse,
UserDeleteBody,
UserNewBody,
UserNewResponse,
)
# Deliberately invalid *upstream provider* key for failure-path tests.
# Not a LiteLLM virtual key; OpenAI must reject it after the proxy accepts the call.
INVALID_UPSTREAM_API_KEY = "sk-upstream-invalid-for-langfuse-e2e-only"
WEATHER_TOOL = ChatTool(
type="function",
function=ChatToolFunction(
name="get_weather",
description="Get the current weather for a city",
parameters={
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
),
)
class TeamCallbackBody(BaseModel):
callback_name: Literal["langfuse_otel", "langfuse", "langsmith", "gcs"]
callback_type: Literal["success", "failure", "success_and_failure"]
callback_vars: dict[str, str]
class TeamCallbackResponse(BaseModel):
model_config = ConfigDict(extra="ignore")
status: str
class GuardrailLitellmParams(BaseModel):
guardrail: str
mode: str
default_on: bool = False
rules: list[dict[str, object]] | None = None
default_action: str | None = None
on_disallowed_action: str | None = None
class GuardrailSpec(BaseModel):
guardrail_name: str
litellm_params: GuardrailLitellmParams
class CreateGuardrailBody(BaseModel):
guardrail: GuardrailSpec
class CreateGuardrailResponse(BaseModel):
model_config = ConfigDict(extra="ignore")
guardrail_id: str | None = None
guardrail_name: str | None = None
class LangfuseObservation(BaseModel):
model_config = ConfigDict(extra="ignore", populate_by_name=True)
id: str
trace_id: str | None = Field(default=None, alias="traceId")
name: str | None = None
type: str | None = None
calculated_total_cost: float | None = Field(default=None, alias="calculatedTotalCost")
level: str | None = None
input: object | None = None
output: object | None = None
metadata: object | None = None
usage: object | None = None
usage_details: object | None = Field(default=None, alias="usageDetails")
model: str | None = None
class LangfuseObservationList(BaseModel):
model_config = ConfigDict(extra="ignore")
data: list[LangfuseObservation] = []
class LangfuseListParams(BaseModel):
model_config = ConfigDict(populate_by_name=True)
limit: int = 100
trace_id: str | None = Field(default=None, alias="traceId")
name: str | None = None
from_start_time: str | None = Field(default=None, alias="fromStartTime")
@dataclass(frozen=True, slots=True)
class LangfuseCreds:
public_key: str
secret_key: str
host: str
@property
def auth_headers(self) -> AuthHeaders:
token = base64.b64encode(f"{self.public_key}:{self.secret_key}".encode()).decode()
return AuthHeaders(authorization=f"Basic {token}")
def callback_vars(self) -> dict[str, str]:
return {
"langfuse_public_key": self.public_key,
"langfuse_secret_key": self.secret_key,
"langfuse_host": self.host,
}
def key_logging_metadata(self) -> KeyMetadata:
return KeyMetadata(
logging=[
KeyLoggingCallback(
callback_name="langfuse_otel",
callback_type="success_and_failure",
callback_vars=KeyLoggingCallbackVars(
langfuse_public_key=self.public_key,
langfuse_secret_key=self.secret_key,
langfuse_host=self.host,
),
)
]
)
def load_langfuse_creds() -> LangfuseCreds:
public_key = os.getenv("LANGFUSE_PUBLIC_KEY")
secret_key = os.getenv("LANGFUSE_SECRET_KEY")
host = (os.getenv("LANGFUSE_BASE_URL") or os.getenv("LANGFUSE_HOST") or "").rstrip("/")
if not (public_key and secret_key and host):
pytest.fail(
"Langfuse e2e requires LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY, and "
"LANGFUSE_BASE_URL (or LANGFUSE_HOST); missing credentials is a hard failure, not a skip"
)
return LangfuseCreds(public_key=public_key, secret_key=secret_key, host=host)
def observation_spend(obs: LangfuseObservation) -> float | None:
"""Langfuse calculatedTotalCost is populated from StandardLogging response_cost."""
return obs.calculated_total_cost
def costs_agree(expected: float, actual: float, *, rel_tol: float = 0.05) -> bool:
"""Costs agree within 5% relative (or 1e-9 absolute for near-zero)."""
return abs(expected - actual) <= max(1e-9, abs(expected) * rel_tol)
def completion_response_id(body: str) -> str | None:
"""SpendLogs.request_id is the chat completion body id, not x-litellm-call-id."""
if not body or body == "<streamed>":
return None
try:
parsed = json.loads(body)
except json.JSONDecodeError:
return None
if not isinstance(parsed, dict):
return None
raw = parsed.get("id")
return raw if isinstance(raw, str) and raw else None
def _matches_run(obs: LangfuseObservation, *, key_alias: str, prompt_marker: str) -> bool:
"""Match a Langfuse generation for this run.
langfuse_otel names generations ``litellm_request`` (not ``litellm:{alias}``).
Prefer the unique prompt marker in input; fall back to key alias in metadata
(user_api_key_alias) or the classic SDK generation name.
"""
if prompt_marker and prompt_marker in json.dumps(obs.input, default=str):
return True
meta_blob = json.dumps(obs.metadata, default=str) if obs.metadata is not None else ""
if key_alias and key_alias in meta_blob:
return True
if obs.name == f"litellm:{key_alias}":
return True
return False
def observation_mentions_tool(obs: LangfuseObservation, tool_name: str) -> bool:
blob = json.dumps(
{"input": obs.input, "output": obs.output, "metadata": obs.metadata},
default=str,
)
return tool_name in blob
def observation_has_guardrail(obs: LangfuseObservation, *, guardrail_name: str) -> bool:
blob = json.dumps(obs.metadata, default=str) if obs.metadata is not None else ""
if guardrail_name in blob or "guardrail" in blob.lower():
return True
if obs.name is not None and "guardrail" in obs.name.lower():
return True
return False
@dataclass(frozen=True, slots=True)
class LoggingClient:
gateway: Gateway
def key_with_alias(self, alias: str, *, models: list[str]) -> str:
def key_with_alias(
self,
alias: str,
*,
models: list[str],
team_id: str | None = None,
user_id: str | None = None,
organization_id: str | None = None,
metadata: KeyMetadata | None = None,
) -> str:
return self.gateway.generate_key(
KeyGenerateBody(key_alias=alias, models=models, user_id=f"e2e-{alias}")
KeyGenerateBody(
key_alias=alias,
models=models,
user_id=user_id or f"e2e-{alias}",
team_id=team_id,
organization_id=organization_id,
metadata=metadata,
)
)
def delete_key(self, key: str) -> None:
self.gateway.delete_key(key)
def create_team(
self,
alias: str,
*,
models: list[str],
organization_id: str | None = None,
) -> str:
return unwrap(
self.gateway.transport.post(
"/team/new",
headers=self.gateway.transport.master,
json=TeamNewBody(
team_alias=alias,
models=models,
organization_id=organization_id,
),
response_type=TeamNewResponse,
)
).team_id
def delete_team(self, team_id: str) -> None:
_ = self.gateway.transport.post(
"/team/delete",
headers=self.gateway.transport.master,
json=TeamDeleteBody(team_ids=[team_id]),
response_type=NoBody,
)
def create_user(self, *, user_email: str, user_id: str | None = None) -> str:
return unwrap(
self.gateway.transport.post(
"/user/new",
headers=self.gateway.transport.master,
json=UserNewBody(
user_email=user_email,
user_role="internal_user",
user_id=user_id,
),
response_type=UserNewResponse,
)
).user_id
def delete_user(self, user_id: str) -> None:
_ = self.gateway.transport.post(
"/user/delete",
headers=self.gateway.transport.master,
json=UserDeleteBody(user_ids=[user_id]),
response_type=NoBody,
)
def create_org(self, alias: str, *, models: list[str]) -> str:
return unwrap(
self.gateway.transport.post(
"/organization/new",
headers=self.gateway.transport.master,
json=OrgNewBody(organization_alias=alias, models=models),
response_type=OrgNewResponse,
)
).organization_id
def delete_org(self, organization_id: str) -> None:
_ = self.gateway.transport.delete(
"/organization/delete",
headers=self.gateway.transport.master,
json=OrgDeleteBody(organization_ids=[organization_id]),
response_type=NoBody,
)
def add_team_langfuse_callback(
self,
team_id: str,
creds: LangfuseCreds,
*,
callback_type: Literal["success", "failure", "success_and_failure"] = "success_and_failure",
) -> None:
response = unwrap(
self.gateway.transport.post(
f"/team/{team_id}/callback",
headers=self.gateway.transport.master,
json=TeamCallbackBody(
callback_name="langfuse_otel",
callback_type=callback_type,
callback_vars=creds.callback_vars(),
),
response_type=TeamCallbackResponse,
)
)
assert response.status == "success", (
f"POST /team/{team_id}/callback must return status=success; got {response.status!r}"
)
def create_tool_permission_guardrail(self, name: str, *, allowed_tool: str) -> str:
"""Register a tool_permission guardrail that allows one tool and denies the rest."""
response = unwrap(
self.gateway.transport.post(
"/guardrails",
headers=self.gateway.transport.master,
json=CreateGuardrailBody(
guardrail=GuardrailSpec(
guardrail_name=name,
litellm_params=GuardrailLitellmParams(
guardrail="tool_permission",
mode="post_call",
default_on=False,
default_action="deny",
on_disallowed_action="block",
rules=[
{
"id": "allow-named-tool",
"tool_name": allowed_tool,
"decision": "allow",
}
],
),
)
),
response_type=CreateGuardrailResponse,
)
)
guardrail_id = response.guardrail_id
assert guardrail_id, f"create guardrail returned no id: {response!r}"
return guardrail_id
def delete_guardrail(self, guardrail_id: str) -> None:
_ = self.gateway.transport.delete(
f"/guardrails/{guardrail_id}",
headers=self.gateway.transport.master,
json=NoBody(),
response_type=NoBody,
)
def create_model(self, model_name: str, litellm_params: LiteLLMParamsBody) -> str:
return self.gateway.create_model(model_name, litellm_params)
def delete_model(self, model_id: str) -> None:
self.gateway.delete_model(model_id)
def chat(self, key: str, model: str, text: str) -> ChatResponse:
return unwrap(
self.gateway.chat(
key,
ChatBody(
model=model,
messages=[ChatMessage(role="user", content=text)],
max_tokens=64,
messages=[ChatMessage(role="user", content=text)],
max_tokens=64,
),
)
)
def chat_raw(
self,
key: str,
model: str,
text: str,
*,
stream: bool = False,
tools: list[ChatTool] | None = None,
tool_choice: str | None = None,
guardrails: list[str] | None = None,
max_tokens: int = 64,
) -> StreamingResponse:
body = ChatBody(
model=model,
messages=[ChatMessage(role="user", content=text)],
max_tokens=max_tokens,
stream=stream,
tools=tools,
tool_choice=tool_choice,
guardrails=guardrails,
)
if stream:
return self.gateway.chat_stream(key, body)
return self.gateway.transport.send(
"/chat/completions",
headers=self.gateway.transport.bearer(key),
json=body,
)
def scrape_metrics(self) -> str:
return self.gateway.probe("/metrics", params=NoBody()).body
def poll_proxy_spend_for_key(
self,
key: str,
*,
response_id: str | None = None,
require_positive_spend: bool = True,
) -> SpendLogRow | None:
"""Poll /spend/logs by virtual key.
When ``response_id`` is set, only that SpendLogs.request_id may match.
When unset, any positive-spend row for the key is accepted. Never falls
back to an unmatched row; missing match returns None.
"""
def _matches(row: SpendLogRow) -> bool:
if response_id is not None and row.request_id != response_id:
return False
if require_positive_spend and not (row.spend is not None and row.spend > 0):
return False
return True
rows = self.gateway.poll_logs_for_key(
key, min_rows=1, predicate=lambda rs: any(_matches(r) for r in rs)
)
for row in rows:
if _matches(row):
return row
return None
def list_langfuse_observations(
self,
creds: LangfuseCreds,
*,
trace_id: str | None = None,
name: str | None = None,
from_start_time: str | None = None,
) -> list[LangfuseObservation]:
result = get(
URL(f"{creds.host}/api/public/observations"),
headers=creds.auth_headers,
params=LangfuseListParams(
limit=100,
trace_id=trace_id,
name=name,
from_start_time=from_start_time,
),
response_type=LangfuseObservationList,
timeout=30.0,
)
match result:
case Success(data=page):
return page.data
case _:
return []
def find_langfuse_observation(
self,
creds: LangfuseCreds,
*,
key_alias: str,
prompt_marker: str,
) -> LangfuseObservation | None:
# langfuse_otel generations are named litellm_request; classic SDK used
# litellm:{key_alias}. Search both, then a recent unfiltered page.
for name in ("litellm_request", f"litellm:{key_alias}"):
for obs in self.list_langfuse_observations(creds, name=name):
if _matches_run(obs, key_alias=key_alias, prompt_marker=prompt_marker):
return obs
for obs in self.list_langfuse_observations(creds):
if _matches_run(obs, key_alias=key_alias, prompt_marker=prompt_marker):
return obs
return None
def poll_langfuse_observation(
self,
creds: LangfuseCreds,
*,
key_alias: str,
prompt_marker: str,
require_positive_cost: bool = False,
) -> LangfuseObservation | None:
deadline = time.monotonic() + POLL_TIMEOUT
last: LangfuseObservation | None = None
while time.monotonic() < deadline:
last = self.find_langfuse_observation(
creds, key_alias=key_alias, prompt_marker=prompt_marker
)
if last is not None:
cost = observation_spend(last)
if not require_positive_cost or (cost is not None and cost > 0):
return last
time.sleep(POLL_INTERVAL)
return last
def poll_langfuse_trace_observations(
self,
creds: LangfuseCreds,
*,
key_alias: str,
prompt_marker: str,
) -> list[LangfuseObservation]:
"""Generation plus any sibling/child observations (guardrail spans, etc.)."""
gen = self.poll_langfuse_observation(
creds, key_alias=key_alias, prompt_marker=prompt_marker
)
if gen is None or not gen.trace_id:
return [] if gen is None else [gen]
return self.list_langfuse_observations(creds, trace_id=gen.trace_id) or [gen]
def build_logging_client() -> LoggingClient:
return LoggingClient(gateway=build_gateway())

534
tests/e2e/logging/test_langfuse_e2e.py Normal file
View file

@ -0,0 +1,534 @@
"""Live e2e: Langfuse OTEL logs_spend for registry cells in logging.yaml P0.
Registry cells:
- logging.langfuse.success.logs_spend (exercised_on chat_completions, messages, embeddings)
- logging.langfuse.failure.logs_spend (exercised_on chat_completions, messages)
- logging.langfuse.stream.logs_spend (exercised_on chat_completions, messages)
Integration under test is ``langfuse_otel`` (OTLP to Langfuse), not the classic
``langfuse`` SDK callback. StandardLoggingPayload.response_cost is the spend
source of truth. Generations are named ``litellm_request``; correlate by unique
prompt marker and user_api_key_alias in metadata.
Dynamic credentials by product surface:
- team: POST /team/{id}/callback with callback_name=langfuse_otel
- user/key: key metadata.logging with callback_name=langfuse_otel
- org: organization + team under it + team callback (no org-level callback API)
Extra success paths assert tool calls and applied guardrails land on the trace.
"""
from __future__ import annotations
import json
import pytest
from e2e_config import unique_marker
from e2e_http import StreamingResponse, require_successful_call
from lifecycle import ResourceManager
from logging_client import (
INVALID_UPSTREAM_API_KEY,
WEATHER_TOOL,
LangfuseCreds,
LoggingClient,
completion_response_id,
costs_agree,
observation_has_guardrail,
observation_mentions_tool,
observation_spend,
)
from models import LiteLLMParamsBody
pytestmark = pytest.mark.e2e
DRIVER_MODEL = "gemini-2.5-flash"
FAIL_BACKEND = "openai/gpt-4o-mini"
def _json_blob(value: object) -> str:
return json.dumps(value, default=str)
def _assert_logs_spend(
client: LoggingClient,
*,
key: str,
outcome: StreamingResponse,
obs_cost: float | None,
scope: str,
require_positive: bool = True,
) -> None:
"""logs_spend: Langfuse cost matches StandardLogging response_cost and proxy spend.
Non-stream responses expose response_cost on x-litellm-response-cost. Streaming
sends headers before final cost is known, so stream paths rely on /spend/logs.
"""
if not require_positive:
assert obs_cost is not None, (
f"{scope}: failure path must still track spend (0 is fine); cost={obs_cost!r}"
)
return
assert obs_cost is not None and obs_cost > 0, (
f"{scope}: Langfuse must log positive spend; calculatedTotalCost={obs_cost!r}"
)
# Stream responses send headers before final cost is known, so the cost header
# is often absent; non-stream must always expose x-litellm-response-cost.
if not outcome.is_streaming:
assert outcome.response_cost is not None and outcome.response_cost > 0, (
f"{scope}: proxy must return positive x-litellm-response-cost; "
f"got {outcome.response_cost!r}"
)
assert costs_agree(outcome.response_cost, obs_cost), (
f"{scope}: Langfuse cost {obs_cost!r} disagrees with "
f"x-litellm-response-cost {outcome.response_cost!r}"
)
elif outcome.response_cost is not None and outcome.response_cost > 0:
assert costs_agree(outcome.response_cost, obs_cost), (
f"{scope}: Langfuse cost {obs_cost!r} disagrees with "
f"x-litellm-response-cost {outcome.response_cost!r}"
)
spend_row = client.poll_proxy_spend_for_key(
key,
response_id=completion_response_id(outcome.body),
require_positive_spend=True,
)
assert spend_row is not None and spend_row.spend is not None and spend_row.spend > 0, (
f"{scope}: proxy /spend/logs never produced a positive spend row for key"
)
assert costs_agree(spend_row.spend, obs_cost), (
f"{scope}: Langfuse cost {obs_cost!r} disagrees with proxy spend "
f"{spend_row.spend!r} (request_id={spend_row.request_id!r})"
)
class TestLangfuseTeamLogging:
"""Team-scoped callback via POST /team/{id}/callback."""
def _team_key(
self,
client: LoggingClient,
resources: ResourceManager,
creds: LangfuseCreds,
*,
models: list[str],
organization_id: str | None = None,
) -> tuple[str, str, str]:
marker = unique_marker()
key_alias = f"e2e-lf-team-key-{marker}"
team_id = client.create_team(
f"e2e-lf-team-{marker}",
models=models,
organization_id=organization_id,
)
resources.defer(lambda: client.delete_team(team_id))
client.add_team_langfuse_callback(team_id, creds)
key = client.key_with_alias(key_alias, models=models, team_id=team_id)
resources.defer(lambda: client.delete_key(key))
return team_id, key, key_alias
@pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["chat_completions"])
def test_success_logs_spend(
self,
client: LoggingClient,
resources: ResourceManager,
langfuse_creds: LangfuseCreds,
) -> None:
_, key, key_alias = self._team_key(
client, resources, langfuse_creds, models=[DRIVER_MODEL]
)
prompt_marker = unique_marker()
outcome = client.chat_raw(
key, DRIVER_MODEL, f"reply with one word only {prompt_marker}"
)
require_successful_call(outcome)
obs = client.poll_langfuse_observation(
langfuse_creds,
key_alias=key_alias,
prompt_marker=prompt_marker,
require_positive_cost=True,
)
assert obs is not None, (
f"team scope: Langfuse never received generation for key_alias={key_alias!r}"
)
_assert_logs_spend(
client,
key=key,
outcome=outcome,
obs_cost=observation_spend(obs),
scope="team-success",
)
@pytest.mark.covers("logging.langfuse.failure.logs_spend", exercised_on=["chat_completions"])
def test_failure_logs_spend(
self,
client: LoggingClient,
resources: ResourceManager,
langfuse_creds: LangfuseCreds,
) -> None:
"""Provider-auth failure still ships a Langfuse observation with spend tracked.
Uses a throwaway deployment whose upstream OpenAI key is
INVALID_UPSTREAM_API_KEY (not a LiteLLM virtual key).
"""
prompt_marker = unique_marker()
model_name = f"e2e-lf-fail-{prompt_marker}"
model_id = client.create_model(
model_name,
LiteLLMParamsBody(model=FAIL_BACKEND, api_key=INVALID_UPSTREAM_API_KEY),
)
resources.defer(lambda: client.delete_model(model_id))
_, key, key_alias = self._team_key(
client, resources, langfuse_creds, models=[model_name]
)
outcome = client.chat_raw(key, model_name, f"this must fail {prompt_marker}")
assert not outcome.ok, (
f"expected upstream provider failure for {INVALID_UPSTREAM_API_KEY!r}, "
f"got {outcome.status_code}: {outcome.body[:200]}"
)
obs = client.poll_langfuse_observation(
langfuse_creds,
key_alias=key_alias,
prompt_marker=prompt_marker,
require_positive_cost=False,
)
assert obs is not None, (
f"team failure path: Langfuse never received generation for key_alias={key_alias!r}"
)
_assert_logs_spend(
client,
key=key,
outcome=outcome,
obs_cost=observation_spend(obs),
scope="team-failure",
require_positive=False,
)
@pytest.mark.covers("logging.langfuse.stream.logs_spend", exercised_on=["chat_completions"])
def test_stream_logs_spend(
self,
client: LoggingClient,
resources: ResourceManager,
langfuse_creds: LangfuseCreds,
) -> None:
_, key, key_alias = self._team_key(
client, resources, langfuse_creds, models=[DRIVER_MODEL]
)
prompt_marker = unique_marker()
outcome = client.chat_raw(
key, DRIVER_MODEL, f"reply with one word only {prompt_marker}", stream=True
)
require_successful_call(outcome)
assert outcome.is_streaming
assert outcome.chunks > 0
obs = client.poll_langfuse_observation(
langfuse_creds,
key_alias=key_alias,
prompt_marker=prompt_marker,
require_positive_cost=True,
)
assert obs is not None
# Streamed body is elided; correlate cost via header + key spend row.
_assert_logs_spend(
client,
key=key,
outcome=outcome,
obs_cost=observation_spend(obs),
scope="team-stream",
)
@pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["chat_completions"])
def test_tool_calls_logged_with_cost(
self,
client: LoggingClient,
resources: ResourceManager,
langfuse_creds: LangfuseCreds,
) -> None:
_, key, key_alias = self._team_key(
client, resources, langfuse_creds, models=[DRIVER_MODEL]
)
prompt_marker = unique_marker()
outcome = client.chat_raw(
key,
DRIVER_MODEL,
f"Use get_weather for Paris. marker={prompt_marker}",
tools=[WEATHER_TOOL],
tool_choice="required",
max_tokens=128,
)
require_successful_call(outcome)
assert "get_weather" in outcome.body or "tool_calls" in outcome.body, (
f"gateway response must include a tool call; body={outcome.body[:300]}"
)
obs = client.poll_langfuse_observation(
langfuse_creds,
key_alias=key_alias,
prompt_marker=prompt_marker,
require_positive_cost=True,
)
assert obs is not None
assert observation_mentions_tool(obs, "get_weather"), (
f"Langfuse generation must record the tool; name={obs.name!r} "
f"input={str(obs.input)[:200]} output={str(obs.output)[:200]}"
)
_assert_logs_spend(
client,
key=key,
outcome=outcome,
obs_cost=observation_spend(obs),
scope="team-tools",
)
@pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["chat_completions"])
def test_tool_permission_guardrail_logged(
self,
client: LoggingClient,
resources: ResourceManager,
langfuse_creds: LangfuseCreds,
) -> None:
"""tool_permission post_call guardrail must appear on the Langfuse trace
(StandardLogging guardrail_information -> Langfuse guardrail span)."""
marker = unique_marker()
guardrail_name = f"e2e-lf-tool-perm-{marker}"
guardrail_id = client.create_tool_permission_guardrail(
guardrail_name, allowed_tool="get_weather"
)
resources.defer(lambda: client.delete_guardrail(guardrail_id))
_, key, key_alias = self._team_key(
client, resources, langfuse_creds, models=[DRIVER_MODEL]
)
prompt_marker = unique_marker()
outcome = client.chat_raw(
key,
DRIVER_MODEL,
f"Use get_weather for Berlin. marker={prompt_marker}",
tools=[WEATHER_TOOL],
tool_choice="required",
guardrails=[guardrail_name],
max_tokens=128,
)
require_successful_call(outcome)
observations = client.poll_langfuse_trace_observations(
langfuse_creds, key_alias=key_alias, prompt_marker=prompt_marker
)
assert observations, (
f"team+guardrail: no Langfuse observations for key_alias={key_alias!r}"
)
gen = next(
(
o
for o in observations
if prompt_marker in _json_blob(o.input)
or key_alias in _json_blob(o.metadata)
or o.name in (f"litellm:{key_alias}", "litellm_request")
),
observations[0],
)
_assert_logs_spend(
client,
key=key,
outcome=outcome,
obs_cost=observation_spend(gen),
scope="team-guardrail",
)
assert any(
observation_has_guardrail(o, guardrail_name=guardrail_name)
or (o.name is not None and "guardrail" in o.name.lower())
for o in observations
), (
f"Langfuse trace must include applied guardrail {guardrail_name!r}; "
f"observation names={[o.name for o in observations]}"
)
class TestLangfuseUserKeyLogging:
"""User-owned key with metadata.logging (key-level dynamic Langfuse credentials).
Product surface: key metadata.logging on /key/generate, not a separate
/user/.../callback route. The key is bound to a real /user/new user_id.
"""
def _user_key(
self,
client: LoggingClient,
resources: ResourceManager,
creds: LangfuseCreds,
*,
models: list[str],
) -> tuple[str, str, str]:
marker = unique_marker()
key_alias = f"e2e-lf-user-key-{marker}"
user_id = client.create_user(
user_email=f"e2e-lf-user-{marker}@example.com",
user_id=f"e2e-lf-user-{marker}",
)
resources.defer(lambda: client.delete_user(user_id))
key = client.key_with_alias(
key_alias,
models=models,
user_id=user_id,
metadata=creds.key_logging_metadata(),
)
resources.defer(lambda: client.delete_key(key))
return user_id, key, key_alias
@pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["chat_completions"])
def test_success_logs_spend(
self,
client: LoggingClient,
resources: ResourceManager,
langfuse_creds: LangfuseCreds,
) -> None:
user_id, key, key_alias = self._user_key(
client, resources, langfuse_creds, models=[DRIVER_MODEL]
)
prompt_marker = unique_marker()
outcome = client.chat_raw(
key, DRIVER_MODEL, f"reply with one word only {prompt_marker}"
)
require_successful_call(outcome)
obs = client.poll_langfuse_observation(
langfuse_creds,
key_alias=key_alias,
prompt_marker=prompt_marker,
require_positive_cost=True,
)
assert obs is not None, (
f"user/key scope: Langfuse never received generation for key_alias={key_alias!r}"
)
meta_blob = _json_blob(obs.metadata)
assert user_id in meta_blob or key_alias in (obs.name or ""), (
f"user/key scope should attribute the user or key; metadata={meta_blob[:300]}"
)
_assert_logs_spend(
client,
key=key,
outcome=outcome,
obs_cost=observation_spend(obs),
scope="user-key",
)
@pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["chat_completions"])
def test_tool_calls_logged_with_cost(
self,
client: LoggingClient,
resources: ResourceManager,
langfuse_creds: LangfuseCreds,
) -> None:
_, key, key_alias = self._user_key(
client, resources, langfuse_creds, models=[DRIVER_MODEL]
)
prompt_marker = unique_marker()
outcome = client.chat_raw(
key,
DRIVER_MODEL,
f"Use get_weather for Tokyo. marker={prompt_marker}",
tools=[WEATHER_TOOL],
tool_choice="required",
max_tokens=128,
)
require_successful_call(outcome)
obs = client.poll_langfuse_observation(
langfuse_creds,
key_alias=key_alias,
prompt_marker=prompt_marker,
require_positive_cost=True,
)
assert obs is not None
assert observation_mentions_tool(obs, "get_weather"), (
f"user/key tool path: tool missing from Langfuse; output={str(obs.output)[:200]}"
)
_assert_logs_spend(
client,
key=key,
outcome=outcome,
obs_cost=observation_spend(obs),
scope="user-key-tools",
)
class TestLangfuseOrgScopedLogging:
"""Org-scoped run: organization + team under it + team Langfuse callback.
There is no /organization/.../callback today; logging attaches at the team
(or key) under the org. This class proves org-linked team keys still deliver
accurate Langfuse spend and team attribution (StandardLogging metadata
user_api_key_team_id / user_api_key_org_id).
"""
def _org_team_key(
self,
client: LoggingClient,
resources: ResourceManager,
creds: LangfuseCreds,
*,
models: list[str],
) -> tuple[str, str, str, str]:
marker = unique_marker()
key_alias = f"e2e-lf-org-key-{marker}"
org_id = client.create_org(f"e2e-lf-org-{marker}", models=models)
resources.defer(lambda: client.delete_org(org_id))
team_id = client.create_team(
f"e2e-lf-org-team-{marker}",
models=models,
organization_id=org_id,
)
resources.defer(lambda: client.delete_team(team_id))
client.add_team_langfuse_callback(team_id, creds)
key = client.key_with_alias(
key_alias,
models=models,
team_id=team_id,
organization_id=org_id,
)
resources.defer(lambda: client.delete_key(key))
return org_id, team_id, key, key_alias
@pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["chat_completions"])
def test_success_logs_spend_with_team_attribution(
self,
client: LoggingClient,
resources: ResourceManager,
langfuse_creds: LangfuseCreds,
) -> None:
org_id, team_id, key, key_alias = self._org_team_key(
client, resources, langfuse_creds, models=[DRIVER_MODEL]
)
prompt_marker = unique_marker()
outcome = client.chat_raw(
key, DRIVER_MODEL, f"reply with one word only {prompt_marker}"
)
require_successful_call(outcome)
obs = client.poll_langfuse_observation(
langfuse_creds,
key_alias=key_alias,
prompt_marker=prompt_marker,
require_positive_cost=True,
)
assert obs is not None, (
f"org scope: Langfuse never received generation for key_alias={key_alias!r}"
)
meta_blob = _json_blob(obs.metadata)
assert team_id in meta_blob, (
f"org-scoped team key must stamp team_id on Langfuse metadata; "
f"team_id={team_id!r} metadata={meta_blob[:400]}"
)
_ = org_id
_assert_logs_spend(
client,
key=key,
outcome=outcome,
obs_cost=observation_spend(obs),
scope="org-team",
)

6
tests/e2e/management/management_client.py
View file

@ -75,7 +75,7 @@ class ManagementClient:
case _:
break
assert last is not None
_ = unwrap(last)
raise AssertionError(last)
def delete_key_strict(self, key: str) -> None:
"""Strict delete for the act phase of a test: a failed delete is a hard
@ -147,7 +147,7 @@ class ManagementClient:
case _:
time.sleep(_TEAM_READY_SLEEP_SECONDS)
assert last is not None
_ = unwrap(last)
raise AssertionError(last)
def add_team_member(self, team_id: str, user_id: str) -> None:
last: Result[NoBody] | None = None
@ -169,7 +169,7 @@ class ManagementClient:
case _:
break
assert last is not None
_ = unwrap(last)
raise AssertionError(last)
def delete_team_member(self, team_id: str, user_id: str) -> None:
_ = unwrap(

37
tests/e2e/models.py
View file

@ -23,6 +23,22 @@ class BudgetWindow(BaseModel):
max_budget: float
class KeyLoggingCallbackVars(BaseModel):
langfuse_public_key: str | None = None
langfuse_secret_key: str | None = None
langfuse_host: str | None = None
class KeyLoggingCallback(BaseModel):
callback_name: str
callback_type: str = "success_and_failure"
callback_vars: KeyLoggingCallbackVars
class KeyMetadata(BaseModel):
logging: list[KeyLoggingCallback] | None = None
class KeyGenerateBody(BaseModel):
models: list[str] = []
duration: str | None = None
@ -31,6 +47,7 @@ class KeyGenerateBody(BaseModel):
budget_duration: str | None = None
user_id: str | None = None
team_id: str | None = None
organization_id: str | None = None
budget_id: str | None = None
key_alias: str | None = None
model_max_budget: dict[str, ModelBudgetEntry] | None = None
@ -39,6 +56,7 @@ class KeyGenerateBody(BaseModel):
tpm_limit: int | None = None
rpm_limit: int | None = None
allowed_routes: list[str] | None = None
metadata: KeyMetadata | None = None
class KeyGenerateResponse(BaseModel):
@ -105,6 +123,17 @@ class ThinkingParam(BaseModel):
budget_tokens: int | None = None
class ChatToolFunction(BaseModel):
name: str
description: str | None = None
parameters: dict[str, object] | None = None
class ChatTool(BaseModel):
type: str = "function"
function: ChatToolFunction
class ChatBody(BaseModel):
model: str
messages: list[ChatMessage]
@ -115,6 +144,9 @@ class ChatBody(BaseModel):
reasoning_effort: str | None = None
thinking: ThinkingParam | None = None
service_tier: str | None = None
tools: list[ChatTool] | None = None
tool_choice: str | None = None
guardrails: list[str] | None = None
class AnthropicMessagesBody(BaseModel):
@ -123,6 +155,10 @@ class AnthropicMessagesBody(BaseModel):
max_tokens: int
class AnthropicMessagesResponse(BaseModel):
model: str | None = None
class OutMessage(BaseModel):
content: str | None = None
reasoning_content: str | None = None
@ -459,6 +495,7 @@ class TeamNewBody(BaseModel):
team_alias: str
models: list[str] = []
team_id: str | None = None
organization_id: str | None = None
class TeamNewResponse(BaseModel):

10
tests/local_testing/test_get_llm_provider.py
View file

@ -509,10 +509,10 @@ def shipped_generalizations():
class TestClaudeModelPatternMatching:
"""
The ``anthropic-claude`` fallback generalization rule routes future Claude
models to the Anthropic provider without requiring a
The ``anthropic-claude-ids`` fallback generalization routing rule routes future
Claude models to the Anthropic provider without requiring a
model_prices_and_context_window.json entry. These tests exercise the rule
end-to-end through ``get_llm_provider`` and ``match_fallback_generalization``.
end-to-end through ``get_llm_provider`` and ``match_routing_generalization``.
"""
@pytest.mark.parametrize(
@ -556,10 +556,10 @@ class TestClaudeModelPatternMatching:
self, model, shipped_generalizations
):
from litellm.litellm_core_utils.fallback_generalizations import (
match_fallback_generalization,
match_routing_generalization,
)
assert match_fallback_generalization(model) is None
assert match_routing_generalization(model) is None
def test_routing_comes_from_the_rule_not_python(self, shipped_generalizations):
"""With the rule cleared, an unknown claude must no longer route to

616
tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py
View file

@ -1,11 +1,13 @@
"""
Tests for the declarative fallback-generalizations mechanism.
Covers both the pure module (litellm.litellm_core_utils.fallback_generalizations)
and its end-to-end wiring into provider routing (get_llm_provider) and model-info
resolution (get_model_info / supports_*).
Covers the pure module (litellm.litellm_core_utils.fallback_generalizations): the
routing/capability rule split, install-time validation, capability unioning; and
its end-to-end wiring into provider routing (get_llm_provider) and model-info
resolution (get_model_info) including the shipped rules in the bundled cost map.
"""
import logging
import os
import sys
@ -14,10 +16,11 @@ import pytest
sys.path.insert(0, os.path.abspath("../../.."))
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.fallback_generalizations import (
get_fallback_generalization_rules,
match_all_fallback_generalizations,
match_fallback_generalization,
match_capability_generalizations,
match_routing_generalization,
set_fallback_generalizations,
)
@ -32,92 +35,205 @@ def restore_generalizations():
set_fallback_generalizations(previous)
class _RecordingHandler(logging.Handler):
def __init__(self):
super().__init__(level=logging.WARNING)
self.messages = []
def emit(self, record):
self.messages.append(record.getMessage())
@pytest.fixture
def warning_messages():
handler = _RecordingHandler()
previous_level = verbose_logger.level
verbose_logger.setLevel(logging.WARNING)
verbose_logger.addHandler(handler)
try:
yield handler.messages
finally:
verbose_logger.removeHandler(handler)
verbose_logger.setLevel(previous_level)
# --------------------------------------------------------------------------- #
# Pure module behaviour
# Engine: routing rules
# --------------------------------------------------------------------------- #
def test_match_returns_model_info_of_first_matching_rule(restore_generalizations):
def test_routing_inference_first_match_wins(restore_generalizations):
restore_generalizations(
[
{"name": "first", "pattern": r"^acme-", "model_info": {"litellm_provider": "openai"}},
{"name": "second", "pattern": r"^acme-pro-", "model_info": {"litellm_provider": "anthropic"}},
]
)
assert match_routing_generalization("acme-pro-1") == "openai"
assert match_routing_generalization("gpt-4o") is None
assert match_routing_generalization("") is None
def test_capability_rules_do_not_route(restore_generalizations):
restore_generalizations([{"name": "caps", "pattern": r"^acme-", "model_info": {"supports_vision": True}}])
assert match_routing_generalization("acme-pro-1") is None
def test_routing_match_is_case_insensitive(restore_generalizations):
restore_generalizations(
[{"name": "r", "pattern": r"^claude-opus", "model_info": {"litellm_provider": "anthropic"}}]
)
assert match_routing_generalization("CLAUDE-OPUS-9-9") == "anthropic"
# --------------------------------------------------------------------------- #
# Engine: capability rules
# --------------------------------------------------------------------------- #
def test_capability_union_is_last_wins_in_file_order(restore_generalizations):
restore_generalizations(
[
{
"name": "first",
"name": "broad",
"pattern": r"^acme-",
"model_info": {"litellm_provider": "openai", "tag": "first"},
"model_info": {"mode": "chat", "supports_vision": True, "max_input_tokens": 1000},
},
{
"name": "second",
"name": "narrow",
"pattern": r"^acme-pro-",
"model_info": {"litellm_provider": "anthropic", "tag": "second"},
"model_info": {"supports_vision": False, "supports_reasoning": True},
},
]
)
# Both rules match "acme-pro-1"; first-in-list wins (documented precedence).
matched = match_fallback_generalization("acme-pro-1")
assert matched is not None
assert matched["tag"] == "first"
assert match_capability_generalizations("acme-pro-1") == {
"mode": "chat",
"supports_vision": False,
"max_input_tokens": 1000,
"supports_reasoning": True,
}
assert match_capability_generalizations("acme-basic-1") == {
"mode": "chat",
"supports_vision": True,
"max_input_tokens": 1000,
}
def test_match_all_returns_every_matching_rule_in_order(restore_generalizations):
def test_routing_rules_are_excluded_from_capability_results(restore_generalizations):
restore_generalizations(
[
{
"name": "first",
"pattern": r"^acme-",
"model_info": {"litellm_provider": "openai", "tag": "first"},
},
{
"name": "second",
"pattern": r"^acme-pro-",
"model_info": {"litellm_provider": "anthropic", "tag": "second"},
},
{"name": "route", "pattern": r"^acme-", "model_info": {"litellm_provider": "openai"}},
{"name": "caps", "pattern": r"^acme-pro-", "model_info": {"supports_vision": True}},
]
)
assert [m["tag"] for m in match_all_fallback_generalizations("acme-pro-1")] == ["first", "second"]
assert match_all_fallback_generalizations("gpt-4o") == []
assert match_capability_generalizations("acme-pro-1") == {"supports_vision": True}
assert match_capability_generalizations("acme-basic-1") is None
def test_provider_scoped_rule_is_skipped_for_other_providers(restore_generalizations):
"""Model-info resolution must fall through a provider-mismatched earlier rule to a
later applicable one, instead of discarding the model name at the first pattern hit."""
restore_generalizations(
[
{
"name": "bedrock-scoped",
"pattern": r"^acme-",
"model_info": {"litellm_provider": "bedrock", "supports_vision": False},
},
{
"name": "openai-scoped",
"pattern": r"^acme-",
"model_info": {"litellm_provider": "openai", "mode": "chat", "supports_vision": True},
},
]
)
litellm.get_model_info.cache_clear()
info = litellm.get_model_info("acme-fast-1", custom_llm_provider="openai")
assert info["litellm_provider"] == "openai"
assert info["supports_vision"] is True
def test_match_is_case_insensitive(restore_generalizations):
restore_generalizations(
[{"name": "r", "pattern": r"^claude-opus", "model_info": {"ok": True}}]
)
assert match_fallback_generalization("CLAUDE-OPUS-9-9") == {"ok": True}
def test_no_match_returns_none(restore_generalizations):
restore_generalizations(
[{"name": "r", "pattern": r"^claude-", "model_info": {"ok": True}}]
)
assert match_fallback_generalization("gpt-4o") is None
assert match_fallback_generalization("") is None
def test_empty_rules_match_nothing(restore_generalizations):
def test_no_capability_match_returns_none(restore_generalizations):
restore_generalizations([{"name": "r", "pattern": r"^claude-", "model_info": {"ok": True}}])
assert match_capability_generalizations("gpt-4o") is None
assert match_capability_generalizations("") is None
restore_generalizations([])
assert match_fallback_generalization("claude-opus-9-9") is None
assert match_capability_generalizations("claude-opus-9-9") is None
def test_reinstalling_rules_replaces_compiled_rules(restore_generalizations):
restore_generalizations([{"name": "r", "pattern": r"^aaa", "model_info": {"v": 1}}])
assert match_capability_generalizations("aaa-1") == {"v": 1}
set_fallback_generalizations([{"name": "r", "pattern": r"^bbb", "model_info": {"v": 2}}])
assert match_capability_generalizations("aaa-1") is None
assert match_capability_generalizations("bbb-1") == {"v": 2}
# --------------------------------------------------------------------------- #
# Engine: install-time validation and legacy-schema shim
# --------------------------------------------------------------------------- #
def test_legacy_mixed_rule_acts_as_both_kinds(restore_generalizations):
"""A legacy rule mixing ``litellm_provider`` with capability keys routes AND
contributes its full model_info (provider included) to the capability union."""
restore_generalizations(
[
{
"name": "legacy-mixed",
"pattern": r"^acme-",
"model_info": {"litellm_provider": "anthropic", "supports_vision": True},
},
{"name": "new-caps", "pattern": r"^acme-pro-", "model_info": {"supports_reasoning": True}},
]
)
assert match_routing_generalization("acme-pro-1") == "anthropic"
assert match_capability_generalizations("acme-pro-1") == {
"litellm_provider": "anthropic",
"supports_vision": True,
"supports_reasoning": True,
}
LEGACY_MAIN_RULES = [
{
"name": "anthropic-claude-adaptive-thinking",
"pattern": "(?:opus|sonnet|haiku)[-._](?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d{1,})[-._]\\d{1,2}(?!\\d))",
"description": "Claude opus/sonnet/haiku at version 4.6 or higher: 4.6 through 4.99, then any 5.x, 6.x or later major. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new families with no code change.",
"extends": "anthropic-claude",
"model_info": {"supports_adaptive_thinking": True},
},
{
"name": "anthropic-claude",
"pattern": "^claude-[a-z]+-\\d+[-.]\\d+(?:-\\d{8})?$",
"description": "Any Claude family-major-minor id, optionally with an 8-digit date suffix, anchored to the whole name. Version-neutral fallback that gives an unmapped Claude provider routing and baseline capabilities; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.",
"model_info": {
"litellm_provider": "anthropic",
"mode": "chat",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"supports_function_calling": True,
"supports_parallel_function_calling": True,
"supports_vision": True,
"supports_tool_choice": True,
"supports_assistant_prefill": True,
"supports_prompt_caching": True,
"supports_response_schema": True,
"supports_reasoning": True,
"supports_pdf_input": True,
"supports_system_messages": True,
},
},
]
def test_legacy_main_schema_keeps_unmapped_claude_working(restore_generalizations):
"""Pins the remote-map transition window: a released proxy running this engine
against main's old-schema block (mixed provider+capability rule plus ``extends``,
copied verbatim above) must keep unmapped-Claude inference and info resolution
working until the new-schema JSON reaches main."""
restore_generalizations([dict(rule) for rule in LEGACY_MAIN_RULES])
litellm.get_model_info.cache_clear()
_, provider, _, _ = litellm.get_llm_provider(model="claude-opus-9-9")
assert provider == "anthropic"
info = litellm.get_model_info("claude-opus-9-9")
assert info["litellm_provider"] == "anthropic"
assert info["supports_adaptive_thinking"] is True
assert info["supports_function_calling"] is True
assert info["max_input_tokens"] == 200000
assert not info.get("input_cost_per_token")
low = litellm.get_model_info("claude-opus-4-0")
assert low["litellm_provider"] == "anthropic"
assert low["supports_function_calling"] is True
assert low.get("supports_adaptive_thinking") is None
def test_non_string_provider_rule_warns_and_is_skipped(restore_generalizations, warning_messages):
restore_generalizations([{"name": "bad-provider", "pattern": r"^acme-", "model_info": {"litellm_provider": 42}}])
assert any("bad-provider" in message for message in warning_messages)
assert match_routing_generalization("acme-1") is None
assert match_capability_generalizations("acme-1") is None
def test_malformed_rules_are_skipped_not_fatal(restore_generalizations):
@ -131,69 +247,7 @@ def test_malformed_rules_are_skipped_not_fatal(restore_generalizations):
{"name": "good", "pattern": r"^claude-", "model_info": {"good": True}},
]
)
# Non-dict entries and dicts with bad fields are all skipped; the one
# valid rule still matches.
assert match_fallback_generalization("claude-opus-9-9") == {"good": True}
def test_setting_rules_invalidates_compiled_cache(restore_generalizations):
restore_generalizations([{"name": "r", "pattern": r"^aaa", "model_info": {"v": 1}}])
assert match_fallback_generalization("aaa-1") == {"v": 1}
# Re-install different rules; the compiled cache must be rebuilt.
set_fallback_generalizations(
[{"name": "r", "pattern": r"^bbb", "model_info": {"v": 2}}]
)
assert match_fallback_generalization("aaa-1") is None
assert match_fallback_generalization("bbb-1") == {"v": 2}
def test_extends_inherits_parent_and_own_overrides(restore_generalizations):
"""A rule's ``extends`` pulls in the parent's model_info; its own keys win on conflict,
so a narrow rule carries only its delta instead of duplicating the parent."""
restore_generalizations(
[
{
"name": "base",
"pattern": r"^base-only$",
"model_info": {
"litellm_provider": "anthropic",
"input_cost_per_token": 5e-06,
"supports_vision": True,
},
},
{
"name": "child",
"pattern": r"^kid-",
"extends": "base",
"model_info": {
"supports_adaptive_thinking": True,
"supports_vision": False,
},
},
]
)
matched = match_fallback_generalization("kid-1")
assert matched == {
"litellm_provider": "anthropic",
"input_cost_per_token": 5e-06,
"supports_vision": False,
"supports_adaptive_thinking": True,
}
def test_extends_with_unknown_parent_keeps_own_model_info(restore_generalizations):
"""A dangling ``extends`` is non-fatal: the rule resolves to its own model_info."""
restore_generalizations(
[
{
"name": "orphan",
"pattern": r"^orphan-",
"extends": "does-not-exist",
"model_info": {"litellm_provider": "openai"},
}
]
)
assert match_fallback_generalization("orphan-1") == {"litellm_provider": "openai"}
assert match_capability_generalizations("claude-opus-9-9") == {"good": True}
# --------------------------------------------------------------------------- #
@ -201,91 +255,61 @@ def test_extends_with_unknown_parent_keeps_own_model_info(restore_generalization
# --------------------------------------------------------------------------- #
@pytest.fixture
def myco_rule(restore_generalizations):
"""A self-contained rule carrying provider, pricing, context and capabilities."""
def test_unknown_model_routes_via_routing_rule(restore_generalizations):
restore_generalizations([{"name": "myco", "pattern": r"^myco-", "model_info": {"litellm_provider": "openai"}}])
_, provider, _, _ = litellm.get_llm_provider(model="myco-fast-1")
assert provider == "openai"
def test_capability_info_backfills_requested_provider(restore_generalizations):
restore_generalizations(
[
{
"name": "myco",
"pattern": r"^myco-[a-z]+-\d+$",
"name": "beeco-caps",
"pattern": r"^beeco-[a-z]+-\d+$",
"model_info": {
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token": 1e-06,
"output_cost_per_token": 2e-06,
"max_input_tokens": 12345,
"max_output_tokens": 678,
"supports_vision": True,
"supports_function_calling": True,
},
}
]
)
return "myco-fast-1"
def test_unknown_model_routes_via_rule(myco_rule):
_, provider, _, _ = litellm.get_llm_provider(model=myco_rule)
assert provider == "openai"
def test_unknown_model_gets_pricing_context_and_capabilities(myco_rule):
info = litellm.get_model_info(myco_rule)
assert info["litellm_provider"] == "openai"
assert info["input_cost_per_token"] == 1e-06
assert info["output_cost_per_token"] == 2e-06
litellm.get_model_info.cache_clear()
info = litellm.get_model_info("beeco-fast-1", custom_llm_provider="groq")
assert info["litellm_provider"] == "groq"
assert info["max_input_tokens"] == 12345
assert info["supports_vision"] is True
other = litellm.get_model_info("beeco-fast-1", custom_llm_provider="openai")
assert other["litellm_provider"] == "openai"
def test_supports_helper_reads_through_generalization(myco_rule):
assert litellm.supports_vision(myco_rule) is True
assert litellm.supports_function_calling(myco_rule) is True
def test_routing_only_match_does_not_resolve_model_info(restore_generalizations):
restore_generalizations([{"name": "route", "pattern": r"^ceeco-", "model_info": {"litellm_provider": "openai"}}])
litellm.get_model_info.cache_clear()
with pytest.raises(Exception):
litellm.get_model_info("ceeco-fast-1", custom_llm_provider="openai")
def test_exact_entry_takes_precedence_over_rule(restore_generalizations):
"""An exact cost-map entry must win over a rule that also matches it."""
restore_generalizations(
[
{
"name": "shadow-gpt4o",
"pattern": r"^gpt-4o$",
"model_info": {
"litellm_provider": "anthropic",
"input_cost_per_token": 999.0,
},
}
]
[{"name": "shadow-gpt4o", "pattern": r"^gpt-4o$", "model_info": {"input_cost_per_token": 999.0}}]
)
litellm.get_model_info.cache_clear()
info = litellm.get_model_info("gpt-4o")
# Resolved from the real exact entry, not the shadowing rule.
assert info["litellm_provider"] == "openai"
assert info["input_cost_per_token"] != 999.0
def test_unknown_model_without_matching_rule_still_unmapped(restore_generalizations):
restore_generalizations(
[
{
"name": "claude",
"pattern": r"^claude-",
"model_info": {"litellm_provider": "anthropic"},
}
]
)
with pytest.raises(Exception):
litellm.get_model_info("totally-unknown-model-xyz")
# --------------------------------------------------------------------------- #
# Shipped anthropic-claude rule
# Shipped rules (bundled cost map)
# --------------------------------------------------------------------------- #
@pytest.fixture
def shipped_cost_map(monkeypatch):
"""Activate the bundled cost map so the shipped anthropic-claude rule is installed."""
"""Activate the bundled cost map so the shipped rules are installed."""
original_cost = litellm.model_cost
previous_rules = list(get_fallback_generalization_rules())
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
@ -299,40 +323,197 @@ def shipped_cost_map(monkeypatch):
set_fallback_generalizations(previous_rules)
def test_shipped_rule_marks_unmapped_high_version_claude_adaptive_without_pricing(
shipped_cost_map,
):
"""An unmapped Claude >= 4.6 resolves via the version-gated adaptive-thinking rule, which
inherits routing and capabilities from the base rule and adds ``supports_adaptive_thinking``.
The rule carries no pricing, so cost stays unpriced (zero, not a fabricated number) rather
than reporting a confidently-wrong price."""
model = "claude-opus-9-9"
def test_shipped_bare_claude_id_routes_to_anthropic(shipped_cost_map):
_, provider, _, _ = litellm.get_llm_provider(model="claude-haiku-4-6")
assert provider == "anthropic"
def test_shipped_bedrock_syntax_claude_id_routes_to_bedrock(shipped_cost_map):
"""Regression: a bedrock-syntax id must infer bedrock even when its version also
matches an unanchored Anthropic capability pattern. The old first-match-wins engine
routed global.anthropic.claude-haiku-4-6 to anthropic via the adaptive rule."""
for model in [
"global.anthropic.claude-haiku-4-6",
"us.anthropic.claude-haiku-4-6",
"anthropic.claude-haiku-4-6",
"eu.anthropic.claude-opus-5-0",
]:
assert model not in litellm.model_cost
_, provider, _, _ = litellm.get_llm_provider(model=model)
assert provider == "bedrock", model
def test_shipped_rules_resolve_unmapped_bedrock_claude_with_bedrock_provider(shipped_cost_map):
model = "us.anthropic.claude-haiku-4-6"
assert model not in litellm.model_cost
info = litellm.get_model_info(model)
assert info["litellm_provider"] == "anthropic"
info = litellm.get_model_info(model, custom_llm_provider="bedrock")
assert info["litellm_provider"] == "bedrock"
assert info["supports_adaptive_thinking"] is True
assert info["supports_function_calling"] is True
assert info["max_input_tokens"] == 200000
assert info.get("supports_mid_conversation_system") is None
assert not info.get("input_cost_per_token")
assert not info.get("output_cost_per_token")
def test_shipped_rule_resolves_unmapped_low_version_claude_without_adaptive(shipped_cost_map):
"""An unmapped Claude < 4.6 falls through to the version-neutral anthropic-claude rule: it
gets provider routing and baseline capabilities but no ``supports_adaptive_thinking`` flag,
so a sub-4.6 alias such as ``claude-opus-4-0`` resolves yet is never marked adaptive."""
model = "claude-opus-4-0"
def test_shipped_rules_stack_adaptive_and_mid_conversation_flags(shipped_cost_map):
model = "claude-opus-4-9"
assert model not in litellm.model_cost
info = litellm.get_model_info(model, custom_llm_provider="anthropic")
assert info["litellm_provider"] == "anthropic"
assert info["supports_adaptive_thinking"] is True
assert info["supports_mid_conversation_system"] is True
assert info["supports_function_calling"] is True
@pytest.mark.parametrize(
"model,provider",
[
("claude-opus-4-9@20260101", "vertex_ai"),
("databricks-claude-opus-5-1", "databricks"),
],
)
def test_shipped_rules_are_provider_neutral_for_unmapped_ids(shipped_cost_map, model, provider):
assert model not in litellm.model_cost
info = litellm.get_model_info(model, custom_llm_provider=provider)
assert info["litellm_provider"] == provider
assert info["supports_adaptive_thinking"] is True
assert info["supports_mid_conversation_system"] is True
assert info["supports_function_calling"] is True
@pytest.mark.parametrize(
"model,provider,adaptive,mid_conversation",
[
("us.anthropic.claude-opus-4-5", "bedrock", None, None),
("claude-haiku-4-6", "anthropic", True, None),
("claude-haiku-4-7", "anthropic", True, None),
("claude-haiku-4-8", "anthropic", True, True),
("claude-haiku-4-9", "anthropic", True, True),
("claude-haiku-4-10", "anthropic", True, True),
("claude-haiku-5-0", "anthropic", True, True),
("claude-sonnet-5-1", "anthropic", True, True),
],
)
def test_shipped_version_boundaries(shipped_cost_map, model, provider, adaptive, mid_conversation):
assert model not in litellm.model_cost
info = litellm.get_model_info(model, custom_llm_provider=provider)
assert info["litellm_provider"] == provider
assert info["supports_function_calling"] is True
assert not info.get("input_cost_per_token")
assert info.get("supports_adaptive_thinking") is adaptive, model
assert info.get("supports_mid_conversation_system") is mid_conversation, model
def test_shipped_rules_cover_new_families_like_fable_at_5_plus(shipped_cost_map):
"""Both version gates accept any claude-<family>- id at major 5 or higher, bare
major or major-minor, so a new family shaped like claude-fable-5 gets adaptive
thinking and mid-conversation system support without a cost-map entry."""
model = "claude-fable-5-1"
assert model not in litellm.model_cost
info = litellm.get_model_info(model, custom_llm_provider="anthropic")
assert info["supports_mid_conversation_system"] is True
assert info["supports_adaptive_thinking"] is True
assert info["supports_function_calling"] is True
def test_shipped_rules_flag_bare_5_plus_majors_of_any_family(shipped_cost_map):
"""A bare 5+ major with no minor gets both flags at the rule level; the mapped
claude-fable-5 entry itself still resolves from the cost map, so this pins the
pattern via the capability union rather than get_model_info."""
matched = match_capability_generalizations("claude-fable-5")
assert matched is not None
assert matched["supports_adaptive_thinking"] is True
assert matched["supports_mid_conversation_system"] is True
def test_shipped_version_gates_are_family_agnostic_at_4x(shipped_cost_map):
"""Both version gates apply to any claude-<family>- id, 4.x included: a non-core
family at 4.9 gets adaptive and mid-conversation, while the same family at 4.5
gets baseline only. Only opus/sonnet/haiku ever shipped 4.x ids, so the
family-agnostic 4.6+ gate changes nothing for real models."""
high = litellm.get_model_info("claude-newfam-4-9", custom_llm_provider="anthropic")
assert high["supports_adaptive_thinking"] is True
assert high["supports_mid_conversation_system"] is True
assert high["supports_function_calling"] is True
low = litellm.get_model_info("claude-newfam-4-5", custom_llm_provider="anthropic")
assert low.get("supports_adaptive_thinking") is None
assert low.get("supports_mid_conversation_system") is None
assert low["supports_function_calling"] is True
def test_shipped_rules_give_bare_majors_the_full_baseline_union(shipped_cost_map):
"""A bare-major unmapped id (no minor) resolves the same baseline union as its
major-minor sibling: the baseline pattern's minor is optional, so claude-newt-5
is not left with version flags but no mode, token limits, or capability facts."""
model = "anthropic/claude-newt-5"
assert model not in litellm.model_cost
info = litellm.get_model_info(model)
assert info["litellm_provider"] == "anthropic"
assert info["mode"] == "chat"
assert info["max_tokens"] == 64000
assert info["supports_function_calling"] is True
assert info.get("supports_adaptive_thinking") is None
assert not info.get("input_cost_per_token")
assert info["supports_adaptive_thinking"] is True
assert info["supports_mid_conversation_system"] is True
def test_shipped_routing_rule_covers_bare_majors(shipped_cost_map):
_, provider, _, _ = litellm.get_llm_provider(model="claude-newt-5")
assert provider == "anthropic"
def test_shipped_adaptive_rule_requires_claude_prefix(shipped_cost_map):
"""A non-Claude name embedding a core-family 4.6+/5.x version substring must not
resolve from the rules; serving it a zero-priced rule entry would silently
swallow cost tracking for arbitrary custom deployment names."""
model = "openai/team-sonnet-5-1-alias"
assert model not in litellm.model_cost
assert match_capability_generalizations("team-sonnet-5-1-alias") is None
with pytest.raises(Exception):
litellm.get_model_info(model)
def test_shipped_exact_entry_beats_rules(shipped_cost_map):
model = "us.anthropic.claude-sonnet-4-6"
assert model in litellm.model_cost
info = litellm.get_model_info(model, custom_llm_provider="bedrock")
assert info["litellm_provider"] == "bedrock_converse"
assert info["input_cost_per_token"] == 3.3e-06
assert info["max_input_tokens"] == 1000000
assert info["supports_adaptive_thinking"] is True
assert info.get("supports_mid_conversation_system") is None
def test_shipped_rules_lose_to_exact_entries_across_cost_ladder_variants(shipped_cost_map):
"""A route-mangled variant of an exactly-mapped model must never resolve from
rules. The cost calculator tries model-name variants in order; a rule-derived
unpriced entry served for an early variant (here bedrock/claude-haiku-4-5-20251001,
whose bare form is exactly mapped under anthropic) would zero out the bill even
though the exact priced bedrock entry is one variant later. An exactly-mapped id
under a mismatched provider raises instead of resolving from rules."""
from litellm import completion_cost
from litellm.types.utils import ModelResponse, Usage
assert "claude-haiku-4-5-20251001" in litellm.model_cost
with pytest.raises(Exception):
litellm.get_model_info("claude-haiku-4-5-20251001", custom_llm_provider="bedrock")
entry = litellm.model_cost["us.anthropic.claude-haiku-4-5-20251001-v1:0"]
response = ModelResponse(model="claude-haiku-4-5-20251001", usage=Usage(prompt_tokens=100, completion_tokens=50))
cost = completion_cost(
completion_response=response,
model="bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0",
custom_llm_provider="bedrock",
)
assert cost == 100 * entry["input_cost_per_token"] + 50 * entry["output_cost_per_token"]
assert cost > 0
def test_shipped_adaptive_rule_gates_on_version_not_pricing(shipped_cost_map):
"""The version-gated ``anthropic-claude-adaptive-thinking`` rule marks an unmapped
Claude adaptive only from >= 4.6, including provider-prefixed ids the anchored pricing
rule cannot match, while leaving < 4.6 (and the dated Opus 4.0 form) non-adaptive."""
"""The version-gated adaptive-thinking capability rule marks an unmapped Claude
adaptive only from >= 4.6, including provider-prefixed ids the anchored routing
rule cannot match, while leaving the dated Opus 4.0 form non-adaptive."""
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
adaptive = "us.anthropic.claude-opus-4-9"
@ -343,12 +524,10 @@ def test_shipped_adaptive_rule_gates_on_version_not_pricing(shipped_cost_map):
assert AnthropicModelInfo._is_adaptive_thinking_model(non_adaptive) is False
def test_shipped_bedrock_rule_resolves_unmapped_future_claude_for_bedrock(shipped_cost_map):
"""An unmapped Bedrock Claude >= 4.8 resolves via the bedrock-scoped
``bedrock-anthropic-claude-mid-conversation-system`` rule even when the lookup
carries ``custom_llm_provider="bedrock"``, which the provider check uses to drop
the anthropic-scoped rules. It inherits base capabilities, gains both
version-gated flags, and stays unpriced."""
def test_shipped_rules_resolve_unmapped_future_bedrock_claude_with_both_flags(shipped_cost_map):
"""An unmapped Bedrock Claude >= 4.8 resolves for custom_llm_provider="bedrock" with
baseline capabilities, both version-gated flags, the bedrock provider backfilled, and
no fabricated pricing."""
model = "us.anthropic.claude-opus-4-9"
assert model not in litellm.model_cost
info = litellm.get_model_info(model, custom_llm_provider="bedrock")
@ -359,11 +538,11 @@ def test_shipped_bedrock_rule_resolves_unmapped_future_claude_for_bedrock(shippe
assert not info.get("input_cost_per_token")
def test_shipped_bedrock_mid_conversation_rule_gates_on_version_and_naming(shipped_cost_map):
"""The bedrock rule only claims Bedrock-style ids at 4.8+, bare 5+ majors and
new families included; pre-4.8 Bedrock ids and native ids never gain the flag,
and the rule outranks the anthropic-scoped ones for Bedrock ids because it is
listed first."""
def test_shipped_mid_conversation_gate_on_bedrock_ids(shipped_cost_map):
"""Bedrock-syntax ids gain ``supports_mid_conversation_system`` only from 4.8 upward,
bare 5+ majors and new families included; 4.7-and-below Bedrock ids never gain it.
The flag comes from the provider-neutral capability rule rather than a bedrock-scoped
one, so the same gate covers native and vertex-shaped ids too."""
for flagged in (
"us.anthropic.claude-opus-4-8",
"jp.anthropic.claude-opus-4-8",
@ -371,17 +550,14 @@ def test_shipped_bedrock_mid_conversation_rule_gates_on_version_and_naming(shipp
"us.anthropic.claude-fable-5",
"anthropic.claude-sonnet-5-20260101-v1:0",
):
matched = match_fallback_generalization(flagged)
matched = match_capability_generalizations(flagged)
assert matched is not None, flagged
assert matched["litellm_provider"] == "bedrock", flagged
assert matched["supports_mid_conversation_system"] is True, flagged
for unflagged in (
"us.anthropic.claude-opus-4-7",
"us.anthropic.claude-sonnet-4-6",
"us.anthropic.claude-haiku-4-5-20251001-v1:0",
"anthropic.claude-3-5-sonnet-20240620-v1:0",
"claude-opus-4-9",
"claude-sonnet-5",
):
matched = match_fallback_generalization(unflagged)
matched = match_capability_generalizations(unflagged)
assert matched is None or not matched.get("supports_mid_conversation_system"), unflagged

49
tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py
View file

@ -13,7 +13,8 @@ sys.path.insert(0, os.path.abspath("../../.."))
from litellm.litellm_core_utils.fallback_generalizations import (
get_fallback_generalization_rules,
match_fallback_generalization,
match_capability_generalizations,
match_routing_generalization,
set_fallback_generalizations,
)
from litellm.litellm_core_utils.get_model_cost_map import (
@ -102,9 +103,7 @@ def test_finalize_pops_key_and_installs_rules():
# The reserved key is removed from the returned model map ...
assert FALLBACK_GENERALIZATIONS_KEY not in finalized
# ... and its rules are installed into the generalizations module.
assert match_fallback_generalization("widget-9") == {
"litellm_provider": "openai"
}
assert match_routing_generalization("widget-9") == "openai"
finally:
set_fallback_generalizations(previous)
@ -116,27 +115,25 @@ def test_finalize_with_no_block_clears_rules():
[{"name": "stale", "pattern": r"^x", "model_info": {"a": 1}}]
)
_finalize_model_cost_map(_make_models(2))
assert match_fallback_generalization("x-1") is None
assert match_capability_generalizations("x-1") is None
finally:
set_fallback_generalizations(previous)
def test_shipped_backup_carries_the_anthropic_claude_rule():
"""The bundled backup must ship the anthropic-claude rule so a fresh install
(or an offline fallback) routes unknown Claude models without code changes."""
def test_shipped_backup_carries_the_claude_routing_rules():
"""The bundled backup must ship the Claude routing rules so a fresh install
(or an offline fallback) routes unknown Claude models without code changes.
Bedrock-syntax ids must hit the bedrock rule before the bare-id Anthropic rule."""
backup = GetModelCostMap.load_local_model_cost_map()
rules = backup.get(FALLBACK_GENERALIZATIONS_KEY, {}).get("rules", [])
names = {r.get("name") for r in rules}
assert "anthropic-claude" in names
rule = next(r for r in rules if r.get("name") == "anthropic-claude")
assert rule["model_info"]["litellm_provider"] == "anthropic"
names = [r.get("name") for r in rules]
assert names.index("bedrock-claude-ids") < names.index("anthropic-claude-ids")
previous = list(get_fallback_generalization_rules())
try:
set_fallback_generalizations(rules)
matched = match_fallback_generalization("claude-opus-4-9")
assert matched is not None and matched["litellm_provider"] == "anthropic"
assert match_routing_generalization("claude-opus-4-9") == "anthropic"
assert match_routing_generalization("global.anthropic.claude-opus-4-9") == "bedrock"
finally:
set_fallback_generalizations(previous)
@ -147,23 +144,19 @@ def test_shipped_backup_marks_claude_4_6_plus_adaptive_not_4_0():
route) and on the version-gated anthropic-claude-adaptive-thinking rule for
unmapped future Claudes, while leaving the dated Claude 4.0 names
("...-4-20250514") unflagged so a date can never be mistaken for a 4.6+ minor
version. The version-neutral anthropic-claude pricing rule must not flag it, so
an unmapped sub-4.6 name is priced but stays non-adaptive. The adaptive rule must
inherit pricing from the pricing rule via ``extends`` and carry only its delta, so
the Opus-tier price block is never duplicated across rules."""
version. The version-neutral claude-family-baseline capability rule must not flag
it, so an unmapped sub-4.6 name resolves but stays non-adaptive. The adaptive rule
carries only its delta; capability unioning stacks it onto the baseline, so the
baseline block is never duplicated across rules and no rule needs ``extends``."""
backup = GetModelCostMap.load_local_model_cost_map()
rules = backup[FALLBACK_GENERALIZATIONS_KEY]["rules"]
pricing_rule = next(r for r in rules if r.get("name") == "anthropic-claude")
adaptive_rule = next(
r for r in rules if r.get("name") == "anthropic-claude-adaptive-thinking"
)
assert "supports_adaptive_thinking" not in pricing_rule["model_info"]
assert adaptive_rule["model_info"]["supports_adaptive_thinking"] is True
assert "extends" not in pricing_rule
assert adaptive_rule.get("extends") == "anthropic-claude"
baseline_rule = next(r for r in rules if r.get("name") == "claude-family-baseline")
adaptive_rule = next(r for r in rules if r.get("name") == "claude-adaptive-thinking")
assert "supports_adaptive_thinking" not in baseline_rule["model_info"]
assert "litellm_provider" not in baseline_rule["model_info"]
assert adaptive_rule["model_info"] == {"supports_adaptive_thinking": True}
assert all("extends" not in r for r in rules)
for adaptive in [
"anthropic.claude-opus-4-8",

23
tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py
View file

@ -1649,17 +1649,18 @@ class TestClaudeOpus48AdaptiveThinking:
@pytest.mark.parametrize(
"model",
[
"us.anthropic.claude-fable-5-preview",
"claude-fable-5-preview",
"us.anthropic.claude-fable-preview",
"claude-fable-preview",
],
)
def test_unmapped_aliases_without_parseable_version_stay_non_adaptive(
self, local_model_cost_map, model
):
"""An alias absent from the map, not matched by any ``fallback_generalizations``
rule, and without a parseable opus/sonnet/haiku >= 4.6 family version stays
non-adaptive. ``fable`` is outside the version-rule family set, so neither the
cost map nor the declarative rule marks it adaptive."""
rule, and without any parseable family version stays non-adaptive. ``fable``
without a major version matches neither the core-family 4.6+ gate nor the
family-agnostic 5+ gate, so neither the cost map nor the declarative rule marks
it adaptive."""
import litellm
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
@ -1676,16 +1677,18 @@ class TestClaudeOpus48AdaptiveThinking:
"claude-opus-5-0",
"claude-opus-4-10",
"claude-opus-4-8-some-future-suffix",
"claude-fable-5-preview",
"us.anthropic.claude-fable-5-preview",
],
)
def test_adaptive_thinking_version_fallback_for_unmapped_high_versions(
self, local_model_cost_map, model
):
"""Provider-prefixed or suffixed Claude names that resolve to no mapped entry and
are not matched by the anchored ``anthropic-claude`` pricing rule still resolve to
adaptive when their opus/sonnet/haiku family version is >= 4.6. The version gate is
the declarative ``anthropic-claude-adaptive-thinking`` rule, so 5.x, 6.x and any
later family are covered with no code change."""
"""Provider-prefixed or suffixed Claude names that resolve to no mapped entry
still resolve to adaptive when the id carries claude-<family>- at version 4.6
or higher, bare 5+ majors included. The version gate is the declarative
``claude-adaptive-thinking`` rule, so 5.x, 6.x and any later family are covered
with no code change."""
import litellm
from litellm.llms.anthropic.common_utils import AnthropicModelInfo

53
tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py
View file

@ -1974,13 +1974,22 @@ def test_bedrock_invoke_transform_merges_list_content_system_role_into_system():
]
def test_bedrock_invoke_transform_keeps_mid_conversation_system_role_in_place(local_model_cost_map):
@pytest.mark.parametrize(
"model",
[
"anthropic.claude-opus-4-8",
"jp.anthropic.claude-opus-4-8",
"us.anthropic.claude-sonnet-5",
"us.anthropic.claude-fable-5",
],
)
def test_bedrock_invoke_transform_keeps_mid_conversation_system_role_in_place(local_model_cost_map, model):
"""Regression test for the Bedrock prompt-cache collapse: hoisting a
mid-conversation ``role: "system"`` message (e.g. Claude Code's
``mid-conversation-system-2026-04-07`` reminders) into the top-level
``system`` field mutates the cache prefix and invalidates the cached message
history, so on models flagged ``supports_mid_conversation_system`` (the Opus
4.8 family, which Invoke accepts the role on) such entries must be forwarded
history, so on models flagged ``supports_mid_conversation_system`` (Claude
4.8+, which Invoke accepts the role on) such entries must be forwarded
in place. Billing-header blocks must still be stripped from the top-level
``system`` field even when nothing is hoisted."""
from litellm.types.router import GenericLiteLLMParams
@ -1994,7 +2003,7 @@ def test_bedrock_invoke_transform_keeps_mid_conversation_system_role_in_place(lo
]
result = cfg.transform_anthropic_messages_request(
model="anthropic.claude-opus-4-8",
model=model,
messages=copy.deepcopy(messages),
anthropic_messages_optional_request_params={
"max_tokens": 256,
@ -2121,9 +2130,9 @@ def test_bedrock_invoke_transform_hoists_all_system_for_unmapped_model(local_mod
def test_bedrock_invoke_transform_keeps_system_in_place_for_unmapped_future_claude(local_model_cost_map):
"""An unmapped Bedrock Claude at 4.8 or higher resolves through the
``bedrock-anthropic-claude-mid-conversation-system`` fallback rule, so a
future model that has not landed in the cost map yet keeps the
cache-preserving in-place behavior instead of falling back to hoist-all."""
``claude-mid-conversation-system`` capability rule, so a future model that
has not landed in the cost map yet keeps the cache-preserving in-place
behavior instead of falling back to hoist-all."""
from litellm.types.router import GenericLiteLLMParams
cfg = AmazonAnthropicClaudeMessagesConfig()
@ -2146,6 +2155,36 @@ def test_bedrock_invoke_transform_keeps_system_in_place_for_unmapped_future_clau
assert "system" not in result
def test_bedrock_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_flag():
"""Exact cost-map hits resolve before fallback-generalization rules, so a
mapped Bedrock Claude 4.8+ entry without ``supports_mid_conversation_system``
silently loses the cache-preserving in-place handling that the
``claude-mid-conversation-system`` capability rule grants unmapped ids.
Every mapped bedrock entry the rule's own pattern matches must carry the
flag explicitly."""
import re
import litellm
cost_map_path = os.path.join(os.path.dirname(litellm.__file__), "model_prices_and_context_window_backup.json")
with open(cost_map_path) as f:
cost_map = json.load(f)
rules = cost_map["fallback_generalizations"]["rules"]
pattern = re.compile(
next(r["pattern"] for r in rules if r["name"] == "claude-mid-conversation-system"),
re.IGNORECASE,
)
missing = [
key
for key, info in cost_map.items()
if isinstance(info, dict)
and str(info.get("litellm_provider", "")).startswith("bedrock")
and pattern.search(key)
and info.get("supports_mid_conversation_system") is not True
]
assert missing == []
def test_as_system_content_blocks_handles_each_shape():
"""``_as_system_content_blocks`` normalizes every system shape: ``None`` -> empty,
a string -> a single text block, a list -> a shallow copy, and any other value

326
tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
View file

@ -660,6 +660,7 @@ async def test_register_client_persists_dcr_client_identity():
assert update_data.credentials["client_id"] == "generated-client"
assert update_data.credentials["client_secret"] == "generated-secret"
assert update_data.credentials["token_endpoint_auth_method"] == "client_secret_basic"
assert update_data.credentials["redirect_uris"] == ["https://proxy.litellm.example/callback"]
assert update_data.oauth2_flow == "authorization_code"
mock_update_server.assert_called_once()
@ -1141,6 +1142,296 @@ async def test_register_client_returns_reused_client_when_concurrent_persist_win
mock_update_server.assert_called_once_with(persisted_server)
def _dcr_redirect_test_server(client_id):
from litellm.proxy._types import MCPTransport
from litellm.types.mcp import MCPAuth
from litellm.types.mcp_server.mcp_server_manager import MCPServer
return MCPServer(
server_id="remote_server",
name="remote_server",
server_name="remote_server",
alias="remote_server",
transport=MCPTransport.http,
auth_type=MCPAuth.oauth2,
client_id=client_id,
client_secret=None,
authorization_url="https://provider.example/oauth/authorize",
token_url="https://provider.example/oauth/token",
registration_url="https://provider.example/oauth/register",
)
@pytest.mark.asyncio
async def test_register_client_re_registers_when_persisted_redirect_uri_no_longer_matches_origin():
"""A persisted DCR client is bound to the redirect_uri it was registered with. When the
proxy's resolved public origin changes, every authorize built for the reused client is
rejected IdP-side and the server is permanently stranded (GH #32473). A positive mismatch
between the recorded redirect_uris and the current callback must therefore re-register on
the admin path and persist the replacement client, with the new binding recorded and the
old client's secret/auth method cleared rather than merged into the new identity."""
try:
from fastapi import Request
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
register_client_with_server,
)
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
except ImportError:
pytest.skip("MCP discoverable endpoints not available")
oauth2_server = _dcr_redirect_test_server(client_id="stale-client")
mock_request = MagicMock(spec=Request)
mock_request.base_url = "https://proxy.litellm.example/"
mock_request.headers = {}
mock_response = MagicMock()
mock_response.json.return_value = {
"client_id": "fresh-client",
"redirect_uris": ["https://proxy.litellm.example/callback"],
}
mock_response.raise_for_status = MagicMock()
mock_async_client = MagicMock()
mock_async_client.post = AsyncMock(return_value=mock_response)
persisted_server = MagicMock()
persisted_server.credentials = {
"client_id": "stale-client",
"client_secret": "stale-secret",
"token_endpoint_auth_method": "client_secret_basic",
"redirect_uris": ["https://old-origin.example/callback"],
}
mock_get_mcp_server = AsyncMock(return_value=persisted_server)
mock_update_mcp_server = AsyncMock(return_value=MagicMock())
mock_update_server = AsyncMock()
with (
patch(
"litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client",
return_value=mock_async_client,
),
patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()),
patch(
"litellm.proxy._experimental.mcp_server.db.get_mcp_server",
new=mock_get_mcp_server,
),
patch(
"litellm.proxy._experimental.mcp_server.db.update_mcp_server",
new=mock_update_mcp_server,
),
patch.object(global_mcp_server_manager, "update_server", new=mock_update_server),
):
response = await register_client_with_server(
request=mock_request,
mcp_server=oauth2_server,
client_name="Litellm Proxy",
grant_types=["authorization_code", "refresh_token"],
response_types=["code"],
token_endpoint_auth_method="none",
persist_credentials=True,
)
mock_async_client.post.assert_called_once()
register_payload = mock_async_client.post.call_args.kwargs["json"]
assert register_payload["redirect_uris"] == ["https://proxy.litellm.example/callback"]
mock_update_mcp_server.assert_called_once()
update_data = mock_update_mcp_server.call_args.kwargs["data"]
assert update_data.credentials["client_id"] == "fresh-client"
assert update_data.credentials["redirect_uris"] == ["https://proxy.litellm.example/callback"]
assert update_data.credentials["client_secret"] is None
assert update_data.credentials["token_endpoint_auth_method"] is None
assert response.status_code == 200
assert json.loads(response.body.decode("utf-8"))["client_id"] == "fresh-client"
@pytest.mark.asyncio
async def test_register_client_grandfathers_persisted_client_without_recorded_redirect_uris():
"""Clients persisted before redirect_uris were recorded (and admin-configured clients,
which never get a recording) have nothing to compare against; treating that as a mismatch
would re-mint a client_id for every existing install on upgrade and orphan all users'
refresh tokens for those servers. A missing recording must read as a match: no DCR call,
no persistence write, existing client returned."""
try:
from fastapi import Request
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
register_client_with_server,
)
except ImportError:
pytest.skip("MCP discoverable endpoints not available")
oauth2_server = _dcr_redirect_test_server(client_id="legacy-client")
mock_request = MagicMock(spec=Request)
mock_request.base_url = "https://proxy.litellm.example/"
mock_request.headers = {}
mock_async_client = MagicMock()
mock_async_client.post = AsyncMock()
persisted_server = MagicMock()
persisted_server.credentials = {"client_id": "legacy-client"}
mock_get_mcp_server = AsyncMock(return_value=persisted_server)
mock_update_mcp_server = AsyncMock()
with (
patch(
"litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client",
return_value=mock_async_client,
),
patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()),
patch(
"litellm.proxy._experimental.mcp_server.db.get_mcp_server",
new=mock_get_mcp_server,
),
patch(
"litellm.proxy._experimental.mcp_server.db.update_mcp_server",
new=mock_update_mcp_server,
),
):
response = await register_client_with_server(
request=mock_request,
mcp_server=oauth2_server,
client_name="Litellm Proxy",
grant_types=["authorization_code", "refresh_token"],
response_types=["code"],
token_endpoint_auth_method="none",
persist_credentials=True,
)
mock_async_client.post.assert_not_called()
mock_update_mcp_server.assert_not_called()
assert response["client_secret"] == "dummy"
assert oauth2_server.client_id == "legacy-client"
@pytest.mark.asyncio
async def test_register_client_keeps_persisted_client_when_recorded_redirect_uri_matches_origin():
"""When the recorded redirect_uris still cover the current callback the persisted client
is valid; re-registering would orphan refresh tokens for no reason."""
try:
from fastapi import Request
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
register_client_with_server,
)
except ImportError:
pytest.skip("MCP discoverable endpoints not available")
oauth2_server = _dcr_redirect_test_server(client_id="kept-client")
mock_request = MagicMock(spec=Request)
mock_request.base_url = "https://proxy.litellm.example/"
mock_request.headers = {}
mock_async_client = MagicMock()
mock_async_client.post = AsyncMock()
persisted_server = MagicMock()
persisted_server.credentials = {
"client_id": "kept-client",
"redirect_uris": ["https://proxy.litellm.example/callback"],
}
mock_get_mcp_server = AsyncMock(return_value=persisted_server)
mock_update_mcp_server = AsyncMock()
with (
patch(
"litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client",
return_value=mock_async_client,
),
patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()),
patch(
"litellm.proxy._experimental.mcp_server.db.get_mcp_server",
new=mock_get_mcp_server,
),
patch(
"litellm.proxy._experimental.mcp_server.db.update_mcp_server",
new=mock_update_mcp_server,
),
):
response = await register_client_with_server(
request=mock_request,
mcp_server=oauth2_server,
client_name="Litellm Proxy",
grant_types=["authorization_code", "refresh_token"],
response_types=["code"],
token_endpoint_auth_method="none",
persist_credentials=True,
)
mock_async_client.post.assert_not_called()
mock_update_mcp_server.assert_not_called()
assert response["client_secret"] == "dummy"
@pytest.mark.asyncio
async def test_register_client_non_admin_reuses_persisted_client_despite_redirect_mismatch():
"""Non-persisting callers (the public register routes and non-admin users) must keep
today's reuse behavior even when the recorded redirect_uris mismatch: re-registering
without persistence would mint an orphan upstream client on every connect while the
stored client keeps being used at authorize time. Only the admin path re-registers."""
try:
from fastapi import Request
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
register_client_with_server,
)
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
except ImportError:
pytest.skip("MCP discoverable endpoints not available")
oauth2_server = _dcr_redirect_test_server(client_id=None)
mock_request = MagicMock(spec=Request)
mock_request.base_url = "https://proxy.litellm.example/"
mock_request.headers = {}
mock_async_client = MagicMock()
mock_async_client.post = AsyncMock()
persisted_server = MagicMock()
persisted_server.credentials = {
"client_id": "persisted-client",
"redirect_uris": ["https://old-origin.example/callback"],
}
mock_get_mcp_server = AsyncMock(return_value=persisted_server)
mock_update_server = AsyncMock()
with (
patch(
"litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client",
return_value=mock_async_client,
),
patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()),
patch(
"litellm.proxy._experimental.mcp_server.db.get_mcp_server",
new=mock_get_mcp_server,
),
patch.object(global_mcp_server_manager, "update_server", new=mock_update_server),
):
response = await register_client_with_server(
request=mock_request,
mcp_server=oauth2_server,
client_name="Litellm Proxy",
grant_types=["authorization_code", "refresh_token"],
response_types=["code"],
token_endpoint_auth_method="none",
persist_credentials=False,
)
mock_async_client.post.assert_not_called()
assert oauth2_server.client_id == "persisted-client"
assert response["client_secret"] == "dummy"
@pytest.mark.asyncio
async def test_register_client_reuses_existing_client_id_without_re_dcr():
"""A server that already has a client_id (admin-configured or previously DCR'd) must be
@ -1848,6 +2139,41 @@ async def test_token_endpoint_respects_x_forwarded_host():
None,
"https://external.com",
),
(
"http://localhost:4000/",
"https",
"proxy.example.com",
"443",
"https://proxy.example.com",
),
(
"http://localhost:4000/",
"http",
"proxy.example.com",
"80",
"http://proxy.example.com",
),
(
"http://internal.local/",
"https",
None,
"443",
"https://internal.local",
),
(
"http://localhost:4000/",
"https",
"proxy.example.com",
"8443",
"https://proxy.example.com:8443",
),
(
"http://localhost:4000/",
"https",
"proxy.example.com:443",
None,
"https://proxy.example.com",
),
],
)
def test_get_request_base_url_comprehensive(

338
tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py
View file

@ -3706,3 +3706,341 @@ async def test_per_tag_untagged_request_governed_by_key_limit_v3(monkeypatch):
await call({"tags": ["cell-99"]})
assert exc_info.value.status_code == 429
assert "tag_per_key" not in str(exc_info.value.detail)
# --------------------------------------------------------------------------
# Streaming success logging mirrors x-ratelimit-* remaining values into
# standard_logging_object.hidden_params.additional_headers so Prometheus /
# logging callbacks see them for streams too (non-streaming already gets
# them via async_post_call_success_hook, which the streaming path skips).
# --------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_streaming_end_to_end_populates_slp_ratelimit_headers(monkeypatch):
"""
End-to-end regression: on a streaming request, the same pre-call +
success-callback pair the proxy uses must land ``x-ratelimit-*``
remaining/limit values in
``kwargs["standard_logging_object"]["hidden_params"]["additional_headers"]``.
"""
monkeypatch.setenv("LITELLM_RATE_LIMIT_WINDOW_SIZE", "60")
_api_key = hash_token("sk-stream-e2e")
user_api_key_dict = UserAPIKeyAuth(
api_key=_api_key,
rpm_limit=100,
tpm_limit=10000,
)
local_cache = DualCache()
handler = _PROXY_MaxParallelRequestsHandler(
internal_usage_cache=InternalUsageCache(local_cache)
)
# Real pre-call: populates data and stashes the response into metadata
# so the success callback can find it via litellm_params.metadata.
data: Dict[str, Any] = {
"model": "gpt-4o-mini",
"metadata": {},
"stream": True,
}
await handler.async_pre_call_hook(
user_api_key_dict=user_api_key_dict,
cache=local_cache,
data=data,
call_type="",
)
# Simulate the wrapper handing the pre-call metadata dict to the
# completion() call: it becomes kwargs["litellm_params"]["metadata"] by
# the time the success callback fires.
mock_response = ModelResponse(
id="mock-stream-e2e",
object="chat.completion",
created=int(datetime.now().timestamp()),
model="gpt-4o-mini",
usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150),
choices=[],
)
mock_kwargs: Dict[str, Any] = {
"standard_logging_object": {
"metadata": {
"user_api_key_hash": _api_key,
"user_api_key_user_id": None,
"user_api_key_team_id": None,
"user_api_key_end_user_id": None,
}
},
"litellm_params": {"metadata": data["metadata"]},
"model": "gpt-4o-mini",
}
async def _noop_increment(increment_list, **_):
return True
monkeypatch.setattr(
handler.internal_usage_cache.dual_cache,
"async_increment_cache_pipeline",
_noop_increment,
)
# async_logging_hook runs before async_log_success_event, so any
# downstream callback that reads the SLP sees the mirrored values.
await handler.async_logging_hook(
kwargs=mock_kwargs,
result=mock_response,
call_type="acompletion",
)
additional_headers = (
mock_kwargs["standard_logging_object"]
.get("hidden_params", {})
.get("additional_headers", {})
)
# api_key-scoped remaining/limit values are the baseline every request
# emits and must always reach the SLP.
remaining_keys = [
k for k in additional_headers if "-remaining-" in k
]
assert (
remaining_keys
), f"streaming success must populate remaining values, got {additional_headers!r}"
limit_keys = [k for k in additional_headers if "-limit-" in k]
assert limit_keys, "streaming success must also populate limit values"
assert (
additional_headers.get("x-ratelimit-api_key-remaining-requests") == 99
), (
"api_key remaining requests should reflect the just-consumed slot;"
f" got {additional_headers!r}"
)
@pytest.mark.asyncio
async def test_streaming_populates_model_per_key_ratelimit_headers(monkeypatch):
"""
Streaming must land the per-(key, model) remaining/limit values in the
SLP under ``x-ratelimit-model_per_key-{remaining|limit}-{requests,tokens}``.
"""
monkeypatch.setenv("LITELLM_RATE_LIMIT_WINDOW_SIZE", "60")
_api_key = hash_token("sk-stream-mirror")
user_api_key_dict = UserAPIKeyAuth(
api_key=_api_key,
metadata={
"model_rpm_limit": {"gpt-4o-mini": 100},
"model_tpm_limit": {"gpt-4o-mini": 10000},
},
)
local_cache = DualCache()
handler = _PROXY_MaxParallelRequestsHandler(
internal_usage_cache=InternalUsageCache(local_cache)
)
async def _noop_increment(increment_list, **_):
return True
monkeypatch.setattr(
handler.internal_usage_cache.dual_cache,
"async_increment_cache_pipeline",
_noop_increment,
)
data: Dict[str, Any] = {"model": "gpt-4o-mini", "metadata": {}, "stream": True}
await handler.async_pre_call_hook(
user_api_key_dict=user_api_key_dict,
cache=local_cache,
data=data,
call_type="",
)
mock_response = ModelResponse(
id="mock-stream",
object="chat.completion",
created=int(datetime.now().timestamp()),
model="gpt-4o-mini",
usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150),
choices=[],
)
mock_kwargs: Dict[str, Any] = {
"standard_logging_object": {
"metadata": {
"user_api_key_hash": _api_key,
"user_api_key_user_id": None,
"user_api_key_team_id": None,
"user_api_key_end_user_id": None,
}
},
"litellm_params": {"metadata": data["metadata"]},
"model": "gpt-4o-mini",
}
await handler.async_logging_hook(
kwargs=mock_kwargs,
result=mock_response,
call_type="acompletion",
)
hidden_params = mock_kwargs["standard_logging_object"].get("hidden_params") or {}
additional_headers = hidden_params.get("additional_headers") or {}
assert (
additional_headers.get("x-ratelimit-model_per_key-remaining-requests") == 99
), f"got {additional_headers!r}"
assert additional_headers.get("x-ratelimit-model_per_key-limit-requests") == 100
# response._hidden_params is also updated for late readers.
response_hidden = getattr(mock_response, "_hidden_params", None) or {}
response_headers = response_hidden.get("additional_headers") or {}
assert response_headers.get("x-ratelimit-model_per_key-remaining-requests") == 99
@pytest.mark.asyncio
async def test_async_log_success_event_no_mirror_when_no_snapshot(monkeypatch):
"""
No pre-call snapshot (no descriptors matched) -> no fabricated
``x-ratelimit-*`` headers.
"""
monkeypatch.setenv("LITELLM_RATE_LIMIT_WINDOW_SIZE", "60")
_api_key = hash_token("sk-stream-no-mirror")
handler = _PROXY_MaxParallelRequestsHandler(
internal_usage_cache=InternalUsageCache(DualCache())
)
async def _noop_increment(increment_list, **_):
return True
monkeypatch.setattr(
handler.internal_usage_cache.dual_cache,
"async_increment_cache_pipeline",
_noop_increment,
)
mock_response = ModelResponse(
id="mock-stream-none",
object="chat.completion",
created=int(datetime.now().timestamp()),
model="gpt-4o-mini",
usage=Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2),
choices=[],
)
mock_kwargs: Dict[str, Any] = {
"standard_logging_object": {
"metadata": {
"user_api_key_hash": _api_key,
"user_api_key_user_id": None,
"user_api_key_team_id": None,
"user_api_key_end_user_id": None,
}
},
"litellm_params": {"metadata": {}},
"model": "gpt-4o-mini",
}
await handler.async_logging_hook(
kwargs=mock_kwargs,
result=mock_response,
call_type="acompletion",
)
hidden_params = mock_kwargs["standard_logging_object"].get("hidden_params") or {}
additional_headers = hidden_params.get("additional_headers") or {}
ratelimit_keys = [k for k in additional_headers if k.startswith("x-ratelimit-")]
assert (
not ratelimit_keys
), f"no snapshot must produce no rate-limit headers, got {ratelimit_keys}"
@pytest.mark.asyncio
async def test_streaming_mirror_matches_non_streaming_header_shape(monkeypatch):
"""
Given the same pre-call state, streaming and non-streaming must write
the identical ``x-ratelimit-*`` key/value shape to their respective
``additional_headers`` slots.
"""
monkeypatch.setenv("LITELLM_RATE_LIMIT_WINDOW_SIZE", "60")
_api_key = hash_token("sk-shape")
user_api_key_dict = UserAPIKeyAuth(
api_key=_api_key,
metadata={
"model_rpm_limit": {"gpt-4o-mini": 50},
"model_tpm_limit": {"gpt-4o-mini": 5000},
},
)
local_cache = DualCache()
handler = _PROXY_MaxParallelRequestsHandler(
internal_usage_cache=InternalUsageCache(local_cache)
)
async def _noop_increment(increment_list, **_):
return True
monkeypatch.setattr(
handler.internal_usage_cache.dual_cache,
"async_increment_cache_pipeline",
_noop_increment,
)
# Drive pre-call once so both paths have the same authoritative snapshot.
data: Dict[str, Any] = {"model": "gpt-4o-mini", "metadata": {}}
await handler.async_pre_call_hook(
user_api_key_dict=user_api_key_dict,
cache=local_cache,
data=data,
call_type="",
)
# Non-streaming path: async_post_call_success_hook mutates response._hidden_params.
non_stream_response = ModelResponse(
id="mock-non-stream",
object="chat.completion",
created=int(datetime.now().timestamp()),
model="gpt-4o-mini",
usage=Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2),
choices=[],
)
non_stream_response._hidden_params = {}
await handler.async_post_call_success_hook(
data=data,
user_api_key_dict=user_api_key_dict,
response=non_stream_response,
)
non_stream_headers = non_stream_response._hidden_params.get(
"additional_headers", {}
)
# Streaming path: async_logging_hook mirrors into standard_logging_object.
stream_kwargs: Dict[str, Any] = {
"standard_logging_object": {
"metadata": {"user_api_key_hash": _api_key}
},
"litellm_params": {"metadata": data["metadata"]},
"model": "gpt-4o-mini",
}
stream_response = ModelResponse(
id="mock-stream",
object="chat.completion",
created=int(datetime.now().timestamp()),
model="gpt-4o-mini",
usage=Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2),
choices=[],
)
await handler.async_logging_hook(
kwargs=stream_kwargs,
result=stream_response,
call_type="acompletion",
)
stream_slp_headers = (
stream_kwargs["standard_logging_object"]
.get("hidden_params", {})
.get("additional_headers", {})
)
def _rl_only(headers: Dict[str, Any]) -> Dict[str, Any]:
return {k: v for k, v in headers.items() if k.startswith("x-ratelimit-")}
assert _rl_only(stream_slp_headers) == _rl_only(non_stream_headers), (
f"streaming={_rl_only(stream_slp_headers)}"
f" non_streaming={_rl_only(non_stream_headers)}"
)
assert "x-ratelimit-model_per_key-remaining-requests" in stream_slp_headers

4
type-discipline-budget.json
View file

@ -1,9 +1,9 @@
{
"LIT001": {
"limit": 23452
"limit": 23409
},
"LIT002": {
"limit": 27522
"limit": 27511
},
"LIT003": {
"limit": 292

6
ui/litellm-dashboard/eslint-metrics.json
View file

@ -1,7 +1,7 @@
{
"@typescript-eslint/no-explicit-any": 1978,
"complexity": 130,
"local/no-large-inline-object-arg": 509,
"@typescript-eslint/no-explicit-any": 1971,
"complexity": 129,
"local/no-large-inline-object-arg": 501,
"local/no-long-condition-chain": 234,
"max-depth": 59,
"no-console": 16

16
ui/litellm-dashboard/package-lock.json generated
View file

@ -28,6 +28,7 @@
"moment": "2.30.1",
"next": "16.2.6",
"openai": "4.104.0",
"openapi-fetch": "^0.17.0",
"papaparse": "5.5.3",
"react": "18.3.1",
"react-copy-to-clipboard": "5.1.1",
@ -10534,6 +10535,15 @@
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
"license": "MIT"
},
"node_modules/openapi-fetch": {
"version": "0.17.0",
"resolved": "https://registry.npmjs.org/openapi-fetch/-/openapi-fetch-0.17.0.tgz",
"integrity": "sha512-PsbZR1wAPcG91eEthKhN+Zn92FMHxv+/faECIwjXdxfTODGSGegYv0sc1Olz+HYPvKOuoXfp+0pA2XVt2cI0Ig==",
"license": "MIT",
"dependencies": {
"openapi-typescript-helpers": "^0.1.0"
}
},
"node_modules/openapi-typescript": {
"version": "7.13.0",
"resolved": "https://registry.npmjs.org/openapi-typescript/-/openapi-typescript-7.13.0.tgz",
@ -10555,6 +10565,12 @@
"typescript": "^5.x"
}
},
"node_modules/openapi-typescript-helpers": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/openapi-typescript-helpers/-/openapi-typescript-helpers-0.1.0.tgz",
"integrity": "sha512-OKTGPthhivLw/fHz6c3OPtg72vi86qaMlqbJuVJ23qOvQ+53uw1n7HdmkJFibloF7QEjDrDkzJiOJuockM/ljw==",
"license": "MIT"
},
"node_modules/openapi-typescript/node_modules/supports-color": {
"version": "10.2.2",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-10.2.2.tgz",

1
ui/litellm-dashboard/package.json
View file

@ -44,6 +44,7 @@
"moment": "2.30.1",
"next": "16.2.6",
"openai": "4.104.0",
"openapi-fetch": "^0.17.0",
"papaparse": "5.5.3",
"react": "18.3.1",
"react-copy-to-clipboard": "5.1.1",

312
ui/litellm-dashboard/src/app/(dashboard)/hooks/customers/useCustomers.test.ts
View file

@ -1,334 +1,104 @@
import { allEndUsersCall } from "@/components/networking";
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
import { renderHook, waitFor } from "@testing-library/react";
import React, { ReactNode } from "react";
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { Customer, CustomersResponse } from "./useCustomers";
import { useCustomers } from "./useCustomers";
import { useCustomers, type EndUser } from "./useCustomers";
// Mock the networking function
vi.mock("@/components/networking", () => ({
allEndUsersCall: vi.fn(),
const mockGet = vi.fn();
vi.mock("@/lib/http/api", () => ({
fetchClient: { GET: (...args: unknown[]) => mockGet(...args) },
}));
// Mock useAuthorized hook - we can override this in individual tests
const mockUseAuthorized = vi.fn();
vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({
default: () => mockUseAuthorized(),
}));
// Import actual roles instead of mocking them
// Mock data
const mockCustomers: Customer[] = [
{
user_id: "customer-1",
alias: "Test Customer 1",
spend: 150.5,
blocked: false,
allowed_model_region: "us-east-1",
default_model: "gpt-3.5-turbo",
budget_id: "budget-1",
litellm_budget_table: {
budget_id: "budget-1",
max_budget: 1000,
soft_budget: 800,
max_parallel_requests: 10,
tpm_limit: 1000,
rpm_limit: 100,
model_max_budget: { "gpt-4": 500 },
budget_duration: "monthly",
budget_reset_at: "2024-02-01T00:00:00Z",
created_at: "2024-01-01T00:00:00Z",
created_by: "admin-1",
updated_at: "2024-01-01T00:00:00Z",
updated_by: "admin-1",
},
},
{
user_id: "customer-2",
alias: null,
spend: 0,
blocked: true,
allowed_model_region: null,
default_model: null,
budget_id: null,
litellm_budget_table: null,
},
const mockCustomers: EndUser[] = [
{ user_id: "customer-1", alias: "Test Customer 1", spend: 150.5, blocked: false },
{ user_id: "customer-2", alias: null, spend: 0, blocked: true },
];
const mockCustomersResponse: CustomersResponse = mockCustomers;
const authorized = {
accessToken: "test-access-token",
userRole: "Admin",
userId: "test-user-id",
token: "test-token",
userEmail: "test@example.com",
premiumUser: false,
disabledPersonalKeyCreation: null,
showSSOBanner: false,
};
describe("useCustomers", () => {
let queryClient: QueryClient;
beforeEach(() => {
queryClient = new QueryClient({
defaultOptions: {
queries: {
retry: false,
},
},
});
// Reset all mocks
queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } });
vi.clearAllMocks();
// Set default mock for useAuthorized (enabled state)
mockUseAuthorized.mockReturnValue({
accessToken: "test-access-token",
userRole: "Admin",
userId: "test-user-id",
token: "test-token",
userEmail: "test@example.com",
premiumUser: false,
disabledPersonalKeyCreation: null,
showSSOBanner: false,
});
mockUseAuthorized.mockReturnValue(authorized);
});
const wrapper = ({ children }: { children: ReactNode }) =>
React.createElement(QueryClientProvider, { client: queryClient }, children);
it("should return customers data when query is successful", async () => {
// Mock successful API call
(allEndUsersCall as any).mockResolvedValue(mockCustomersResponse);
it("fetches /customer/list and returns the typed list on success", async () => {
mockGet.mockResolvedValue({ data: mockCustomers });
const { result } = renderHook(() => useCustomers(), { wrapper });
// Initially loading
expect(result.current.isLoading).toBe(true);
expect(result.current.data).toBeUndefined();
// Wait for success
await waitFor(() => {
expect(result.current.isLoading).toBe(false);
expect(result.current.isSuccess).toBe(true);
});
expect(result.current.data).toEqual(mockCustomersResponse);
expect(result.current.error).toBeNull();
expect(allEndUsersCall).toHaveBeenCalledWith("test-access-token");
expect(allEndUsersCall).toHaveBeenCalledTimes(1);
expect(result.current.data).toEqual(mockCustomers);
expect(mockGet).toHaveBeenCalledWith("/customer/list");
expect(mockGet).toHaveBeenCalledTimes(1);
});
it("should handle error when allEndUsersCall fails", async () => {
const errorMessage = "Failed to fetch customers";
const testError = new Error(errorMessage);
// Mock failed API call
(allEndUsersCall as any).mockRejectedValue(testError);
it("surfaces an error when the request rejects", async () => {
const testError = new Error("Failed to fetch customers");
mockGet.mockRejectedValue(testError);
const { result } = renderHook(() => useCustomers(), { wrapper });
// Initially loading
expect(result.current.isLoading).toBe(true);
// Wait for error
await waitFor(() => {
expect(result.current.isLoading).toBe(false);
expect(result.current.isError).toBe(true);
});
expect(result.current.error).toEqual(testError);
expect(result.current.data).toBeUndefined();
expect(allEndUsersCall).toHaveBeenCalledWith("test-access-token");
expect(allEndUsersCall).toHaveBeenCalledTimes(1);
});
it("should not execute query when accessToken is missing", async () => {
// Mock missing accessToken
mockUseAuthorized.mockReturnValue({
accessToken: null,
userRole: "Admin",
userId: "test-user-id",
token: null,
userEmail: "test@example.com",
premiumUser: false,
disabledPersonalKeyCreation: null,
showSSOBanner: false,
});
it("falls back to an empty list when the response has no body", async () => {
mockGet.mockResolvedValue({ data: undefined });
const { result } = renderHook(() => useCustomers(), { wrapper });
// Query should not execute
expect(result.current.isLoading).toBe(false);
expect(result.current.data).toBeUndefined();
expect(result.current.isFetched).toBe(false);
// API should not be called
expect(allEndUsersCall).not.toHaveBeenCalled();
});
it("should not execute query when userRole is not an admin role", async () => {
// Mock non-admin userRole
mockUseAuthorized.mockReturnValue({
accessToken: "test-access-token",
userRole: "member", // Not in all_admin_roles
userId: "test-user-id",
token: "test-token",
userEmail: "test@example.com",
premiumUser: false,
disabledPersonalKeyCreation: null,
showSSOBanner: false,
});
const { result } = renderHook(() => useCustomers(), { wrapper });
// Query should not execute
expect(result.current.isLoading).toBe(false);
expect(result.current.data).toBeUndefined();
expect(result.current.isFetched).toBe(false);
// API should not be called
expect(allEndUsersCall).not.toHaveBeenCalled();
});
it("should not execute query when userRole is null", async () => {
// Mock null userRole
mockUseAuthorized.mockReturnValue({
accessToken: "test-access-token",
userRole: null,
userId: "test-user-id",
token: "test-token",
userEmail: "test@example.com",
premiumUser: false,
disabledPersonalKeyCreation: null,
showSSOBanner: false,
});
const { result } = renderHook(() => useCustomers(), { wrapper });
// Query should not execute
expect(result.current.isLoading).toBe(false);
expect(result.current.data).toBeUndefined();
expect(result.current.isFetched).toBe(false);
// API should not be called
expect(allEndUsersCall).not.toHaveBeenCalled();
});
it("should not execute query when userRole is empty string", async () => {
// Mock empty string userRole
mockUseAuthorized.mockReturnValue({
accessToken: "test-access-token",
userRole: "",
userId: "test-user-id",
token: "test-token",
userEmail: "test@example.com",
premiumUser: false,
disabledPersonalKeyCreation: null,
showSSOBanner: false,
});
const { result } = renderHook(() => useCustomers(), { wrapper });
// Query should not execute
expect(result.current.isLoading).toBe(false);
expect(result.current.data).toBeUndefined();
expect(result.current.isFetched).toBe(false);
// API should not be called
expect(allEndUsersCall).not.toHaveBeenCalled();
});
it("should not execute query when both accessToken and userRole are missing", async () => {
// Mock both auth values missing
mockUseAuthorized.mockReturnValue({
accessToken: null,
userRole: null,
userId: "test-user-id",
token: null,
userEmail: "test@example.com",
premiumUser: false,
disabledPersonalKeyCreation: null,
showSSOBanner: false,
});
const { result } = renderHook(() => useCustomers(), { wrapper });
// Query should not execute
expect(result.current.isLoading).toBe(false);
expect(result.current.data).toBeUndefined();
expect(result.current.isFetched).toBe(false);
// API should not be called
expect(allEndUsersCall).not.toHaveBeenCalled();
});
it("should execute query when accessToken is present and userRole is Admin", async () => {
// Mock successful API call
(allEndUsersCall as any).mockResolvedValue(mockCustomersResponse);
// Ensure auth values are set (already done in beforeEach)
const { result } = renderHook(() => useCustomers(), { wrapper });
// Wait for query to execute
await waitFor(() => {
expect(result.current.isLoading).toBe(false);
});
expect(allEndUsersCall).toHaveBeenCalledWith("test-access-token");
expect(allEndUsersCall).toHaveBeenCalledTimes(1);
});
it("should execute query when accessToken is present and userRole is proxy_admin", async () => {
// Mock successful API call
(allEndUsersCall as any).mockResolvedValue(mockCustomersResponse);
// Mock proxy_admin role
mockUseAuthorized.mockReturnValue({
accessToken: "test-access-token",
userRole: "proxy_admin",
userId: "test-user-id",
token: "test-token",
userEmail: "test@example.com",
premiumUser: false,
disabledPersonalKeyCreation: null,
showSSOBanner: false,
});
const { result } = renderHook(() => useCustomers(), { wrapper });
// Wait for query to execute
await waitFor(() => {
expect(result.current.isLoading).toBe(false);
});
expect(allEndUsersCall).toHaveBeenCalledWith("test-access-token");
expect(allEndUsersCall).toHaveBeenCalledTimes(1);
});
it("should return empty customers array when API returns empty data", async () => {
// Mock API returning empty customers array
(allEndUsersCall as any).mockResolvedValue([]);
const { result } = renderHook(() => useCustomers(), { wrapper });
// Wait for success
await waitFor(() => {
expect(result.current.isLoading).toBe(false);
expect(result.current.isSuccess).toBe(true);
});
expect(result.current.data).toEqual([]);
expect(allEndUsersCall).toHaveBeenCalledWith("test-access-token");
});
it("should handle network timeout error", async () => {
const timeoutError = new Error("Network timeout");
// Mock network timeout
(allEndUsersCall as any).mockRejectedValue(timeoutError);
it("does not fetch when the access token is missing", () => {
mockUseAuthorized.mockReturnValue({ ...authorized, accessToken: null, token: null });
const { result } = renderHook(() => useCustomers(), { wrapper });
// Wait for error
await waitFor(() => {
expect(result.current.isError).toBe(true);
});
expect(result.current.isFetched).toBe(false);
expect(mockGet).not.toHaveBeenCalled();
});
expect(result.current.error).toEqual(timeoutError);
expect(result.current.data).toBeUndefined();
it("does not fetch when the user is not an admin", () => {
mockUseAuthorized.mockReturnValue({ ...authorized, userRole: "member" });
const { result } = renderHook(() => useCustomers(), { wrapper });
expect(result.current.isFetched).toBe(false);
expect(mockGet).not.toHaveBeenCalled();
});
});

37
ui/litellm-dashboard/src/app/(dashboard)/hooks/customers/useCustomers.ts
View file

@ -1,42 +1,19 @@
import { allEndUsersCall } from "@/components/networking";
import { useQuery } from "@tanstack/react-query";
import { createQueryKeys } from "../common/queryKeysFactory";
import { fetchClient } from "@/lib/http/api";
import { all_admin_roles } from "@/utils/roles";
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
import type { components } from "@/lib/http/schema";
export type EndUser = components["schemas"]["CustomerResponse"];
const customersKeys = createQueryKeys("customers");
export interface Customer {
user_id: string;
alias?: string | null;
spend: number;
blocked: boolean;
allowed_model_region?: string | null;
default_model?: string | null;
budget_id?: string | null;
litellm_budget_table?: {
budget_id: string;
max_budget?: number | null;
soft_budget?: number | null;
max_parallel_requests?: number | null;
tpm_limit?: number | null;
rpm_limit?: number | null;
model_max_budget?: Record<string, unknown> | null;
budget_duration?: string | null;
budget_reset_at?: string | null;
created_at: string;
created_by: string;
updated_at: string;
updated_by: string;
} | null;
}
export type CustomersResponse = Customer[];
export const useCustomers = () => {
const { accessToken, userRole } = useAuthorized();
return useQuery<CustomersResponse>({
return useQuery({
queryKey: customersKeys.list({}),
queryFn: async () => await allEndUsersCall(accessToken!),
queryFn: async () => (await fetchClient.GET("/customer/list")).data ?? [],
enabled: Boolean(accessToken) && all_admin_roles.includes(userRole!),
});
};

112
ui/litellm-dashboard/src/app/(dashboard)/workflows/WorkflowRuns.tsx
View file

@ -1,9 +1,16 @@
import React, { useState, useEffect, useCallback, useMemo } from "react";
import { Button, Collapse, Drawer, Empty, Spin, Tooltip, Typography } from "antd";
import { ReloadOutlined } from "@ant-design/icons";
import type { ColumnDef } from "@tanstack/react-table";
import type { ColumnDef, ColumnFiltersState } from "@tanstack/react-table";
import { proxyBaseUrl } from "@/components/networking";
import { DataTable } from "@/components/shared/DataTable";
import {
DataTable,
DataTableFilterDrawer,
DataTableFilterField,
DataTableToolbar,
} from "@/components/shared/DataTable";
import { Input } from "@/components/ui/input";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
const { Text } = Typography;
@ -59,6 +66,15 @@ const STATUS_DOT: Record<RunStatus, string> = {
failed: "#ef4444",
};
const RUN_STATUS_OPTIONS: RunStatus[] = ["pending", "running", "paused", "completed", "failed"];
const STATUS_LABELS: Record<RunStatus, string> = {
pending: "Pending",
running: "Running",
paused: "Paused",
completed: "Completed",
failed: "Failed",
};
const EVENT_COLOR: Record<string, { bar: string; border: string; text: string }> = {
"step.started": { bar: "#f0fdf4", border: "#86efac", text: "#16a34a" },
"step.failed": { bar: "#fef2f2", border: "#fca5a5", text: "#dc2626" },
@ -482,6 +498,9 @@ const WorkflowRuns: React.FC<WorkflowRunsProps> = ({ accessToken }) => {
const [messages, setMessages] = useState<WorkflowRunMessage[]>([]);
const [loadingDetail, setLoadingDetail] = useState(false);
const [drawerOpen, setDrawerOpen] = useState(false);
const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>([]);
const [globalFilter, setGlobalFilter] = useState("");
const [filtersOpen, setFiltersOpen] = useState(false);
const fetchRuns = useCallback(async () => {
if (!accessToken) return;
@ -547,7 +566,9 @@ const WorkflowRuns: React.FC<WorkflowRunsProps> = ({ accessToken }) => {
() => [
{
id: "run",
accessorFn: (row) => `${runTitle(row)} ${row.run_id}`,
header: "Run",
meta: { title: "Run", skeleton: "twoLine" },
cell: ({ row }) => {
const run = row.original;
return (
@ -564,13 +585,18 @@ const WorkflowRuns: React.FC<WorkflowRunsProps> = ({ accessToken }) => {
{
accessorKey: "workflow_type",
header: "Type",
meta: { title: "Type" },
filterFn: "includesString",
cell: ({ row }) => (
<span style={{ fontFamily: "monospace", fontSize: 12, color: "#71717a" }}>{row.original.workflow_type}</span>
),
},
{
id: "status",
accessorKey: "status",
header: "Status",
meta: { title: "Status" },
filterFn: "equalsString",
cell: ({ row }) => {
const run = row.original;
return (
@ -586,6 +612,7 @@ const WorkflowRuns: React.FC<WorkflowRunsProps> = ({ accessToken }) => {
{
accessorKey: "created_at",
header: "Created",
meta: { title: "Created" },
cell: ({ row }) => <span style={{ fontSize: 12, color: "#a1a1aa" }}>{timeAgo(row.original.created_at)}</span>,
},
],
@ -603,28 +630,11 @@ const WorkflowRuns: React.FC<WorkflowRunsProps> = ({ accessToken }) => {
}}
>
{/* page header */}
<div
style={{
display: "flex",
alignItems: "center",
justifyContent: "space-between",
marginBottom: 20,
}}
>
<div>
<div style={{ fontSize: 18, fontWeight: 600, color: "#18181b" }}>Workflow Runs</div>
<div style={{ fontSize: 13, color: "#71717a", marginTop: 2 }}>
Durable state tracking for agents and automated workflows
</div>
<div style={{ marginBottom: 20 }}>
<div style={{ fontSize: 18, fontWeight: 600, color: "#18181b" }}>Workflow Runs</div>
<div style={{ fontSize: 13, color: "#71717a", marginTop: 2 }}>
Durable state tracking for agents and automated workflows
</div>
<Button
icon={<ReloadOutlined />}
onClick={fetchRuns}
loading={loadingRuns}
style={{ color: "#71717a", borderColor: "#e4e4e7" }}
>
Refresh
</Button>
</div>
<DataTable
@ -641,8 +651,64 @@ const WorkflowRuns: React.FC<WorkflowRunsProps> = ({ accessToken }) => {
}
paginationMode="client"
pageSizeOptions={[50, 100]}
filterMode="client"
columnFilters={columnFilters}
onColumnFiltersChange={setColumnFilters}
globalFilter={globalFilter}
onGlobalFilterChange={setGlobalFilter}
onRowClick={fetchRunDetail}
size="compact"
toolbar={(table) => (
<>
<DataTableToolbar
table={table}
searchValue={globalFilter}
onSearchChange={setGlobalFilter}
searchPlaceholder="Search runs…"
onRefresh={fetchRuns}
isRefreshing={loadingRuns}
onOpenFilters={() => setFiltersOpen(true)}
/>
<DataTableFilterDrawer
table={table}
open={filtersOpen}
onOpenChange={setFiltersOpen}
title="Filters"
description="Narrow down workflow runs"
>
{({ get, set }) => (
<>
<DataTableFilterField label="Status">
<Select
items={STATUS_LABELS}
value={(get("status") as string) || null}
onValueChange={(value: string | null) => set("status", value ?? "")}
>
<SelectTrigger className="w-full">
<SelectValue placeholder="All statuses" />
</SelectTrigger>
<SelectContent>
<SelectItem value={null}>All statuses</SelectItem>
{RUN_STATUS_OPTIONS.map((status) => (
<SelectItem key={status} value={status}>
{STATUS_LABELS[status]}
</SelectItem>
))}
</SelectContent>
</Select>
</DataTableFilterField>
<DataTableFilterField label="Type">
<Input
value={(get("workflow_type") as string) ?? ""}
onChange={(event) => set("workflow_type", event.target.value)}
placeholder="Filter by type…"
/>
</DataTableFilterField>
</>
)}
</DataTableFilterDrawer>
</>
)}
/>
{/* detail drawer */}

57
ui/litellm-dashboard/src/components/DashboardHeader.test.tsx Normal file
View file

@ -0,0 +1,57 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { act, fireEvent, render, screen, waitFor } from "@testing-library/react";
import { DashboardHeader } from "./DashboardHeader";
const { mockUsePluginMode, mockUseUISettings, state } = vi.hoisted(() => {
const state = {
plugins: [] as { name: string; display_name: string; url: string }[],
enableChatUI: false,
};
return {
state,
mockUsePluginMode: vi.fn(() => ({ mode: "ai-gateway", setMode: vi.fn(), plugins: state.plugins })),
mockUseUISettings: vi.fn(() => ({ data: { values: { enable_chat_ui: state.enableChatUI } } })),
};
});
vi.mock("@/contexts/PluginModeContext", () => ({ usePluginMode: mockUsePluginMode }));
vi.mock("@/app/(dashboard)/hooks/uiSettings/useUISettings", () => ({ useUISettings: mockUseUISettings }));
vi.mock("next/navigation", () => ({ usePathname: () => "/ui/" }));
vi.mock("@/utils/migratedPages", () => ({ migratedHref: (seg: string) => `/ui/${seg}` }));
vi.mock("@/hooks/useWorker", () => ({ useWorker: () => ({ isControlPlane: false, selectedWorker: null }) }));
vi.mock("@/app/(dashboard)/hooks/useDisableShowPrompts", () => ({ useDisableShowPrompts: () => false }));
vi.mock("@/components/Navbar/BlogDropdown/BlogDropdown", () => ({ BlogDropdown: () => null }));
vi.mock("@/components/Navbar/CommunityEngagementButtons/CommunityEngagementButtons", () => ({
CommunityEngagementButtons: () => null,
}));
vi.mock("@/components/Navbar/NotificationsBell/NotificationsBell", () => ({ NotificationsBell: () => null }));
vi.mock("@/components/Navbar/WorkerDropdown/WorkerDropdown", () => ({ default: () => null }));
describe("DashboardHeader breadcrumb", () => {
afterEach(() => {
state.plugins = [];
state.enableChatUI = false;
});
it("roots the breadcrumb in the AI Gateway selector (with a Chat option) and drops the static section crumb when the selector is available", async () => {
state.enableChatUI = true;
render(<DashboardHeader page="logs" />);
expect(screen.getByText("Logs")).toBeInTheDocument();
expect(screen.queryByText("Observability")).not.toBeInTheDocument();
const selector = screen.getByRole("button", { name: /AI Gateway/i });
act(() => {
fireEvent.click(selector);
});
await waitFor(() => expect(screen.getByText("Chat")).toBeInTheDocument());
});
it("keeps the AI Gateway selector at the root even when there is nothing to switch to (discovery)", () => {
render(<DashboardHeader page="logs" />);
expect(screen.getByRole("button", { name: /AI Gateway/i })).toBeInTheDocument();
expect(screen.getByText("Logs")).toBeInTheDocument();
expect(screen.queryByText("Observability")).not.toBeInTheDocument();
});
});

14
ui/litellm-dashboard/src/components/DashboardHeader.tsx
View file

@ -27,7 +27,7 @@ interface DashboardHeaderProps {
// Top bar for the dashboard shell. Sits only over the content column (the brand
// lives in the sidebar header); mirrors the design's breadcrumb-left / tools-right layout.
export function DashboardHeader({ page }: DashboardHeaderProps) {
const { section, title } = getBreadcrumb(page);
const { title } = getBreadcrumb(page);
const { isControlPlane, selectedWorker } = useWorker();
const showWorkerSwitch = isControlPlane && selectedWorker !== null;
const hideCommunityLinks = useDisableShowPrompts();
@ -44,12 +44,10 @@ export function DashboardHeader({ page }: DashboardHeaderProps) {
<header className="flex h-14 flex-none items-center justify-between gap-4 border-b border-border bg-background px-4">
<Breadcrumb className="min-w-0">
<BreadcrumbList className="flex-nowrap">
{section && (
<>
<BreadcrumbItem className="whitespace-nowrap">{section}</BreadcrumbItem>
<BreadcrumbSeparator />
</>
)}
<BreadcrumbItem className="flex-none">
<ViewSwitcher />
</BreadcrumbItem>
<BreadcrumbSeparator />
<BreadcrumbItem className="min-w-0">
<BreadcrumbPage className="truncate">{title}</BreadcrumbPage>
</BreadcrumbItem>
@ -76,8 +74,6 @@ export function DashboardHeader({ page }: DashboardHeaderProps) {
{!hideCommunityLinks && <CommunityEngagementButtons />}
<Separator orientation="vertical" className="mx-1.5 h-5" />
<NotificationsBell />
<Separator orientation="vertical" className="mx-1.5 h-5" />
<ViewSwitcher />
</div>
</header>
);

32
ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.test.tsx
View file

@ -49,9 +49,23 @@ describe("ViewSwitcher", () => {
state.setMode.mockClear();
});
it("renders nothing with no plugins, chat disabled, and a non-admin user", () => {
const { container } = render(<ViewSwitcher />);
expect(container.firstChild).toBeNull();
it("still renders the selector with a disabled Chat hint when there are no plugins and chat is off", async () => {
render(<ViewSwitcher />);
const button = screen.getByRole("button");
expect(button).toHaveTextContent("AI Gateway");
act(() => {
fireEvent.click(button);
});
await waitFor(() => expect(screen.getByText("Chat")).toBeInTheDocument());
expect(screen.getByText(/Admins can enable in Settings/i)).toBeInTheDocument();
act(() => {
fireEvent.click(screen.getByText("Chat"));
});
expect(assignSpy).not.toHaveBeenCalled();
expect(state.setMode).not.toHaveBeenCalled();
});
it("labels the button from the active plugin and lists AI Gateway + each plugin", async () => {
@ -95,7 +109,7 @@ describe("ViewSwitcher", () => {
fireEvent.click(screen.getByRole("button"));
});
await waitFor(() => expect(screen.getByText("Chat")).toBeInTheDocument());
expect(screen.queryByText(/Enable in Admin Settings/i)).not.toBeInTheDocument();
expect(screen.queryByText(/Admins can enable in Settings/i)).not.toBeInTheDocument();
act(() => {
fireEvent.click(screen.getByText("Chat"));
@ -121,7 +135,7 @@ describe("ViewSwitcher", () => {
expect(assignSpy).toHaveBeenCalledWith("/ui/");
});
it("hides the Chat entry from everyone when disabled", async () => {
it("shows Chat as a disabled, non-navigating entry with an admin hint when disabled", async () => {
state.enableChatUI = false;
state.plugins = [{ name: "obs", display_name: "Observability", url: "http://localhost:9000" }];
render(<ViewSwitcher />);
@ -130,6 +144,12 @@ describe("ViewSwitcher", () => {
fireEvent.click(screen.getByRole("button"));
});
await waitFor(() => expect(screen.getByText("Observability")).toBeInTheDocument());
expect(screen.queryByText("Chat")).not.toBeInTheDocument();
expect(screen.getByText("Chat")).toBeInTheDocument();
expect(screen.getByText(/Admins can enable in Settings/i)).toBeInTheDocument();
act(() => {
fireEvent.click(screen.getByText("Chat"));
});
expect(assignSpy).not.toHaveBeenCalled();
});
});

52
ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.tsx
View file

@ -1,7 +1,8 @@
import React from "react";
import { usePathname } from "next/navigation";
import { Dropdown } from "antd";
import { AppstoreOutlined, CheckOutlined, DownOutlined } from "@ant-design/icons";
import { AppstoreOutlined, CheckOutlined } from "@ant-design/icons";
import { ChevronsUpDown } from "lucide-react";
import type { MenuProps } from "antd";
import { usePluginMode } from "@/contexts/PluginModeContext";
import { useUISettings } from "@/app/(dashboard)/hooks/uiSettings/useUISettings";
@ -17,8 +18,6 @@ export default function ViewSwitcher() {
const chatEnabled = Boolean(uiSettings?.values?.enable_chat_ui);
if (plugins.length === 0 && !chatEnabled) return null;
const chatHref = migratedHref(CHAT);
const normalizedPathname = (pathname ?? "").replace(/\/+$/, "");
const isChatRoute = chatEnabled && (normalizedPathname === chatHref || normalizedPathname.startsWith(`${chatHref}/`));
@ -30,6 +29,29 @@ export default function ViewSwitcher() {
...plugins.map((p) => ({ key: p.name, label: p.display_name })),
];
const chatItem = chatEnabled
? {
key: CHAT,
label: (
<div className="flex items-center justify-between gap-6 py-0.5">
<span className="font-medium">Chat</span>
{isChatRoute && <CheckOutlined className="text-blue-600" />}
</div>
),
}
: {
key: CHAT,
disabled: true,
label: (
<div className="flex max-w-[220px] flex-col py-0.5">
<span className="font-medium">Chat</span>
<span className="whitespace-normal text-xs leading-snug text-muted-foreground">
Admins can enable in Settings
</span>
</div>
),
};
const items: MenuProps["items"] = [
...modeEntries.map((e) => ({
key: e.key,
@ -40,19 +62,7 @@ export default function ViewSwitcher() {
</div>
),
})),
...(chatEnabled
? [
{
key: CHAT,
label: (
<div className="flex items-center justify-between gap-6 py-0.5">
<span className="font-medium">Chat</span>
{isChatRoute && <CheckOutlined className="text-blue-600" />}
</div>
),
},
]
: []),
chatItem,
];
const onClick: MenuProps["onClick"] = ({ key }) => {
@ -72,11 +82,13 @@ export default function ViewSwitcher() {
<Dropdown menu={{ items, onClick, selectedKeys: [isChatRoute ? CHAT : mode] }} trigger={["click"]}>
<button
type="button"
className="flex items-center gap-2 rounded-md border border-gray-200 px-2.5 py-1.5 text-sm font-medium text-gray-700 transition-colors hover:bg-gray-50"
className="flex h-8 max-w-[220px] items-center gap-1.5 rounded-md border border-border bg-background pl-1.5 pr-2 text-sm font-medium text-foreground transition-colors hover:bg-accent"
>
<AppstoreOutlined className="text-gray-500" />
<span>{activeLabel}</span>
<DownOutlined className="text-[10px] text-gray-400" />
<span className="flex size-5 flex-none items-center justify-center rounded bg-muted text-muted-foreground">
<AppstoreOutlined className="text-[13px]" />
</span>
<span className="truncate">{activeLabel}</span>
<ChevronsUpDown className="size-3.5 flex-none text-muted-foreground" />
</button>
</Dropdown>
);

40
ui/litellm-dashboard/src/components/mcp_tools/DcrBridgeToggle.tsx Normal file
View file

@ -0,0 +1,40 @@
import React from "react";
import { Form, Switch, Tooltip } from "antd";
import { InfoCircleOutlined } from "@ant-design/icons";
import { isClientForwardedTokenMode } from "./types";
/**
* DCR-bridge toggle for the client-forwarded token modes (true_passthrough /
* oauth_delegate); self-gates to those two auth types and renders nothing
* otherwise. When on, OAuth-only clients like Claude Desktop can register and
* sign in through the gateway; when off, the gateway relays the upstream
* server's own OAuth metadata instead. `initialChecked` seeds the antd
* Form.Item `initialValue` (not the Switch's DOM defaultChecked): the create
* form defaults it on, the edit form seeds it from the stored value.
*/
export default function DcrBridgeToggle({
authType,
initialChecked,
}: {
authType?: string | null;
initialChecked?: boolean;
}) {
if (!isClientForwardedTokenMode(authType)) return null;
return (
<Form.Item
label={
<span className="text-sm font-medium text-gray-700 flex items-center">
Gateway-hosted sign-in (DCR bridge)
<Tooltip title="Lets OAuth-only clients like Claude Desktop register and sign in through the gateway. Turn off to relay the upstream server's own OAuth metadata instead (for clients pre-registered with the upstream IdP).">
<InfoCircleOutlined className="ml-2 text-blue-400 hover:text-blue-600 cursor-help" />
</Tooltip>
</span>
}
name="dcr_bridge"
valuePropName="checked"
initialValue={initialChecked}
>
<Switch />
</Form.Item>
);
}

54
ui/litellm-dashboard/src/components/mcp_tools/PassthroughAuthorizeSection.test.tsx Normal file
View file

@ -0,0 +1,54 @@
import React from "react";
import { describe, it, expect } from "vitest";
import { render, screen } from "@testing-library/react";
import { Form } from "antd";
import PassthroughAuthorizeSection from "./PassthroughAuthorizeSection";
const WithForm: React.FC<{ children: React.ReactNode }> = ({ children }) => {
const [form] = Form.useForm();
return <Form form={form}>{children}</Form>;
};
const noopFlow = { startOAuthFlow: () => {}, status: "idle", error: null, tokenResponse: null };
describe("PassthroughAuthorizeSection credential-class-aware copy", () => {
it("shows keep-existing copy when the credential class is unchanged (true_passthrough <-> oauth_delegate)", () => {
render(
<WithForm>
<PassthroughAuthorizeSection
authType="oauth_delegate"
oauthFlow={noopFlow}
isEditing
savedAuthType="true_passthrough"
/>
</WithForm>,
);
expect(screen.getByPlaceholderText("Leave blank to keep the currently saved app (if any)")).toBeInTheDocument();
expect(screen.getByPlaceholderText("Leave blank to keep the currently saved secret (if any)")).toBeInTheDocument();
});
it("shows the discard warning copy when switching from a different class (oauth2 -> true_passthrough)", () => {
render(
<WithForm>
<PassthroughAuthorizeSection
authType="true_passthrough"
oauthFlow={noopFlow}
isEditing
savedAuthType="oauth2"
/>
</WithForm>,
);
expect(screen.getByPlaceholderText("Leave blank to use dynamic client registration")).toBeInTheDocument();
expect(screen.getByPlaceholderText("Leave blank for public clients / PKCE")).toBeInTheDocument();
expect(screen.getByText(/Switching the auth type discards the previously saved app/)).toBeInTheDocument();
});
it("shows the keep+warn banner when the upstream may no longer match", () => {
render(
<WithForm>
<PassthroughAuthorizeSection authType="true_passthrough" oauthFlow={noopFlow} appMayNotMatchUpstream />
</WithForm>,
);
expect(screen.getByText(/registered for the previous upstream/)).toBeInTheDocument();
});
});

89
ui/litellm-dashboard/src/components/mcp_tools/PassthroughAuthorizeSection.tsx
View file

@ -1,6 +1,7 @@
import React from "react";
import { Button, Form, Input } from "antd";
import { isClientForwardedTokenMode } from "./types";
import { Button, Checkbox, Form, Input } from "antd";
import DcrBridgeToggle from "./DcrBridgeToggle";
import { credentialAuthClass, isClientForwardedTokenMode } from "./types";
interface PassthroughOAuthFlow {
startOAuthFlow: () => void | Promise<void>;
@ -11,20 +12,42 @@ interface PassthroughOAuthFlow {
/**
* Browser-only Authorize & Fetch for the client-forwarded token modes
* (true_passthrough / oauth_delegate). LiteLLM never stores upstream
* credentials for these modes, so the token obtained here lives in this
* browser session only: it is forwarded per-server for the tools preview and
* allowlist configuration, and is never written to the server row or the
* per-user credential store. The optional client credentials cover IdPs
* without dynamic client registration (e.g. a pre-registered Slack app) and
* ride the temporary authorize session only.
* (true_passthrough / oauth_delegate). Tokens are never stored: the token
* obtained here lives in this browser session only, forwarded per-server for
* the tools preview and allowlist configuration, and is never written to the
* server row or the per-user credential store. The optional OAuth client
* credentials cover IdPs without dynamic client registration (e.g. a
* pre-registered Slack app); unlike the token they ARE saved onto the server
* as declared config, so internal users' Authorize relays through the org's
* app instead of dead-ending on upstreams that cannot mint clients.
*
* Blank fields follow the same convention as the M2M credential fields. On
* create they mean "no app configured" (dynamic client registration). On edit
* they mean "keep existing" ONLY when the credential class is unchanged: the
* backend merges a partial update within the client-forwarded class, so a
* true_passthrough <-> oauth_delegate switch keeps the stored app, but a switch
* from a different class (e.g. oauth2) replaces it, so blanks then mean "no
* app". Removing a stored app is an explicit checkbox (edit only) that writes
* an explicit-null credential.
*/
export default function PassthroughAuthorizeSection({
authType,
oauthFlow,
dcrBridgeInitialChecked,
isEditing = false,
savedAuthType,
removeStoredApp = false,
onRemoveStoredAppChange,
appMayNotMatchUpstream = false,
}: {
authType?: string | null;
oauthFlow: PassthroughOAuthFlow;
dcrBridgeInitialChecked?: boolean;
isEditing?: boolean;
savedAuthType?: string | null;
removeStoredApp?: boolean;
onRemoveStoredAppChange?: (remove: boolean) => void;
appMayNotMatchUpstream?: boolean;
}) {
if (!isClientForwardedTokenMode(authType)) return null;
const authorizeButtonLabels: Record<string, string> = {
@ -32,32 +55,61 @@ export default function PassthroughAuthorizeSection({
exchanging: "Exchanging authorization code...",
};
const authorizeButtonLabel = authorizeButtonLabels[oauthFlow.status] ?? "Authorize & Fetch Tools (browser-only)";
// On edit, "keep existing" only holds when the stored credential class is unchanged; a cross-class
// switch (e.g. oauth2 -> true_passthrough) replaces credentials, so blanks then mean "no app".
const classUnchanged = isEditing && credentialAuthClass(savedAuthType) === credentialAuthClass(authType);
const clientIdPlaceholder = classUnchanged
? "Leave blank to keep the currently saved app (if any)"
: "Leave blank to use dynamic client registration";
const clientSecretPlaceholder = classUnchanged
? "Leave blank to keep the currently saved secret (if any)"
: "Leave blank for public clients / PKCE";
const clientIdExtra = classUnchanged
? "Set this to make everyone authorize through a specific app; required for upstreams without dynamic client registration (e.g. a pre-registered Slack app)."
: "Switching the auth type discards the previously saved app; enter a client ID here or leave blank to use dynamic client registration.";
return (
<div className="rounded-lg border border-dashed border-gray-300 p-4 space-y-2 mb-4">
<p className="text-sm text-gray-600">
Callers bring their own upstream token for this auth type, so LiteLLM stores no upstream credentials. To preview
tools and configure the tool allowlist, authorize against the upstream here: the token stays in this browser
session only and is never saved to LiteLLM.
Callers bring their own upstream token for this auth type, so LiteLLM never stores tokens. To preview tools and
configure the tool allowlist, authorize against the upstream here: the token stays in this browser session only
and is never saved to LiteLLM. An OAuth app configured below IS saved with the server, so internal users who
authorize from the Tools page go through it.
</p>
{appMayNotMatchUpstream && (
<p className="text-sm text-amber-600">
You changed the upstream URL or endpoints; the OAuth app entered here was registered for the previous upstream
and may not be valid. Update the client ID, or clear it to use dynamic client registration.
</p>
)}
<Form.Item
label={<span className="text-sm font-medium text-gray-700">OAuth Client ID (optional, not saved)</span>}
label={<span className="text-sm font-medium text-gray-700">OAuth Client ID (optional)</span>}
name={["credentials", "client_id"]}
extra="Only needed when the upstream does not support dynamic client registration (e.g. a pre-registered Slack app). Used for this browser authorization only."
extra={clientIdExtra}
>
<Input.Password
placeholder="Leave blank to use dynamic client registration"
placeholder={clientIdPlaceholder}
disabled={removeStoredApp}
className="rounded-lg border-gray-300 focus:border-blue-500 focus:ring-blue-500"
/>
</Form.Item>
<Form.Item
label={<span className="text-sm font-medium text-gray-700">OAuth Client Secret (optional, not saved)</span>}
label={<span className="text-sm font-medium text-gray-700">OAuth Client Secret (optional)</span>}
name={["credentials", "client_secret"]}
>
<Input.Password
placeholder="Leave blank for public clients / PKCE"
placeholder={clientSecretPlaceholder}
disabled={removeStoredApp}
className="rounded-lg border-gray-300 focus:border-blue-500 focus:ring-blue-500"
/>
</Form.Item>
<DcrBridgeToggle authType={authType} initialChecked={dcrBridgeInitialChecked} />
{isEditing && onRemoveStoredAppChange && (
<Checkbox checked={removeStoredApp} onChange={(e) => onRemoveStoredAppChange(e.target.checked)}>
<span className="text-sm text-gray-700">
Remove the saved OAuth app on save (the server goes back to dynamic client registration)
</span>
</Checkbox>
)}
<Button
onClick={oauthFlow.startOAuthFlow}
disabled={oauthFlow.status === "authorizing" || oauthFlow.status === "exchanging"}
@ -67,7 +119,8 @@ export default function PassthroughAuthorizeSection({
{oauthFlow.error && <p className="text-sm text-red-500">{oauthFlow.error}</p>}
{oauthFlow.status === "success" && oauthFlow.tokenResponse?.access_token && (
<p className="text-sm text-green-600">
Token held for this browser session. Tools can now be previewed and configured; nothing was saved to LiteLLM.
Token held for this browser session. Tools can now be previewed and configured; the token was not saved to
LiteLLM.
</p>
)}
</div>

613
ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx
View file

@ -31,6 +31,7 @@ const oauthHook = vi.hoisted(() => ({
| ((token: Record<string, unknown> | null, registeredClient?: { clientId?: string; clientSecret?: string }) => void)
| null,
getCredentials: null as (() => Record<string, unknown> | undefined) | null,
getTemporaryPayload: null as (() => Record<string, unknown> | null) | null,
}));
vi.mock("@/hooks/useMcpOAuthFlow", () => ({
useMcpOAuthFlow: (opts: {
@ -39,9 +40,11 @@ vi.mock("@/hooks/useMcpOAuthFlow", () => ({
registeredClient?: { clientId?: string; clientSecret?: string },
) => void;
getCredentials?: () => Record<string, unknown> | undefined;
getTemporaryPayload?: () => Record<string, unknown> | null;
}) => {
oauthHook.onTokenReceived = opts.onTokenReceived;
oauthHook.getCredentials = opts.getCredentials ?? null;
oauthHook.getTemporaryPayload = opts.getTemporaryPayload ?? null;
return {
startOAuthFlow: vi.fn(),
status: "idle",
@ -202,8 +205,8 @@ describe("CreateMCPServer", () => {
await waitFor(() => {
expect(screen.getByRole("button", { name: "Authorize & Fetch Tools (browser-only)" })).toBeInTheDocument();
});
expect(screen.getByText("OAuth Client ID (optional, not saved)")).toBeInTheDocument();
expect(screen.getByText("OAuth Client Secret (optional, not saved)")).toBeInTheDocument();
expect(screen.getByText("OAuth Client ID (optional)")).toBeInTheDocument();
expect(screen.getByText("OAuth Client Secret (optional)")).toBeInTheDocument();
},
);
@ -436,6 +439,434 @@ describe("CreateMCPServer", () => {
);
});
it.each([
["true_passthrough", "True Passthrough (no LiteLLM auth)"],
["oauth_delegate", "OAuth Delegate (client-supplied upstream token)"],
])(
"persists admin-entered OAuth app credentials on create for %s while the token stays browser-held",
async (_authType, optionLabel) => {
oauthHook.tokenResponse = { access_token: "upstream-tok", token_type: "Bearer" };
await selectHttpTransport();
const user = userEvent.setup({ delay: null });
await user.type(getServerNameInput(), "CF_App_Server");
await user.type(screen.getByPlaceholderText("https://your-mcp-server.com"), "https://example.com/mcp");
await selectAntOption("Authentication", optionLabel);
// Admin declares the org's pre-registered upstream app; unlike the browser-authorized
// token, this is config and must survive onto the server row so internal users'
// Tools-page Authorize relays through it (required for non-DCR upstreams like Slack).
await user.type(
screen.getByPlaceholderText("Leave blank to use dynamic client registration"),
"org-app-client-id",
);
await user.type(screen.getByPlaceholderText("Leave blank for public clients / PKCE"), "org-app-secret");
await waitFor(() => expect(oauthHook.onTokenReceived).toBeTruthy());
await act(async () => {
oauthHook.onTokenReceived!({ access_token: "upstream-tok", token_type: "Bearer" }, undefined);
});
const createdServer = {
server_id: "new-cf-app-server",
server_name: "CF_App_Server",
alias: "CF_App_Server",
url: "https://example.com/mcp",
transport: "http",
auth_type: _authType,
created_at: "2024-01-01T00:00:00Z",
created_by: "user-1",
updated_at: "2024-01-01T00:00:00Z",
updated_by: "user-1",
};
vi.mocked(networking.createMCPServer).mockResolvedValue(createdServer);
const submitButton = screen.getByRole("button", { name: "Add MCP Server" });
await act(async () => {
fireEvent.click(submitButton);
});
await waitFor(() => expect(networking.createMCPServer).toHaveBeenCalledTimes(1));
const [, payload] = vi.mocked(networking.createMCPServer).mock.calls[0];
// The declared app persists; the browser-authorized token still appears nowhere in the
// payload and no per-user DB credential is written.
expect(payload.credentials).toEqual({
client_id: "org-app-client-id",
client_secret: "org-app-secret",
});
expect(JSON.stringify(payload)).not.toContain("upstream-tok");
expect(networking.storeMCPOAuthUserCredential).not.toHaveBeenCalled();
expect(setToken).toHaveBeenCalledWith(
"new-cf-app-server",
expect.objectContaining({ access_token: "upstream-tok" }),
undefined,
);
},
);
it("preserves admin-entered app credentials when the URL changes after authorize for true_passthrough", async () => {
await selectHttpTransport();
const user = userEvent.setup({ delay: null });
await user.type(getServerNameInput(), "CF_Keep_Server");
await user.type(screen.getByPlaceholderText("https://your-mcp-server.com"), "https://example.com/mcp");
await selectAntOption("Authentication", "True Passthrough (no LiteLLM auth)");
await user.type(
screen.getByPlaceholderText("Leave blank to use dynamic client registration"),
"org-app-client-id",
);
await user.type(screen.getByPlaceholderText("Leave blank for public clients / PKCE"), "org-app-secret");
await waitFor(() => expect(oauthHook.onTokenReceived).toBeTruthy());
await act(async () => {
oauthHook.onTokenReceived!({ access_token: "upstream-tok", token_type: "Bearer" }, undefined);
});
// Editing the URL after authorize invalidates the held token (identity change), but the
// declared app is config, not minted material: it must survive the invalidation instead of
// being silently reset, or the server would persist without the configured app.
await act(async () => {
fireEvent.change(screen.getByPlaceholderText("https://your-mcp-server.com"), {
target: { value: "https://other.example.com/mcp" },
});
});
const keptAppServer = {
server_id: "kept-app-server",
server_name: "CF_Keep_Server",
alias: "CF_Keep_Server",
url: "https://other.example.com/mcp",
transport: "http",
auth_type: "true_passthrough",
created_at: "2024-01-01T00:00:00Z",
created_by: "user-1",
updated_at: "2024-01-01T00:00:00Z",
updated_by: "user-1",
};
vi.mocked(networking.createMCPServer).mockResolvedValue(keptAppServer);
await act(async () => {
fireEvent.click(screen.getByRole("button", { name: "Add MCP Server" }));
});
await waitFor(() => expect(networking.createMCPServer).toHaveBeenCalledTimes(1));
const [, payload] = vi.mocked(networking.createMCPServer).mock.calls[0];
expect(payload.url).toBe("https://other.example.com/mcp");
expect(payload.credentials).toEqual({
client_id: "org-app-client-id",
client_secret: "org-app-secret",
});
expect(JSON.stringify(payload)).not.toContain("upstream-tok");
});
it("wipes oauth2-minted credentials when the auth type switches to a client-forwarded mode", async () => {
await selectHttpTransport();
const user = userEvent.setup({ delay: null });
await user.type(getServerNameInput(), "Switch_Server");
await user.type(screen.getByPlaceholderText("https://your-mcp-server.com"), "https://example.com/mcp");
await selectAntOption("Authentication", "OAuth");
// The oauth2 onTokenReceived branch writes the fetched token AND the DCR client into
// form.credentials; both are minted for the oauth2 identity.
await waitFor(() => expect(oauthHook.onTokenReceived).toBeTruthy());
await act(async () => {
oauthHook.onTokenReceived!(
{ access_token: "oauth2-minted-tok", refresh_token: "oauth2-minted-refresh", token_type: "Bearer" },
{ clientId: "dcr-minted-client", clientSecret: "dcr-minted-secret" },
);
});
// Switching into a client-forwarded mode changes the identity with auth_type in the changed
// values, so the preserve carve-out must NOT apply: the minted material would otherwise ride
// into a mode that now persists credentials onto the server row.
await selectAntOption("Authentication", "True Passthrough (no LiteLLM auth)");
const switchedServer = {
server_id: "switched-server",
server_name: "Switch_Server",
alias: "Switch_Server",
url: "https://example.com/mcp",
transport: "http",
auth_type: "true_passthrough",
created_at: "2024-01-01T00:00:00Z",
created_by: "user-1",
updated_at: "2024-01-01T00:00:00Z",
updated_by: "user-1",
};
vi.mocked(networking.createMCPServer).mockResolvedValue(switchedServer);
await act(async () => {
fireEvent.click(screen.getByRole("button", { name: "Add MCP Server" }));
});
await waitFor(() => expect(networking.createMCPServer).toHaveBeenCalledTimes(1));
const [, payload] = vi.mocked(networking.createMCPServer).mock.calls[0];
expect(payload.credentials).toBeUndefined();
expect(JSON.stringify(payload)).not.toContain("dcr-minted-client");
expect(JSON.stringify(payload)).not.toContain("oauth2-minted-tok");
});
it("keeps the DCR-minted client out of form.credentials but reuses it via getCredentials", async () => {
await selectHttpTransport();
const user = userEvent.setup({ delay: null });
await user.type(getServerNameInput(), "DCR_Server");
await user.type(screen.getByPlaceholderText("https://your-mcp-server.com"), "https://example.com/mcp");
await selectAntOption("Authentication", "OAuth");
await waitFor(() => expect(oauthHook.onTokenReceived).toBeTruthy());
await act(async () => {
oauthHook.onTokenReceived!(
{ access_token: "oauth2-tok", token_type: "Bearer" },
{ clientId: "dcr-client", clientSecret: "dcr-secret" },
);
});
// The DCR client must NOT be in the form store (or it could be collected as a CF server's app),
// but getCredentials merges it so a re-authorize reuses the registered client instead of re-DCRing.
expect(oauthHook.getCredentials?.()?.client_id).toBe("dcr-client");
// getTemporaryPayload must mirror getCredentials for oauth2, or a re-authorize's temp session omits
// the registered client and useMcpOAuthFlow re-registers instead of reusing it.
expect(oauthHook.getTemporaryPayload?.()?.credentials).toMatchObject({ client_id: "dcr-client" });
});
it("clears the DCR ref and the upstream warning when the modal closes so nothing leaks to the next session", async () => {
const { rerender } = render(<CreateMCPServer {...defaultProps} />);
await selectAntOption("Transport Type", "Streamable HTTP");
await waitFor(() => expect(screen.getByPlaceholderText("https://your-mcp-server.com")).toBeInTheDocument());
const user = userEvent.setup({ delay: null });
await user.type(getServerNameInput(), "Leak_Server");
await user.type(screen.getByPlaceholderText("https://your-mcp-server.com"), "https://example.com/mcp");
await selectAntOption("Authentication", "OAuth");
await waitFor(() => expect(oauthHook.onTokenReceived).toBeTruthy());
await act(async () => {
oauthHook.onTokenReceived!(
{ access_token: "oauth2-tok", token_type: "Bearer" },
{ clientId: "leak-client", clientSecret: "leak-secret" },
);
});
// Ref is held while the modal is open.
expect(oauthHook.getCredentials?.()?.client_id).toBe("leak-client");
// A parent dismiss (isModalVisible -> false) that does not route through Cancel/Create must still
// clear the DCR ref, or the next server's oauth2 submit would carry this server's registered client.
await act(async () => {
rerender(<CreateMCPServer {...defaultProps} isModalVisible={false} />);
});
expect(oauthHook.getCredentials?.()?.client_id).toBeUndefined();
expect(oauthHook.getTemporaryPayload?.()?.credentials ?? {}).not.toMatchObject({ client_id: "leak-client" });
});
it("persists the DCR client on an oauth2 submit via the ref", async () => {
await selectHttpTransport();
const user = userEvent.setup({ delay: null });
await user.type(getServerNameInput(), "DCR_Submit_Server");
await user.type(screen.getByPlaceholderText("https://your-mcp-server.com"), "https://example.com/mcp");
await selectAntOption("Authentication", "OAuth");
await waitFor(() => expect(oauthHook.onTokenReceived).toBeTruthy());
await act(async () => {
oauthHook.onTokenReceived!(
{ access_token: "oauth2-tok", token_type: "Bearer" },
{ clientId: "dcr-client", clientSecret: "dcr-secret" },
);
});
const dcrSubmitServer = {
server_id: "dcr-submit",
server_name: "DCR_Submit_Server",
alias: "DCR_Submit_Server",
url: "https://example.com/mcp",
transport: "http",
auth_type: "oauth2",
created_at: "2024-01-01T00:00:00Z",
created_by: "user-1",
updated_at: "2024-01-01T00:00:00Z",
updated_by: "user-1",
};
vi.mocked(networking.createMCPServer).mockResolvedValue(dcrSubmitServer);
await act(async () => {
fireEvent.click(screen.getByRole("button", { name: "Add MCP Server" }));
});
await waitFor(() => expect(networking.createMCPServer).toHaveBeenCalledTimes(1));
const [, payload] = vi.mocked(networking.createMCPServer).mock.calls[0];
expect(payload.credentials.client_id).toBe("dcr-client");
expect(payload.credentials.client_secret).toBe("dcr-secret");
});
// These two tests drive multiple antd auth-type switches; use single-shot fireEvent.change for the
// text fields (not per-keystroke userEvent.type) and a wider timeout so they do not flake under CI
// resource contention. The behavior under test is the credential preserve across the switches.
const fillText = (el: HTMLElement, value: string) => fireEvent.change(el, { target: { value } });
it("preserves the typed app across a switch between the two client-forwarded modes", async () => {
await selectHttpTransport();
fillText(getServerNameInput(), "CF_Switch_Keep");
fillText(screen.getByPlaceholderText("https://your-mcp-server.com"), "https://example.com/mcp");
await selectAntOption("Authentication", "True Passthrough (no LiteLLM auth)");
fillText(screen.getByPlaceholderText("Leave blank to use dynamic client registration"), "app-id");
fillText(screen.getByPlaceholderText("Leave blank for public clients / PKCE"), "app-secret");
await waitFor(() => expect(oauthHook.onTokenReceived).toBeTruthy());
await act(async () => {
oauthHook.onTokenReceived!({ access_token: "cf-tok", token_type: "Bearer" }, undefined);
});
await selectAntOption("Authentication", "OAuth Delegate (client-supplied upstream token)");
const switched = {
server_id: "cf-switch-keep",
server_name: "CF_Switch_Keep",
alias: "CF_Switch_Keep",
url: "https://example.com/mcp",
transport: "http",
auth_type: "oauth_delegate",
created_at: "2024-01-01T00:00:00Z",
created_by: "user-1",
updated_at: "2024-01-01T00:00:00Z",
updated_by: "user-1",
};
vi.mocked(networking.createMCPServer).mockResolvedValue(switched);
await act(async () => {
fireEvent.click(screen.getByRole("button", { name: "Add MCP Server" }));
});
await waitFor(() => expect(networking.createMCPServer).toHaveBeenCalledTimes(1));
const [, payload] = vi.mocked(networking.createMCPServer).mock.calls[0];
expect(payload.credentials).toEqual({ client_id: "app-id", client_secret: "app-secret" });
}, 60_000);
it("preserves the typed app across a client-forwarded -> oauth2 -> client-forwarded round trip", async () => {
await selectHttpTransport();
fillText(getServerNameInput(), "CF_Round");
fillText(screen.getByPlaceholderText("https://your-mcp-server.com"), "https://example.com/mcp");
await selectAntOption("Authentication", "True Passthrough (no LiteLLM auth)");
fillText(screen.getByPlaceholderText("Leave blank to use dynamic client registration"), "app-id");
fillText(screen.getByPlaceholderText("Leave blank for public clients / PKCE"), "app-secret");
await waitFor(() => expect(oauthHook.onTokenReceived).toBeTruthy());
await act(async () => {
oauthHook.onTokenReceived!({ access_token: "cf-tok", token_type: "Bearer" }, undefined);
});
await selectAntOption("Authentication", "OAuth");
await selectAntOption("Authentication", "True Passthrough (no LiteLLM auth)");
const cfRoundServer = {
server_id: "cf-round",
server_name: "CF_Round",
alias: "CF_Round",
url: "https://example.com/mcp",
transport: "http",
auth_type: "true_passthrough",
created_at: "2024-01-01T00:00:00Z",
created_by: "user-1",
updated_at: "2024-01-01T00:00:00Z",
updated_by: "user-1",
};
vi.mocked(networking.createMCPServer).mockResolvedValue(cfRoundServer);
await act(async () => {
fireEvent.click(screen.getByRole("button", { name: "Add MCP Server" }));
});
await waitFor(() => expect(networking.createMCPServer).toHaveBeenCalledTimes(1));
const [, payload] = vi.mocked(networking.createMCPServer).mock.calls[0];
expect(payload.credentials).toEqual({ client_id: "app-id", client_secret: "app-secret" });
}, 60_000);
it("keeps the typed app but warns when the URL changes after a client-forwarded authorize", async () => {
await selectHttpTransport();
const user = userEvent.setup({ delay: null });
await user.type(getServerNameInput(), "CF_Warn");
await user.type(screen.getByPlaceholderText("https://your-mcp-server.com"), "https://example.com/mcp");
await selectAntOption("Authentication", "True Passthrough (no LiteLLM auth)");
await user.type(screen.getByPlaceholderText("Leave blank to use dynamic client registration"), "app-id");
await waitFor(() => expect(oauthHook.onTokenReceived).toBeTruthy());
await act(async () => {
oauthHook.onTokenReceived!({ access_token: "cf-tok", token_type: "Bearer" }, undefined);
});
await act(async () => {
fireEvent.change(screen.getByPlaceholderText("https://your-mcp-server.com"), {
target: { value: "https://other.example.com/mcp" },
});
});
// Keep + warn: the app stays in the field, and a non-blocking warning appears.
expect(screen.getByText(/OAuth app entered here was registered for the previous upstream/)).toBeInTheDocument();
});
it("keeps client_secret when only client_id is edited after a client-forwarded authorize", async () => {
await selectHttpTransport();
const user = userEvent.setup({ delay: null });
await user.type(getServerNameInput(), "CF_Keystroke");
await user.type(screen.getByPlaceholderText("https://your-mcp-server.com"), "https://example.com/mcp");
await selectAntOption("Authentication", "True Passthrough (no LiteLLM auth)");
await user.type(screen.getByPlaceholderText("Leave blank to use dynamic client registration"), "app-id");
await user.type(screen.getByPlaceholderText("Leave blank for public clients / PKCE"), "app-secret");
await waitFor(() => expect(oauthHook.onTokenReceived).toBeTruthy());
await act(async () => {
oauthHook.onTokenReceived!({ access_token: "cf-tok", token_type: "Bearer" }, undefined);
});
// Editing only client_id fires an invalidation whose changedValues carries only the client_id
// sub-field; the preserve + deep-merge re-apply must keep client_secret from being dropped.
await user.type(screen.getByPlaceholderText("Leave blank to use dynamic client registration"), "2");
const cfKeystrokeServer = {
server_id: "cf-keystroke",
server_name: "CF_Keystroke",
alias: "CF_Keystroke",
url: "https://example.com/mcp",
transport: "http",
auth_type: "true_passthrough",
created_at: "2024-01-01T00:00:00Z",
created_by: "user-1",
updated_at: "2024-01-01T00:00:00Z",
updated_by: "user-1",
};
vi.mocked(networking.createMCPServer).mockResolvedValue(cfKeystrokeServer);
await act(async () => {
fireEvent.click(screen.getByRole("button", { name: "Add MCP Server" }));
});
await waitFor(() => expect(networking.createMCPServer).toHaveBeenCalledTimes(1));
const [, payload] = vi.mocked(networking.createMCPServer).mock.calls[0];
expect(payload.credentials).toEqual({ client_id: "app-id2", client_secret: "app-secret" });
});
it("replaces the token set on re-authorize instead of leaving stale siblings", async () => {
await selectHttpTransport();
const user = userEvent.setup({ delay: null });
await user.type(getServerNameInput(), "Reauth_Server");
await user.type(screen.getByPlaceholderText("https://your-mcp-server.com"), "https://example.com/mcp");
await selectAntOption("Authentication", "OAuth");
await waitFor(() => expect(oauthHook.onTokenReceived).toBeTruthy());
const firstToken = { access_token: "T1", refresh_token: "R1", scope: "read", token_type: "Bearer" };
await act(async () => {
oauthHook.onTokenReceived!(firstToken, undefined);
});
await act(async () => {
oauthHook.onTokenReceived!({ access_token: "T2", token_type: "Bearer" }, undefined);
});
const creds = oauthHook.getCredentials?.() ?? {};
expect(creds.access_token).toBe("T2");
expect(creds.refresh_token).toBeUndefined();
expect(creds.scope).toBeUndefined();
});
it("should not show auth value field when None auth type is selected", async () => {
await selectHttpTransport();
@ -1509,3 +1940,181 @@ describe("CreateMCPServer oauth2_flow persistence", () => {
expect(payload.oauth2_flow).toBeUndefined();
});
});
describe("CreateMCPServer dcr_bridge toggle", () => {
beforeEach(() => {
vi.clearAllMocks();
oauthHook.tokenResponse = null;
oauthHook.onTokenReceived = null;
});
const createdServer = {
server_id: "new-cf-server",
server_name: "CF_Server",
alias: "CF_Server",
url: "https://example.com/mcp",
transport: "http",
auth_type: "true_passthrough",
created_at: "2024-01-01T00:00:00Z",
created_by: "user-1",
updated_at: "2024-01-01T00:00:00Z",
updated_by: "user-1",
};
const getDcrToggle = () => document.getElementById("dcr_bridge");
async function setupHttpServerForm() {
render(<CreateMCPServer {...defaultProps} />);
await selectAntOption("Transport Type", "Streamable HTTP");
await waitFor(() => {
expect(screen.getByPlaceholderText("https://your-mcp-server.com")).toBeInTheDocument();
});
await act(async () => {
fireEvent.change(getServerNameInput(), { target: { value: "CF_Server" } });
});
await act(async () => {
fireEvent.change(screen.getByPlaceholderText("https://your-mcp-server.com"), {
target: { value: "https://example.com/mcp" },
});
});
}
async function submitCreate() {
const submitButton = screen.getByRole("button", { name: "Add MCP Server" });
await act(async () => {
fireEvent.click(submitButton);
});
await waitFor(() => {
expect(networking.createMCPServer).toHaveBeenCalledTimes(1);
});
const [, payload] = vi.mocked(networking.createMCPServer).mock.calls[0];
return payload;
}
it.each([["True Passthrough (no LiteLLM auth)"], ["OAuth Delegate (client-supplied upstream token)"]])(
"renders the toggle default-checked when %s is selected",
async (optionLabel) => {
await setupHttpServerForm();
await selectAntOption("Authentication", optionLabel);
await waitFor(() => {
expect(getDcrToggle()).toBeInTheDocument();
});
expect(screen.getByText("Gateway-hosted sign-in (DCR bridge)")).toBeInTheDocument();
expect(getDcrToggle()).toHaveAttribute("aria-checked", "true");
},
);
it.each([["None"], ["API Key"], ["OAuth"]])("does not render the toggle for %s", async (optionLabel) => {
await setupHttpServerForm();
await selectAntOption("Authentication", optionLabel);
await waitFor(() => {
expect(screen.queryByText("Gateway-hosted sign-in (DCR bridge)")).not.toBeInTheDocument();
});
expect(getDcrToggle()).not.toBeInTheDocument();
});
it("renders the toggle between the OAuth client fields and the Authorize button", async () => {
await setupHttpServerForm();
await selectAntOption("Authentication", "True Passthrough (no LiteLLM auth)");
await waitFor(() => {
expect(getDcrToggle()).toBeInTheDocument();
});
const toggle = getDcrToggle() as HTMLElement;
const secretInput = screen.getByPlaceholderText("Leave blank for public clients / PKCE");
const authorizeButton = screen.getByRole("button", { name: "Authorize & Fetch Tools (browser-only)" });
expect(secretInput.compareDocumentPosition(toggle) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy();
expect(toggle.compareDocumentPosition(authorizeButton) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy();
});
it.each([
["true_passthrough", "True Passthrough (no LiteLLM auth)"],
["oauth_delegate", "OAuth Delegate (client-supplied upstream token)"],
])("sends dcr_bridge: true by default on create for %s", async (authType, optionLabel) => {
vi.mocked(networking.createMCPServer).mockResolvedValue({ ...createdServer, auth_type: authType });
await setupHttpServerForm();
await selectAntOption("Authentication", optionLabel);
await waitFor(() => {
expect(getDcrToggle()).toBeInTheDocument();
});
const payload = await submitCreate();
expect(payload.dcr_bridge).toBe(true);
});
it("sends an explicit dcr_bridge: false when the toggle is unchecked", async () => {
vi.mocked(networking.createMCPServer).mockResolvedValue({ ...createdServer, auth_type: "oauth_delegate" });
await setupHttpServerForm();
await selectAntOption("Authentication", "OAuth Delegate (client-supplied upstream token)");
await waitFor(() => {
expect(getDcrToggle()).toBeInTheDocument();
});
await act(async () => {
fireEvent.click(getDcrToggle()!);
});
expect(getDcrToggle()).toHaveAttribute("aria-checked", "false");
const payload = await submitCreate();
expect(payload.dcr_bridge).toBe(false);
});
it.each([
["none", "None"],
["api_key", "API Key"],
["oauth2", "OAuth"],
])("forces an explicit dcr_bridge: false for %s", async (authType, optionLabel) => {
vi.mocked(networking.createMCPServer).mockResolvedValue({ ...createdServer, auth_type: authType });
await setupHttpServerForm();
await selectAntOption("Authentication", optionLabel);
const payload = await submitCreate();
expect(payload.dcr_bridge).toBe(false);
});
it("forces dcr_bridge: false when the auth type is switched away after toggling", async () => {
vi.mocked(networking.createMCPServer).mockResolvedValue({ ...createdServer, auth_type: "none" });
await setupHttpServerForm();
await selectAntOption("Authentication", "True Passthrough (no LiteLLM auth)");
await waitFor(() => {
expect(getDcrToggle()).toBeInTheDocument();
});
await act(async () => {
fireEvent.click(getDcrToggle()!);
});
await selectAntOption("Authentication", "None");
await waitFor(() => {
expect(getDcrToggle()).not.toBeInTheDocument();
});
const payload = await submitCreate();
expect(payload.dcr_bridge).toBe(false);
});
it("preserves the toggle value when switching between the two client-forwarded modes", async () => {
vi.mocked(networking.createMCPServer).mockResolvedValue({ ...createdServer, auth_type: "oauth_delegate" });
await setupHttpServerForm();
await selectAntOption("Authentication", "True Passthrough (no LiteLLM auth)");
await waitFor(() => {
expect(getDcrToggle()).toBeInTheDocument();
});
expect(getDcrToggle()).toHaveAttribute("aria-checked", "true");
// The Form.Item is mounted in both client-forwarded modes, so switching between them keeps the
// live toggle value rather than forcing it back to the default or to false.
await selectAntOption("Authentication", "OAuth Delegate (client-supplied upstream token)");
await waitFor(() => {
expect(getDcrToggle()).toBeInTheDocument();
});
expect(getDcrToggle()).toHaveAttribute("aria-checked", "true");
const payload = await submitCreate();
expect(payload.dcr_bridge).toBe(true);
});
});

139
ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx
View file

@ -18,6 +18,8 @@ import {
getOAuthAuthorizationIdentity,
CLEARED_ON_INVALIDATION,
isHeldOAuthTokenStale,
preservedDeclaredAppCredentials,
withoutMintedTokenCredentials,
} from "./types";
import OAuthFormFields from "./OAuthFormFields";
import TruePassthroughWarning from "./TruePassthroughWarning";
@ -60,6 +62,8 @@ const AUTH_TYPES_REQUIRING_CREDENTIALS = [
AUTH_TYPE.OAUTH2,
AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE,
AUTH_TYPE.AWS_SIGV4,
AUTH_TYPE.TRUE_PASSTHROUGH,
AUTH_TYPE.OAUTH_DELEGATE,
];
const CREATE_OAUTH_UI_STATE_KEY = "litellm-mcp-oauth-create-state";
@ -106,6 +110,14 @@ const CreateMCPServer: React.FC<CreateMCPServerProps> = ({
// was fetched; undefined when no valid token is held. If any mint-relevant field diverges from this,
// the held token is stale and is discarded so the admin must re-authorize.
const [authorizedIdentity, setAuthorizedIdentity] = useState<string | undefined>(undefined);
// The DCR-minted OAuth client from an interactive (oauth2) Authorize. Held OUT of form.credentials so
// it can never be collected as a client-forwarded server's declared app; injected into the payload
// only on an oauth2 submit (where persisting the registered client is correct), and cleared on any
// invalidation or modal close. An abandoned authorize leaves it null, which is the desired asymmetry.
const dcrClientRef = React.useRef<{ client_id: string; client_secret?: string } | null>(null);
// Set when the upstream identity (url/endpoints) changed while a declared app is present, so the
// section can warn that the saved app may not match the new upstream (the app is kept, not wiped).
const [appMayNotMatchUpstream, setAppMayNotMatchUpstream] = useState(false);
// Single hook call shared by MCPConnectionStatus and MCPToolConfiguration to avoid duplicate requests.
const {
@ -147,6 +159,9 @@ const CreateMCPServer: React.FC<CreateMCPServerProps> = ({
searchValue,
aliasManuallyEdited,
logoUrl,
// Persist the identity so invalidation stays armed across the OAuth redirect round trip: a
// post-restore url/mode edit must still discard the held token instead of silently keeping it.
authorizedIdentity,
};
setSecureItem(CREATE_OAUTH_UI_STATE_KEY, JSON.stringify(uiState));
} catch (err) {
@ -162,7 +177,12 @@ const CreateMCPServer: React.FC<CreateMCPServerProps> = ({
reset: resetOAuthFlow,
} = useMcpOAuthFlow({
accessToken,
getCredentials: () => form.getFieldValue("credentials"),
// Merge the ref-held DCR client so a re-authorize reuses the registered client instead of
// re-registering; the form store itself never holds the DCR client (see onTokenReceived).
getCredentials: () => ({
...((form.getFieldValue("credentials") as Record<string, unknown> | undefined) ?? {}),
...(dcrClientRef.current ?? {}),
}),
getTemporaryPayload: () => {
const values = form.getFieldsValue(true);
const transport = values.transport || transportType;
@ -184,7 +204,12 @@ const CreateMCPServer: React.FC<CreateMCPServerProps> = ({
url,
transport: transport === TRANSPORT.OPENAPI ? "http" : transport,
auth_type: isClientForwardedTokenMode(values.auth_type) ? values.auth_type : AUTH_TYPE.OAUTH2,
credentials: values.credentials,
// Mirror getCredentials: merge the ref-held DCR client for oauth2 so a re-authorize reuses the
// registered client (useMcpOAuthFlow keys reuse off credentials.client_id) instead of re-DCRing;
// the client-forwarded modes carry only the declared app.
credentials: isClientForwardedTokenMode(values.auth_type)
? preservedDeclaredAppCredentials(values.credentials)
: { ...((values.credentials as Record<string, unknown> | undefined) ?? {}), ...(dcrClientRef.current ?? {}) },
authorization_url: values.authorization_url,
token_url: values.token_url,
registration_url: values.registration_url,
@ -209,23 +234,36 @@ const CreateMCPServer: React.FC<CreateMCPServerProps> = ({
// edit form's onTokenReceived early return.
setAuthorizedIdentity(getOAuthAuthorizationIdentity(form.getFieldsValue(true)));
NotificationsManager.success(
"Token held for this browser session. Tools can now be previewed and configured; nothing will be saved to LiteLLM.",
"Token held for this browser session. Tools can now be previewed and configured; the token is not saved to LiteLLM.",
);
return;
}
const credentials = {
// The DCR-minted client is held in a ref, NOT written into form.credentials, so it can never be
// collected as a client-forwarded server's declared app; it is injected into the payload only on
// an oauth2 submit. An admin-typed client already lives in form.credentials and is left untouched.
dcrClientRef.current = registeredClient?.clientId
? {
client_id: registeredClient.clientId,
...(registeredClient.clientSecret && { client_secret: registeredClient.clientSecret }),
}
: null;
const current = (form.getFieldValue("credentials") as Record<string, unknown> | undefined) ?? {};
const nextCredentials = {
...(preservedDeclaredAppCredentials(current) ?? {}),
...(current.scopes !== undefined && { scopes: current.scopes }),
access_token: token.access_token,
...(token.refresh_token && { refresh_token: token.refresh_token }),
...(token.expires_in && { expires_in: token.expires_in }),
...(token.scope && { scope: token.scope }),
...(registeredClient?.clientId && { client_id: registeredClient.clientId }),
...(registeredClient?.clientSecret && { client_secret: registeredClient.clientSecret }),
};
form.setFieldsValue({ credentials });
// Capture the identity AFTER writing the DCR'd credentials so the held token is not spuriously
// invalidated by its own credential write.
// Path-replace (not deep-merge) so a re-authorize with fewer token fields does not leave stale
// siblings from the previous token behind; the admin-typed client keys and scopes are carried
// explicitly above.
form.setFieldValue("credentials", nextCredentials);
// Capture the identity AFTER writing the token so the held token is not spuriously invalidated by
// its own credential write.
setAuthorizedIdentity(getOAuthAuthorizationIdentity(form.getFieldsValue(true)));
NotificationsManager.success(
@ -246,7 +284,17 @@ const CreateMCPServer: React.FC<CreateMCPServerProps> = ({
clearTools();
resetOAuthFlow();
setAuthorizedIdentity(undefined);
dcrClientRef.current = null;
// Capture the admin-typed app before resetFields destroys it, then re-apply it: the app is
// upstream-scoped config, not minted material, so it survives every invalidation (the token is
// what gets discarded). Token-shaped keys are excluded by the helper's key filter.
const keptAppCredentials = preservedDeclaredAppCredentials(form.getFieldValue("credentials"));
form.resetFields([...CLEARED_ON_INVALIDATION]);
if (keptAppCredentials) {
form.setFieldsValue({ credentials: keptAppCredentials });
}
// Re-apply the in-flight edit last; rc-field-form deep-merges nested objects, so a changed
// credentials sub-field composes with the preserved sibling instead of replacing the object.
const preserved = Object.fromEntries(
CLEARED_ON_INVALIDATION.filter((key) => key in changedValues).map((key) => [key, changedValues[key]]),
);
@ -274,7 +322,18 @@ const CreateMCPServer: React.FC<CreateMCPServerProps> = ({
setTransportType(restoredTransport);
}
if (parsed.formValues) {
setPendingRestoredValues({ values: parsed.formValues, transport: restoredTransport });
// Assign the cleaned credentials (strip minted token material so a stale token never rehydrates);
// the declared app the admin typed is kept. Create has no server-side stored app to merge.
const restoredValues = {
...parsed.formValues,
credentials: withoutMintedTokenCredentials(parsed.formValues.credentials),
};
setPendingRestoredValues({ values: restoredValues, transport: restoredTransport });
}
if (typeof parsed.authorizedIdentity === "string") {
// Re-arm invalidation: without this the remounted form has authorizedIdentity=undefined, so a
// post-restore mode/url edit would never fire the stale-token discard.
setAuthorizedIdentity(parsed.authorizedIdentity);
}
if (parsed.costConfig) {
setCostConfig(parsed.costConfig);
@ -380,6 +439,7 @@ const CreateMCPServer: React.FC<CreateMCPServerProps> = ({
available_on_public_internet: availableOnPublicInternetRaw,
delegate_auth_to_upstream: delegateAuthToUpstreamRaw,
oauth_passthrough: oauthPassthroughRaw,
dcr_bridge: dcrBridgeRaw,
token_validation_json: rawTokenValidationJson,
...restValues
} = values;
@ -486,6 +546,11 @@ const CreateMCPServer: React.FC<CreateMCPServerProps> = ({
available_on_public_internet: Boolean(availableOnPublicInternetRaw),
delegate_auth_to_upstream: Boolean(delegateAuthToUpstreamRaw),
oauth_passthrough: Boolean(oauthPassthroughRaw),
// ``dcr_bridge`` is only meaningful for the client-forwarded token
// modes (true_passthrough / oauth_delegate) and defaults on when the
// toggle is shown; force false for any other auth type so a stale
// ``true`` is never persisted. Mirrors the sibling flags above.
dcr_bridge: isClientForwardedTokenMode(restValues.auth_type) ? Boolean(dcrBridgeRaw ?? true) : false,
...(restValues.auth_type === AUTH_TYPE.OAUTH2
? {
oauth2_flow:
@ -500,8 +565,20 @@ const CreateMCPServer: React.FC<CreateMCPServerProps> = ({
const includeCredentials =
restValues.auth_type && AUTH_TYPES_REQUIRING_CREDENTIALS.includes(restValues.auth_type);
if (includeCredentials && credentialsPayload && Object.keys(credentialsPayload).length > 0) {
payload.credentials = credentialsPayload;
// Client-forwarded rows persist ONLY the declared app; strip any token material that lingered in
// the form (e.g. from a prior oauth2 authorize on the same session) so it can never reach the row.
const submitCredentials = isClientForwardedTokenMode(restValues.auth_type)
? preservedDeclaredAppCredentials(credentialsPayload)
: credentialsPayload;
if (includeCredentials && submitCredentials && Object.keys(submitCredentials).length > 0) {
payload.credentials = submitCredentials;
}
// An interactive (oauth2) create persists its DCR-minted client from the ref (kept out of the
// form store); reuse a re-authorize's registered client instead of re-registering.
if (restValues.auth_type === AUTH_TYPE.OAUTH2 && dcrClientRef.current) {
payload.credentials = { ...(payload.credentials ?? {}), ...dcrClientRef.current };
}
if (accessToken != null) {
@ -576,6 +653,9 @@ const CreateMCPServer: React.FC<CreateMCPServerProps> = ({
setHasToolAllowlistInteraction(false);
setAliasManuallyEdited(false);
setLogoUrl(undefined);
setAuthorizedIdentity(undefined);
dcrClientRef.current = null;
setAppMayNotMatchUpstream(false);
setModalVisible(false);
};
@ -655,6 +735,8 @@ const CreateMCPServer: React.FC<CreateMCPServerProps> = ({
clearTools();
resetOAuthFlow();
setAuthorizedIdentity(undefined);
dcrClientRef.current = null;
setAppMayNotMatchUpstream(false);
}
}, [isModalVisible, form, clearTools, resetOAuthFlow]);
@ -663,12 +745,31 @@ const CreateMCPServer: React.FC<CreateMCPServerProps> = ({
const handleFormValuesChange = (changedValues: Record<string, unknown>, allValues: Record<string, unknown>) => {
// Any change to a mint-relevant field (url, auth_type, oauth_flow_type, client creds/scopes, or the
// authorization/token/registration endpoints — see getOAuthAuthorizationIdentity) makes a held token
// stale, so discard it and force a fresh authorize. When that happens, formValues must be rebuilt
// from the form's post-reset state, not the pre-reset allValues snapshot: the snapshot still holds
// the discarded token in credentials, and useTestMCPConnection reads formValues for tool preview.
if (isHeldOAuthTokenStale(allValues, authorizedIdentity)) {
// stale, so discard it and force a fresh authorize. The stale check reads getFieldsValue(true): the
// onValuesChange allValues argument holds only MOUNTED paths, so an unmounted identity field (e.g.
// an oauth_flow_type initialValue while in a client-forwarded mode) would compare as changed on
// every keystroke and churn the held token. When a clear happens, formValues is rebuilt from the
// form's post-reset state (not the pre-reset snapshot, which still holds the discarded token).
// Editing the client fields is the admin managing/acknowledging the app, so it always dismisses
// the "may not match upstream" warning regardless of the stale-token branch below.
// Editing the client fields is the admin managing/acknowledging the app, so it dismisses the "may
// not match upstream" warning. Otherwise a url/endpoint change while a declared app is present keeps
// the app but flags that it may not match the new upstream (the "keep + warn" behavior). This is
// independent of the held-token stale check below so it fires even without an authorize this session.
if ("credentials" in changedValues) {
setAppMayNotMatchUpstream(false);
} else {
const upstreamChanged = ["url", "spec_path", "authorization_url", "token_url", "registration_url"].some(
(key) => key in changedValues,
);
const hasDeclaredApp = preservedDeclaredAppCredentials(form.getFieldValue("credentials")) !== undefined;
if (upstreamChanged && hasDeclaredApp) {
setAppMayNotMatchUpstream(true);
}
}
if (isHeldOAuthTokenStale(form.getFieldsValue(true), authorizedIdentity)) {
clearHeldOAuthToken(changedValues);
setFormValues({ ...form.getFieldsValue(true), ...changedValues });
setFormValues(form.getFieldsValue(true));
return;
}
setFormValues(allValues);
@ -989,12 +1090,14 @@ const CreateMCPServer: React.FC<CreateMCPServerProps> = ({
<PassthroughAuthorizeSection
authType={authType}
dcrBridgeInitialChecked
oauthFlow={{
startOAuthFlow,
status: oauthStatus,
error: oauthError,
tokenResponse: oauthTokenResponse,
}}
appMayNotMatchUpstream={appMayNotMatchUpstream}
/>
{shouldShowAuthValueField && (

421
ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx
View file

@ -1,7 +1,9 @@
import React from "react";
import { describe, it, expect, vi, beforeEach } from "vitest";
import { render, screen, waitFor, fireEvent, act } from "@testing-library/react";
import MCPServerEdit from "./mcp_server_edit";
import userEvent from "@testing-library/user-event";
import MCPServerEdit, { EDIT_OAUTH_UI_STATE_KEY } from "./mcp_server_edit";
import { setSecureItem } from "@/utils/secureStorage";
import * as networking from "../networking";
import NotificationsManager from "../molecules/notifications_manager";
import { selectAntOption } from "./testUtils";
@ -1377,6 +1379,258 @@ describe("MCPServerEdit (OAuth token persistence on save)", () => {
},
);
it.each([["true_passthrough"], ["oauth_delegate"]])(
"persists admin-entered OAuth app credentials in the update payload for the %s mode",
async (authType) => {
mockOauth.tokenResponse = { access_token: "cf-tok", expires_in: 1800, token_type: "bearer" };
vi.mocked(networking.updateMCPServer).mockResolvedValue({
...interactiveOAuthServer,
auth_type: authType,
});
render(
<MCPServerEdit
mcpServer={{ ...interactiveOAuthServer, auth_type: authType }}
accessToken="access-token"
userID="user-1"
onCancel={vi.fn()}
onSuccess={vi.fn()}
availableAccessGroups={[]}
/>,
);
const user = userEvent.setup({ delay: null });
await user.type(
screen.getByPlaceholderText("Leave blank to keep the currently saved app (if any)"),
"org-app-client-id",
);
await user.type(
screen.getByPlaceholderText("Leave blank to keep the currently saved secret (if any)"),
"org-app-secret",
);
await act(async () => {
fireEvent.click(screen.getAllByRole("button", { name: "Save Changes" })[0]);
});
await waitFor(() => expect(networking.updateMCPServer).toHaveBeenCalledTimes(1));
const [, payload] = vi.mocked(networking.updateMCPServer).mock.calls[0];
// The declared app is config and persists onto the row; the browser-held token still never
// reaches the payload or the per-user credential store.
expect(payload.credentials).toMatchObject({
client_id: "org-app-client-id",
client_secret: "org-app-secret",
});
expect(JSON.stringify(payload)).not.toContain("cf-tok");
expect(networking.storeMCPOAuthUserCredential).not.toHaveBeenCalled();
},
);
it.each([["true_passthrough"], ["oauth_delegate"]])(
"preserves admin-entered app credentials when the URL changes after authorize for the %s mode",
async (authType) => {
vi.mocked(networking.updateMCPServer).mockResolvedValue({
...interactiveOAuthServer,
auth_type: authType,
});
render(
<MCPServerEdit
mcpServer={{ ...interactiveOAuthServer, auth_type: authType }}
accessToken="access-token"
userID="user-1"
onCancel={vi.fn()}
onSuccess={vi.fn()}
availableAccessGroups={[]}
/>,
);
const user = userEvent.setup({ delay: null });
await user.type(
screen.getByPlaceholderText("Leave blank to keep the currently saved app (if any)"),
"org-app-client-id",
);
await user.type(
screen.getByPlaceholderText("Leave blank to keep the currently saved secret (if any)"),
"org-app-secret",
);
act(() => {
mockOauth.onTokenReceived?.({ access_token: "cf-tok", token_type: "bearer" });
});
// The URL edit invalidates the held browser token (removeToken fires), but the declared app
// is config and must survive the invalidation into the update payload.
await act(async () => {
fireEvent.change(screen.getByPlaceholderText("https://your-mcp-server.com"), {
target: { value: "https://other.example.com/mcp" },
});
});
expect(mockRemoveToken).toHaveBeenCalledWith("oauth_server_1", "user-1");
await act(async () => {
fireEvent.click(screen.getAllByRole("button", { name: "Save Changes" })[0]);
});
await waitFor(() => expect(networking.updateMCPServer).toHaveBeenCalledTimes(1));
const [, payload] = vi.mocked(networking.updateMCPServer).mock.calls[0];
expect(payload.url).toBe("https://other.example.com/mcp");
expect(payload.credentials).toMatchObject({
client_id: "org-app-client-id",
client_secret: "org-app-secret",
});
expect(JSON.stringify(payload)).not.toContain("cf-tok");
},
);
it("sends an explicit-null credential write when removing the saved app for true_passthrough", async () => {
vi.mocked(networking.updateMCPServer).mockResolvedValue({
...interactiveOAuthServer,
auth_type: "true_passthrough",
});
render(
<MCPServerEdit
mcpServer={{ ...interactiveOAuthServer, auth_type: "true_passthrough" }}
accessToken="access-token"
userID="user-1"
onCancel={vi.fn()}
onSuccess={vi.fn()}
availableAccessGroups={[]}
/>,
);
// Blank fields keep the stored app (the backend merges partial credential updates), so the
// edit form states that convention and removal is an explicit checkbox that saves nulls.
expect(screen.getByPlaceholderText("Leave blank to keep the currently saved app (if any)")).toBeInTheDocument();
fireEvent.click(
screen.getByRole("checkbox", {
name: /Remove the saved OAuth app on save/,
}),
);
await act(async () => {
fireEvent.click(screen.getAllByRole("button", { name: "Save Changes" })[0]);
});
await waitFor(() => expect(networking.updateMCPServer).toHaveBeenCalledTimes(1));
const [, payload] = vi.mocked(networking.updateMCPServer).mock.calls[0];
expect(payload.credentials).toEqual({ client_id: null, client_secret: null });
});
it("warns that the saved app may not match after a URL change on a client-forwarded server", async () => {
render(
<MCPServerEdit
mcpServer={{
...interactiveOAuthServer,
auth_type: "true_passthrough",
credentials: { client_id: "stored-client" },
}}
accessToken="access-token"
userID="user-1"
onCancel={vi.fn()}
onSuccess={vi.fn()}
availableAccessGroups={[]}
/>,
);
// No warning until the upstream changes.
expect(screen.queryByText(/registered for the previous upstream/)).not.toBeInTheDocument();
await act(async () => {
fireEvent.change(screen.getByPlaceholderText("https://your-mcp-server.com"), {
target: { value: "https://different.example.com/mcp" },
});
});
// Keep + warn parity with the create form: the stored app is kept, and the banner appears.
expect(screen.getByText(/registered for the previous upstream/)).toBeInTheDocument();
});
it("preserves a stored client_id on OAuth-resume restore even when the saved snapshot is token-only", async () => {
// Post-redirect restore: the sessionStorage snapshot carries only a minted token (no client keys),
// while the loaded server has a stored client_id. The restore must merge the server's declared app
// under the snapshot before stripping tokens, so the stored client_id is never cleared to blank.
setSecureItem(
EDIT_OAUTH_UI_STATE_KEY,
JSON.stringify({
serverId: "oauth_server_1",
formValues: { auth_type: "true_passthrough", credentials: { access_token: "leftover-token" } },
}),
);
render(
<MCPServerEdit
mcpServer={{
...interactiveOAuthServer,
auth_type: "true_passthrough",
credentials: { client_id: "stored-client" },
}}
accessToken="access-token"
userID="user-1"
onCancel={vi.fn()}
onSuccess={vi.fn()}
availableAccessGroups={[]}
/>,
);
const clientIdField = await screen.findByPlaceholderText("Leave blank to keep the currently saved app (if any)");
await waitFor(() => expect((clientIdField as HTMLInputElement).value).toBe("stored-client"));
// The leftover minted token must not have rehydrated anywhere.
expect(document.body.innerHTML).not.toContain("leftover-token");
});
it("resets the remove-app checkbox on a server switch so it never deletes the next server's stored app", async () => {
vi.mocked(networking.updateMCPServer).mockResolvedValue({
...interactiveOAuthServer,
auth_type: "true_passthrough",
});
const { rerender } = render(
<MCPServerEdit
mcpServer={{ ...interactiveOAuthServer, server_id: "server-A", auth_type: "true_passthrough" }}
accessToken="access-token"
userID="user-1"
onCancel={vi.fn()}
onSuccess={vi.fn()}
availableAccessGroups={[]}
/>,
);
// Check "remove saved app" on server A.
fireEvent.click(screen.getByRole("checkbox", { name: /Remove the saved OAuth app on save/ }));
expect(
(screen.getByRole("checkbox", { name: /Remove the saved OAuth app on save/ }) as HTMLInputElement).checked,
).toBe(true);
// Switch the panel to server B without unmounting.
rerender(
<MCPServerEdit
mcpServer={{ ...interactiveOAuthServer, server_id: "server-B", auth_type: "true_passthrough" }}
accessToken="access-token"
userID="user-1"
onCancel={vi.fn()}
onSuccess={vi.fn()}
availableAccessGroups={[]}
/>,
);
// The checkbox must have reset, so saving server B does not send the explicit-null delete write.
expect(
(screen.getByRole("checkbox", { name: /Remove the saved OAuth app on save/ }) as HTMLInputElement).checked,
).toBe(false);
await act(async () => {
fireEvent.click(screen.getAllByRole("button", { name: "Save Changes" })[0]);
});
await waitFor(() => expect(networking.updateMCPServer).toHaveBeenCalledTimes(1));
const [, payload] = vi.mocked(networking.updateMCPServer).mock.calls[0];
expect(payload.credentials).not.toEqual({ client_id: null, client_secret: null });
});
it("forwards a newly authorized browser-held token for tool loading before the form is saved", async () => {
// Regression: fetchTools keyed the browser-held decision off the saved mcpServer.auth_type, so
// after switching the form to true_passthrough and authorizing, the fresh token was not sent as
@ -1737,3 +1991,168 @@ describe("MCPServerEdit (max concurrent requests)", () => {
expect(payload.max_concurrent_requests).toBeNull();
});
});
describe("MCPServerEdit (dcr_bridge toggle)", () => {
beforeEach(() => {
vi.clearAllMocks();
mockOauth.tokenResponse = null;
});
const getDcrToggle = () => document.getElementById("dcr_bridge");
function renderEdit(server: Record<string, unknown>) {
render(
<MCPServerEdit
mcpServer={{ ...interactiveOAuthServer, ...server }}
accessToken="access-token"
onCancel={vi.fn()}
onSuccess={vi.fn()}
availableAccessGroups={[]}
/>,
);
}
async function saveAndGetPayload() {
const saveButtons = screen.getAllByRole("button", { name: "Save Changes" });
await act(async () => {
fireEvent.click(saveButtons[0]);
});
await waitFor(() => {
expect(networking.updateMCPServer).toHaveBeenCalledTimes(1);
});
const [, payload] = vi.mocked(networking.updateMCPServer).mock.calls[0];
return payload;
}
it.each([["true_passthrough"], ["oauth_delegate"]])("renders the toggle for a %s server", async (authType) => {
renderEdit({ auth_type: authType });
await waitFor(() => {
expect(getDcrToggle()).toBeInTheDocument();
});
expect(screen.getByText("Gateway-hosted sign-in (DCR bridge)")).toBeInTheDocument();
});
it.each([["oauth2"], ["api_key"], ["none"]])("does not render the toggle for an %s server", async (authType) => {
renderEdit({ auth_type: authType });
await waitFor(() => {
expect(screen.getAllByRole("button", { name: "Save Changes" }).length).toBeGreaterThan(0);
});
expect(screen.queryByText("Gateway-hosted sign-in (DCR bridge)")).not.toBeInTheDocument();
expect(getDcrToggle()).not.toBeInTheDocument();
});
it("renders the toggle between the OAuth client fields and the Authorize button", async () => {
renderEdit({ auth_type: "true_passthrough" });
await waitFor(() => {
expect(getDcrToggle()).toBeInTheDocument();
});
const toggle = getDcrToggle() as HTMLElement;
const secretInput = screen.getByPlaceholderText("Leave blank to keep the currently saved secret (if any)");
const authorizeButton = screen.getByRole("button", { name: "Authorize & Fetch Tools (browser-only)" });
expect(secretInput.compareDocumentPosition(toggle) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy();
expect(toggle.compareDocumentPosition(authorizeButton) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy();
});
it("initializes unchecked from a null stored value and saves an explicit false", async () => {
vi.mocked(networking.updateMCPServer).mockResolvedValue({
...interactiveOAuthServer,
auth_type: "true_passthrough",
});
renderEdit({ auth_type: "true_passthrough", dcr_bridge: null });
await waitFor(() => {
expect(getDcrToggle()).toBeInTheDocument();
});
expect(getDcrToggle()).toHaveAttribute("aria-checked", "false");
const payload = await saveAndGetPayload();
expect(payload.dcr_bridge).toBe(false);
});
it("initializes checked from a stored true and saves an explicit true", async () => {
vi.mocked(networking.updateMCPServer).mockResolvedValue({
...interactiveOAuthServer,
auth_type: "oauth_delegate",
dcr_bridge: true,
});
renderEdit({ auth_type: "oauth_delegate", dcr_bridge: true });
await waitFor(() => {
expect(getDcrToggle()).toBeInTheDocument();
});
expect(getDcrToggle()).toHaveAttribute("aria-checked", "true");
const payload = await saveAndGetPayload();
expect(payload.dcr_bridge).toBe(true);
});
it("saves an explicit false after the admin unchecks a stored true", async () => {
vi.mocked(networking.updateMCPServer).mockResolvedValue({
...interactiveOAuthServer,
auth_type: "true_passthrough",
dcr_bridge: false,
});
renderEdit({ auth_type: "true_passthrough", dcr_bridge: true });
await waitFor(() => {
expect(getDcrToggle()).toBeInTheDocument();
});
await act(async () => {
fireEvent.click(getDcrToggle()!);
});
expect(getDcrToggle()).toHaveAttribute("aria-checked", "false");
const payload = await saveAndGetPayload();
expect(payload.dcr_bridge).toBe(false);
});
it("forces dcr_bridge: false when the auth type is switched away", async () => {
vi.mocked(networking.updateMCPServer).mockResolvedValue({
...interactiveOAuthServer,
auth_type: "api_key",
});
renderEdit({ auth_type: "true_passthrough", dcr_bridge: true });
await waitFor(() => {
expect(getDcrToggle()).toBeInTheDocument();
});
await selectAntOption("Authentication", "API Key");
await waitFor(() => {
expect(getDcrToggle()).not.toBeInTheDocument();
});
// Mirrors the sibling delegate_auth_to_upstream / oauth_passthrough force-false: a stale true is
// never left behind to silently re-activate if the mode is switched back.
const payload = await saveAndGetPayload();
expect(payload.dcr_bridge).toBe(false);
});
it("preserves the toggle value when switching between the two client-forwarded modes", async () => {
vi.mocked(networking.updateMCPServer).mockResolvedValue({
...interactiveOAuthServer,
auth_type: "oauth_delegate",
dcr_bridge: true,
});
renderEdit({ auth_type: "true_passthrough", dcr_bridge: true });
await waitFor(() => {
expect(getDcrToggle()).toBeInTheDocument();
});
expect(getDcrToggle()).toHaveAttribute("aria-checked", "true");
// The Form.Item stays mounted across the two client-forwarded modes, so the live toggle value is
// preserved rather than forced false by the switch.
await selectAntOption("Authentication", "OAuth Delegate (client-supplied upstream token)");
await waitFor(() => {
expect(getDcrToggle()).toBeInTheDocument();
});
expect(getDcrToggle()).toHaveAttribute("aria-checked", "true");
const payload = await saveAndGetPayload();
expect(payload.dcr_bridge).toBe(true);
});
});

103
ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx
View file

@ -8,6 +8,8 @@ import {
getOAuthAuthorizationIdentity,
CLEARED_ON_INVALIDATION,
isHeldOAuthTokenStale,
preservedDeclaredAppCredentials,
withoutMintedTokenCredentials,
OAUTH_FLOW,
MCP_OAUTH2_FLOW_M2M,
MCP_OAUTH2_FLOW_INTERACTIVE,
@ -56,6 +58,8 @@ const AUTH_TYPES_REQUIRING_CREDENTIALS = [
AUTH_TYPE.OAUTH2,
AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE,
AUTH_TYPE.AWS_SIGV4,
AUTH_TYPE.TRUE_PASSTHROUGH,
AUTH_TYPE.OAUTH_DELEGATE,
];
export const EDIT_OAUTH_UI_STATE_KEY = "litellm-mcp-oauth-edit-state";
@ -74,6 +78,10 @@ const MCPServerEdit: React.FC<MCPServerEditProps> = ({
const [toolsError, setToolsError] = useState<string | null>(null);
const [searchValue, setSearchValue] = useState<string>("");
const [aliasManuallyEdited, setAliasManuallyEdited] = useState(false);
const [removeStoredApp, setRemoveStoredApp] = useState(false);
// Set when the upstream identity (url/endpoints) changed while a declared app is present, so the
// section warns that the saved app may not match the new upstream (the app is kept, not wiped).
const [appMayNotMatchUpstream, setAppMayNotMatchUpstream] = useState(false);
const [allowedTools, setAllowedTools] = useState<string[]>([]);
const [hasToolAllowlistInteraction, setHasToolAllowlistInteraction] = useState(false);
const [toolNameToDisplayName, setToolNameToDisplayName] = useState<Record<string, string>>({});
@ -179,7 +187,9 @@ const MCPServerEdit: React.FC<MCPServerEditProps> = ({
url,
transport,
auth_type: isClientForwardedTokenMode(values.auth_type) ? values.auth_type : AUTH_TYPE.OAUTH2,
credentials: values.credentials,
credentials: isClientForwardedTokenMode(values.auth_type)
? preservedDeclaredAppCredentials(values.credentials)
: values.credentials,
mcp_access_groups: values.mcp_access_groups || mcpServer.mcp_access_groups,
static_headers: staticHeaders,
command: values.command,
@ -202,19 +212,23 @@ const MCPServerEdit: React.FC<MCPServerEditProps> = ({
};
setToken(mcpServer.server_id, browserHeldToken, userID);
NotificationsManager.success(
"Token held for this browser session. Tools can now be loaded and configured; nothing was saved to LiteLLM.",
"Token held for this browser session. Tools can now be loaded and configured; the token is not saved to LiteLLM.",
);
return;
}
const credentials = {
const current = (form.getFieldValue("credentials") as Record<string, unknown> | undefined) ?? {};
const nextCredentials = {
...(preservedDeclaredAppCredentials(current) ?? {}),
...(current.scopes !== undefined && { scopes: current.scopes }),
access_token: token.access_token,
...(token.refresh_token && { refresh_token: token.refresh_token }),
...(token.expires_in && { expires_in: token.expires_in }),
...(token.scope && { scope: token.scope }),
};
form.setFieldsValue({ credentials });
// Path-replace (not deep-merge) so a re-authorize with fewer token fields does not leave stale
// siblings behind; the admin-typed client keys and scopes are carried explicitly above.
form.setFieldValue("credentials", nextCredentials);
// Re-capture after writing credentials so the token is not invalidated by its own credential write.
authorizedIdentityRef.current = getOAuthAuthorizationIdentity(form.getFieldsValue(true));
@ -276,6 +290,7 @@ const MCPServerEdit: React.FC<MCPServerEditProps> = ({
env_vars: initialEnvVars,
extra_headers: mcpServer.extra_headers || [],
oauth_flow_type: oauth2FlowToFormValue(mcpServer.oauth2_flow),
dcr_bridge: Boolean(mcpServer.dcr_bridge),
token_validation_json: mcpServer.token_validation
? JSON.stringify(mcpServer.token_validation, null, 2)
: undefined,
@ -295,6 +310,11 @@ const MCPServerEdit: React.FC<MCPServerEditProps> = ({
}
syncedServerIdRef.current = mcpServer.server_id;
form.setFieldsValue(initialValues);
// Reset per-server OAuth UI state so it never carries across a server switch without an unmount: a
// stale removeStoredApp would send an explicit-null credential write that deletes the new server's
// stored app, and a stale warning would show on a server whose upstream did not change.
setAppMayNotMatchUpstream(false);
setRemoveStoredApp(false);
}, [mcpServer.server_id, initialValues, form]);
// Initialize cost config from existing server data
@ -332,8 +352,24 @@ const MCPServerEdit: React.FC<MCPServerEditProps> = ({
return;
}
if (parsed.formValues) {
setPendingRestoredValues({ ...mcpServer, ...parsed.formValues });
// Rebuild credentials from the declared app in EITHER the loaded server or the saved snapshot,
// then strip minted token material. Merging the two (server under snapshot) before stripping is
// what guarantees a token-only snapshot never clears a stored client_id/client_secret: the
// server's declared app survives and only the token keys drop. Assigning the cleaned result (not
// spreading the raw snapshot) also ensures a stale token can never rehydrate into the form.
const restoredCredentials = withoutMintedTokenCredentials({
...(mcpServer.credentials ?? {}),
...((parsed.formValues.credentials as Record<string, unknown> | undefined) ?? {}),
});
const restoredValues = {
...mcpServer,
...parsed.formValues,
credentials: restoredCredentials,
};
setPendingRestoredValues(restoredValues);
}
// The ref is re-armed by onTokenReceived when the redirect completes the code exchange, so there
// is no separate restore-side re-arm here (writing a ref inside an effect is disallowed).
if (parsed.costConfig) {
setCostConfig(parsed.costConfig);
}
@ -407,7 +443,13 @@ const MCPServerEdit: React.FC<MCPServerEditProps> = ({
}
setTools([]);
resetOAuthFlow();
// The admin-typed app is upstream-scoped config, not minted material, so it survives every
// invalidation; only the held token is discarded. Token-shaped keys are excluded by the filter.
const keptAppCredentials = preservedDeclaredAppCredentials(form.getFieldValue("credentials"));
form.resetFields([...CLEARED_ON_INVALIDATION]);
if (keptAppCredentials) {
form.setFieldsValue({ credentials: keptAppCredentials });
}
const preserved = Object.fromEntries(
CLEARED_ON_INVALIDATION.filter((key) => key in changedValues).map((key) => [key, changedValues[key]]),
);
@ -417,6 +459,21 @@ const MCPServerEdit: React.FC<MCPServerEditProps> = ({
};
const handleFormValuesChange = (changedValues: Record<string, unknown>) => {
// Editing the client fields dismisses the "may not match upstream" warning; otherwise a url/endpoint
// change while a declared app is present keeps the app but flags that it may not match the new
// upstream (the "keep + warn" behavior). Mirrors the create form; independent of the held-token
// stale check so it fires even without an authorize this session (the stored app is for the old url).
if ("credentials" in changedValues) {
setAppMayNotMatchUpstream(false);
} else {
const upstreamChanged = ["url", "spec_path", "authorization_url", "token_url", "registration_url"].some(
(key) => key in changedValues,
);
const hasDeclaredApp = preservedDeclaredAppCredentials(form.getFieldValue("credentials")) !== undefined;
if (upstreamChanged && hasDeclaredApp) {
setAppMayNotMatchUpstream(true);
}
}
if (isHeldOAuthTokenStale(form.getFieldsValue(true), authorizedIdentityRef.current)) {
clearHeldOAuthToken(changedValues);
}
@ -627,6 +684,7 @@ const MCPServerEdit: React.FC<MCPServerEditProps> = ({
available_on_public_internet: availableOnPublicInternetRaw,
delegate_auth_to_upstream: delegateAuthToUpstreamRaw,
oauth_passthrough: oauthPassthroughRaw,
dcr_bridge: dcrBridgeRaw,
token_validation_json: rawTokenValidationJson,
...restValues
} = values;
@ -837,6 +895,15 @@ const MCPServerEdit: React.FC<MCPServerEditProps> = ({
? Boolean(oauthPassthroughRaw ?? mcpServer.oauth_passthrough)
: false;
})(),
// ``dcr_bridge`` is only meaningful for the client-forwarded token
// modes (true_passthrough / oauth_delegate). The Form.Item is
// conditionally rendered so the value drops out of the form on
// auth_type change; force false for any other configuration to avoid
// persisting a stale ``true`` that would silently re-activate if the
// mode is later switched back.
dcr_bridge: isClientForwardedTokenMode(restValues.auth_type)
? Boolean(dcrBridgeRaw ?? mcpServer.dcr_bridge)
: false,
...(restValues.auth_type === AUTH_TYPE.OAUTH2 && restValues.oauth_flow_type
? {
oauth2_flow:
@ -850,8 +917,22 @@ const MCPServerEdit: React.FC<MCPServerEditProps> = ({
const includeCredentials =
restValues.auth_type && AUTH_TYPES_REQUIRING_CREDENTIALS.includes(restValues.auth_type);
if (includeCredentials && credentialsPayload && Object.keys(credentialsPayload).length > 0) {
payload.credentials = credentialsPayload;
// Client-forwarded rows persist ONLY the declared app; strip any token material lingering in the
// form (e.g. from a prior oauth2 authorize this session) so it can never reach the row.
const submitCredentials = isClientForwardedTokenMode(restValues.auth_type)
? preservedDeclaredAppCredentials(credentialsPayload)
: credentialsPayload;
if (includeCredentials && submitCredentials && Object.keys(submitCredentials).length > 0) {
payload.credentials = submitCredentials;
}
// Explicit removal of a saved app for the client-forwarded modes, applied AFTER the filter so it
// always wins. Blank fields are the keep-existing convention (the backend merges partial
// credential updates), so removal must be an explicit-null write: encrypt skips nulls and the
// merge overrides the stored keys, returning the server to dynamic client registration.
if (removeStoredApp && isClientForwardedTokenMode(restValues.auth_type)) {
payload.credentials = { client_id: null, client_secret: null };
}
const updated = await updateMCPServer(accessToken, payload);
@ -895,6 +976,7 @@ const MCPServerEdit: React.FC<MCPServerEditProps> = ({
}
NotificationsManager.success("MCP Server updated successfully");
setAppMayNotMatchUpstream(false);
onSuccess(updated);
} catch (error: any) {
NotificationsManager.fromBackend("Failed to update MCP Server" + (error?.message ? `: ${error.message}` : ""));
@ -1040,6 +1122,11 @@ const MCPServerEdit: React.FC<MCPServerEditProps> = ({
error: oauthError,
tokenResponse: oauthTokenResponse,
}}
isEditing
savedAuthType={mcpServer.auth_type}
removeStoredApp={removeStoredApp}
onRemoveStoredAppChange={setRemoveStoredApp}
appMayNotMatchUpstream={appMayNotMatchUpstream}
/>
</>
)}

51
ui/litellm-dashboard/src/components/mcp_tools/types.test.tsx
View file

@ -10,6 +10,9 @@ import {
getOAuthAuthorizationIdentity,
isHeldOAuthTokenStale,
oauth2FlowToFormValue,
preservedDeclaredAppCredentials,
withoutMintedTokenCredentials,
credentialAuthClass,
} from "./types";
describe("getOAuthAuthorizationIdentity", () => {
@ -180,3 +183,51 @@ describe("oauth2FlowToFormValue", () => {
expect(oauth2FlowToFormValue(undefined)).toBeUndefined();
});
});
describe("preservedDeclaredAppCredentials", () => {
it("keeps only non-empty string declared-app keys and never token-shaped keys", () => {
expect(preservedDeclaredAppCredentials(undefined)).toBeUndefined();
expect(preservedDeclaredAppCredentials({})).toBeUndefined();
expect(preservedDeclaredAppCredentials({ client_id: 123 })).toBeUndefined();
expect(preservedDeclaredAppCredentials({ client_id: "" })).toBeUndefined();
expect(preservedDeclaredAppCredentials({ client_id: "a", access_token: "t", scopes: ["s"] })).toEqual({
client_id: "a",
});
expect(preservedDeclaredAppCredentials({ client_secret: "s" })).toEqual({ client_secret: "s" });
expect(preservedDeclaredAppCredentials({ client_id: "a", client_secret: "b", refresh_token: "r" })).toEqual({
client_id: "a",
client_secret: "b",
});
});
});
describe("withoutMintedTokenCredentials", () => {
it("drops token keys and keeps the declared app and other config", () => {
expect(withoutMintedTokenCredentials(undefined)).toBeUndefined();
const mixed = {
client_id: "a",
client_secret: "b",
access_token: "t",
refresh_token: "r",
expires_in: 3600,
scope: "read",
scopes: ["read"],
};
expect(withoutMintedTokenCredentials(mixed)).toEqual({ client_id: "a", client_secret: "b", scopes: ["read"] });
});
it("returns undefined (not {}) when only minted keys are present, so a restore never blanks the fields", () => {
expect(withoutMintedTokenCredentials({ access_token: "t", refresh_token: "r", expires_in: 3600 })).toBeUndefined();
// A declared client is always kept, so a stored client_id can never be overwritten with empty.
expect(withoutMintedTokenCredentials({ client_id: "x", access_token: "t" })).toEqual({ client_id: "x" });
});
});
describe("credentialAuthClass", () => {
it("collapses the client-forwarded modes to one class and leaves others distinct", () => {
expect(credentialAuthClass(AUTH_TYPE.TRUE_PASSTHROUGH)).toBe("client_forwarded");
expect(credentialAuthClass(AUTH_TYPE.OAUTH_DELEGATE)).toBe("client_forwarded");
expect(credentialAuthClass(AUTH_TYPE.OAUTH2)).toBe(AUTH_TYPE.OAUTH2);
expect(credentialAuthClass(null)).toBeNull();
});
});

51
ui/litellm-dashboard/src/components/mcp_tools/types.tsx
View file

@ -96,6 +96,54 @@ export const getOAuthAuthorizationIdentity = (values: Record<string, unknown>):
// edit forms so what gets wiped cannot drift.
export const CLEARED_ON_INVALIDATION = ["credentials"] as const;
// The declared-app filter over form.credentials. It is a pure key filter with no mode/transition
// guard because the surrounding code establishes that a client_id/client_secret in form.credentials
// is ALWAYS admin-typed in every reachable state: the create form holds the DCR-minted client in a
// ref and never writes it into the form store, the edit form's onTokenReceived never writes client
// keys, and the invalidation reset clears the whole object atomically. So preserving the string
// client keys across any invalidation (URL/endpoint edit, true_passthrough<->oauth_delegate switch,
// or a round trip through another mode) is always legitimate, while the output key filter excludes
// token-shaped keys so a preserve can never carry minted material through. Shared by both forms.
const DECLARED_APP_CREDENTIAL_KEYS = ["client_id", "client_secret"] as const;
// Minted token material the oauth2 authorize path writes beside the app keys; stripped from restored
// snapshots and from any credentials that transit to the temp-session preview so a stale token never
// reaches the backend or a client-forwarded server row.
export const MINTED_TOKEN_CREDENTIAL_KEYS = ["access_token", "refresh_token", "expires_in", "scope"] as const;
export const preservedDeclaredAppCredentials = (
credentials: Record<string, unknown> | null | undefined,
): Record<string, string> | undefined => {
if (!credentials) return undefined;
const kept = Object.fromEntries(
DECLARED_APP_CREDENTIAL_KEYS.filter((key) => typeof credentials[key] === "string" && credentials[key] !== "").map(
(key) => [key, credentials[key] as string],
),
);
return Object.keys(kept).length > 0 ? kept : undefined;
};
// Drop minted token keys, keeping everything else (the declared app plus any non-token config).
export const withoutMintedTokenCredentials = (
credentials: Record<string, unknown> | null | undefined,
): Record<string, unknown> | undefined => {
if (!credentials) return undefined;
const kept = Object.fromEntries(
Object.entries(credentials).filter(([key]) => !(MINTED_TOKEN_CREDENTIAL_KEYS as readonly string[]).includes(key)),
);
// Return undefined (not {}) when only minted keys were present, so a restore spreads `credentials:
// undefined` (the fields keep their placeholder / keep-existing state) rather than blanking them.
return Object.keys(kept).length > 0 ? kept : undefined;
};
// The client-forwarded modes share one credential class (same declared app, same authorize relay), so
// a switch between them must NOT be treated as an app change. Mirrors the backend _credential_auth_class
// in db.py; kept in sync so the UI's keep-existing copy and the backend's merge cannot disagree.
export const credentialAuthClass = (authType: string | null | undefined): string | null => {
if (authType === AUTH_TYPE.TRUE_PASSTHROUGH || authType === AUTH_TYPE.OAUTH_DELEGATE) return "client_forwarded";
return authType ?? null;
};
// True when a token was authorized in this session (authorizedIdentity recorded at mint time) and the
// form's current identity no longer matches it. Every invalidation decision in both forms goes through
// this single check: onValuesChange for user edits, and an explicit recheck after any programmatic
@ -319,7 +367,10 @@ export interface MCPServer {
available_on_public_internet?: boolean;
delegate_auth_to_upstream?: boolean;
oauth_passthrough?: boolean;
dcr_bridge?: boolean | null;
max_concurrent_requests?: number | null;
/** Redacted to null in server responses; present when constructing a server locally. */
credentials?: Record<string, unknown> | null;
/** Stdio-only fields (present when transport === 'stdio') */
command?: string | null;

14
ui/litellm-dashboard/src/components/networking.tsx
View file

@ -22,7 +22,8 @@ export const getCallbackConfigsCall = async (accessToken: string) => {
* Helper file for calls being made to proxy
*/
import MessageManager from "@/components/molecules/message_manager";
import { clearTokenCookies, storeLoginToken } from "@/utils/cookieUtils";
import { clearTokenCookies, getCookie, storeLoginToken } from "@/utils/cookieUtils";
import { decodeToken } from "@/utils/jwtUtils";
import { TagNewRequest, TagUpdateRequest, TagListResponse, TagInfoResponse } from "./tag_management/types";
import { Team } from "./key_team_helpers/key_list";
import { EmailEventSettingsResponse, EmailEventSettingsUpdateRequest } from "./email_events/types";
@ -38,6 +39,12 @@ import type {
import { MCP_TOOLS_PREVIEW_FORBIDDEN_MESSAGE } from "./mcp_tools/constants";
import { createApiClient, deriveErrorMessage } from "@/lib/http/client";
import { resolveApiBase } from "@/lib/http/resolveApiBase";
import {
registerAuthHeaderNameGetter,
registerAuthTokenGetter,
registerBaseUrlGetter,
registerErrorHandler,
} from "@/lib/http/runtime";
import { serverRootPath, setServerRootPath } from "@/lib/serverRootPath";
export { serverRootPath };
@ -371,6 +378,11 @@ const apiClient = createApiClient({
onError: handleError,
});
registerBaseUrlGetter(getProxyBaseUrl);
registerAuthHeaderNameGetter(getGlobalLitellmHeaderName);
registerAuthTokenGetter(() => decodeToken(getCookie("token"))?.key ?? null);
registerErrorHandler(handleError);
export const makeModelGroupPublic = async (accessToken: string, modelGroups: string[]) => {
const url = proxyBaseUrl ? `${proxyBaseUrl}/model_group/make_public` : `/model_group/make_public`;
const response = await fetch(url, {

118
ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx
View file

@ -29,6 +29,16 @@ const nameCellColumns: ColumnDef<Person, unknown>[] = [
},
];
const filterableColumns: ColumnDef<Person, unknown>[] = [
{
accessorKey: "name",
header: "Name",
meta: { title: "Name" },
filterFn: (row, columnId, value) => row.getValue<string>(columnId) === value,
cell: ({ row }) => <span data-testid="name-cell">{row.original.name}</span>,
},
];
const headerCycleColumns: ColumnDef<Person, unknown>[] = [
{
accessorKey: "name",
@ -195,10 +205,110 @@ describe("DataTable pagination", () => {
});
});
describe("DataTable filtering", () => {
it("client mode filters rows by columnFilters", () => {
const { rerender } = render(
<DataTable
data={CHARLIE_ALICE_BOB}
columns={filterableColumns}
filterMode="client"
columnFilters={[]}
onColumnFiltersChange={vi.fn()}
/>,
);
expect(names()).toEqual(["Charlie", "Alice", "Bob"]);
rerender(
<DataTable
data={CHARLIE_ALICE_BOB}
columns={filterableColumns}
filterMode="client"
columnFilters={[{ id: "name", value: "Alice" }]}
onColumnFiltersChange={vi.fn()}
/>,
);
expect(names()).toEqual(["Alice"]);
});
it("client global filter matches substrings across columns", () => {
const { rerender } = render(
<DataTable
data={CHARLIE_ALICE_BOB}
columns={nameEmailColumns}
filterMode="client"
globalFilter=""
onGlobalFilterChange={vi.fn()}
/>,
);
expect(names()).toEqual(["Charlie", "Alice", "Bob"]);
rerender(
<DataTable
data={CHARLIE_ALICE_BOB}
columns={nameEmailColumns}
filterMode="client"
globalFilter="ali"
onGlobalFilterChange={vi.fn()}
/>,
);
expect(names()).toEqual(["Alice"]);
});
it("server mode never filters locally even when columnFilters is set", () => {
render(
<DataTable
data={CHARLIE_ALICE_BOB}
columns={filterableColumns}
filterMode="server"
columnFilters={[{ id: "name", value: "Alice" }]}
onColumnFiltersChange={vi.fn()}
/>,
);
expect(names()).toEqual(["Charlie", "Alice", "Bob"]);
});
it("throws when server filtering is missing required props", () => {
const spy = vi.spyOn(console, "error").mockImplementation(() => {});
expect(() => render(<DataTable data={[]} columns={filterableColumns} filterMode="server" />)).toThrow(
/filterMode='server'/,
);
spy.mockRestore();
});
});
describe("DataTable loading", () => {
it("renders skeleton rows while loading and real rows once loaded", () => {
const { rerender } = render(<DataTable data={CHARLIE_ALICE_BOB} columns={nameCellColumns} isLoading />);
expect(screen.getAllByTestId("skeleton-row").length).toBeGreaterThan(0);
expect(screen.queryByTestId("name-cell")).toBeNull();
rerender(<DataTable data={CHARLIE_ALICE_BOB} columns={nameCellColumns} />);
expect(screen.queryAllByTestId("skeleton-row")).toHaveLength(0);
expect(names()).toEqual(["Charlie", "Alice", "Bob"]);
});
it("varies skeleton shape and width per column instead of one fixed bar", () => {
const columns: ColumnDef<Person, unknown>[] = [
{ accessorKey: "name", header: "Name", meta: { skeleton: "twoLine" }, cell: () => null },
{ accessorKey: "email", header: "Email", cell: () => null },
];
render(<DataTable data={CHARLIE_ALICE_BOB} columns={columns} isLoading />);
const firstRow = screen.getAllByTestId("skeleton-row").at(0);
expect(firstRow).toBeDefined();
const bars = Array.from(firstRow?.querySelectorAll('[data-slot="skeleton"]') ?? []);
// twoLine column contributes a main + sub bar (2); the text column contributes 1
expect(bars).toHaveLength(3);
// per-column widths differ instead of every cell sharing one fixed width
expect(new Set(bars.map((bar) => bar.className)).size).toBeGreaterThan(1);
});
});
describe("DataTable column visibility", () => {
it("hides a column when toggled off in the view-options menu", async () => {
const user = userEvent.setup();
render(
const { container } = render(
<DataTable
data={CHARLIE_ALICE_BOB}
columns={nameEmailColumns}
@ -206,13 +316,13 @@ describe("DataTable column visibility", () => {
/>,
);
expect(screen.getByText("Email")).toBeInTheDocument();
expect(container.querySelector('th[data-header-id="email"]')).not.toBeNull();
await user.click(screen.getByTestId("view-options-trigger"));
await user.click(await screen.findByTestId("view-option-email"));
await waitFor(() => expect(screen.queryByText("Email")).not.toBeInTheDocument());
await waitFor(() => expect(container.querySelector('th[data-header-id="email"]')).toBeNull());
await user.click(screen.getByTestId("view-option-email"));
await waitFor(() => expect(screen.getByText("Email")).toBeInTheDocument());
await waitFor(() => expect(container.querySelector('th[data-header-id="email"]')).not.toBeNull());
});
it("omits columns that opt out of hiding from the menu", async () => {

162
ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx
View file

@ -4,12 +4,14 @@ import {
type Cell,
type Column,
type ColumnDef,
type ColumnFiltersState,
type ColumnPinningState,
type ColumnSizingState,
type ExpandedState,
flexRender,
getCoreRowModel,
getExpandedRowModel,
getFilteredRowModel,
getPaginationRowModel,
getSortedRowModel,
type Header,
@ -21,9 +23,11 @@ import {
useReactTable,
type VisibilityState,
} from "@tanstack/react-table";
import { SearchX } from "lucide-react";
import * as React from "react";
import { Fragment, useState } from "react";
import { Skeleton } from "@/components/ui/skeleton";
import {
Table as TableRoot,
TableBody,
@ -37,7 +41,7 @@ import { cn } from "@/lib/cva.config";
import "./columnMeta";
import { DataTablePagination, DEFAULT_PAGE_SIZE_OPTIONS } from "./DataTablePagination";
import type { ColumnPinnedSide, DataTableProps, DataTableSize, PaginationMode, SortingMode } from "./types";
import type { ColumnPinnedSide, DataTableProps, DataTableSize, FilterMode, PaginationMode, SortingMode } from "./types";
const INTERACTIVE_SELECTOR = "button, a, input, select, textarea, [role=checkbox], [data-row-click-exempt]";
@ -60,14 +64,22 @@ export function validateDataTableConfig<TData extends RowData, TValue>(
props.pagination === undefined || props.onPaginationChange === undefined || props.rowCount === undefined;
const serverPaginationIncomplete = props.paginationMode === "server" && serverPaginationPropsMissing;
const serverFilteringIncomplete =
props.filterMode === "server" && (props.columnFilters === undefined || props.onColumnFiltersChange === undefined);
const bothSortingSources = props.defaultSorting !== undefined && props.sorting !== undefined;
const bothFilterSources = props.defaultColumnFilters !== undefined && props.columnFilters !== undefined;
return [
serverSortingIncomplete ? "sortingMode='server' requires both `sorting` and `onSortingChange`." : null,
serverPaginationIncomplete
? "paginationMode='server' requires `pagination`, `onPaginationChange`, and `rowCount`."
: null,
serverFilteringIncomplete ? "filterMode='server' requires both `columnFilters` and `onColumnFiltersChange`." : null,
bothSortingSources ? "Provide either `defaultSorting` (uncontrolled) or `sorting` (controlled), not both." : null,
bothFilterSources
? "Provide either `defaultColumnFilters` (uncontrolled) or `columnFilters` (controlled), not both."
: null,
].filter((message): message is string => message !== null);
}
@ -93,9 +105,11 @@ function derivePinning<TData, TValue>(columns: ColumnDef<TData, TValue>[]): Colu
function buildRowModels<TData>(
sortingMode: SortingMode,
paginationMode: PaginationMode,
filterMode: FilterMode,
getRowCanExpand: ((row: Row<TData>) => boolean) | undefined,
): Partial<TableOptions<TData>> {
return {
...(filterMode === "client" ? { getFilteredRowModel: getFilteredRowModel() } : {}),
...(sortingMode === "client" ? { getSortedRowModel: getSortedRowModel() } : {}),
...(paginationMode === "client" ? { getPaginationRowModel: getPaginationRowModel() } : {}),
...(getRowCanExpand !== undefined ? { getRowCanExpand, getExpandedRowModel: getExpandedRowModel() } : {}),
@ -307,6 +321,65 @@ function MessageRow({ colSpan, children }: { colSpan: number; children: React.Re
);
}
function DefaultEmptyState() {
return (
<div className="flex flex-col items-center gap-1 py-6">
<div className="mb-1 flex size-10 items-center justify-center rounded-lg bg-muted">
<SearchX className="size-5 text-muted-foreground" />
</div>
<div className="text-sm font-medium text-foreground">No results</div>
<div className="text-sm text-muted-foreground">No rows match your search or filters.</div>
</div>
);
}
const SKELETON_WIDTHS = ["w-[58%]", "w-[44%]", "w-[70%]", "w-[50%]", "w-[64%]", "w-[48%]"] as const;
function SkeletonCell<TData>({ column, index }: { column: Column<TData, unknown> | undefined; index: number }) {
const meta = column?.columnDef.meta;
const width = SKELETON_WIDTHS[index % SKELETON_WIDTHS.length];
if (meta?.skeleton === "twoLine") {
return (
<div className="flex flex-col gap-2">
<Skeleton className={cn("h-3.5", width)} />
<Skeleton className="h-2.5 w-2/5 opacity-65" />
</div>
);
}
return <Skeleton className={cn("h-3.5", width, meta?.numeric ? "ml-auto" : "")} />;
}
function SkeletonRows<TData>({
rowCount,
columns,
size,
message,
}: {
rowCount: number;
columns: readonly Column<TData, unknown>[];
size: DataTableSize;
message?: string;
}) {
const rowKeys = Array.from({ length: Math.max(rowCount, 1) }, (_, index) => index);
const cells = columns.length > 0 ? columns : [undefined];
return (
<Fragment>
{rowKeys.map((rowKey) => (
<TableRow key={`skeleton-${rowKey}`} className="hover:bg-transparent" data-testid="skeleton-row">
{cells.map((column, columnKey) => (
<TableCell key={column?.id ?? columnKey} className={size === "compact" ? "px-2 py-1" : ""}>
<SkeletonCell column={column} index={columnKey} />
{rowKey === 0 && columnKey === 0 && message !== undefined ? (
<span className="sr-only">{message}</span>
) : null}
</TableCell>
))}
</TableRow>
))}
</Fragment>
);
}
function useControllable<T>(
controlled: T | undefined,
controlledOnChange: OnChangeFn<T> | undefined,
@ -334,6 +407,12 @@ function useDataTableInstance<TData extends RowData, TValue>(props: DataTablePro
onPaginationChange,
rowCount,
pageSizeOptions = DEFAULT_PAGE_SIZE_OPTIONS,
filterMode = "none",
columnFilters,
onColumnFiltersChange,
defaultColumnFilters,
globalFilter,
onGlobalFilterChange,
enableColumnResizing = false,
columnResizeMode = "onEnd",
defaultColumnVisibility,
@ -348,6 +427,12 @@ function useDataTableInstance<TData extends RowData, TValue>(props: DataTablePro
pageIndex: 0,
pageSize: pageSizeOptions[0] ?? 25,
});
const filterState = useControllable<ColumnFiltersState>(
columnFilters,
onColumnFiltersChange,
defaultColumnFilters ?? [],
);
const globalFilterState = useControllable<string>(globalFilter, onGlobalFilterChange, "");
const expandedState = useControllable<ExpandedState>(expanded, onExpandedChange, {});
const [columnVisibility, setColumnVisibility] = useState<VisibilityState>(defaultColumnVisibility ?? {});
const [columnSizing, setColumnSizing] = useState<ColumnSizingState>({});
@ -360,6 +445,8 @@ function useDataTableInstance<TData extends RowData, TValue>(props: DataTablePro
state: {
sorting: sortingState.value,
pagination: paginationState.value,
columnFilters: filterState.value,
globalFilter: globalFilterState.value,
expanded: expandedState.value,
columnVisibility,
columnSizing,
@ -367,16 +454,19 @@ function useDataTableInstance<TData extends RowData, TValue>(props: DataTablePro
initialState: { columnPinning },
manualSorting: sortingMode === "server",
manualPagination: paginationMode === "server",
manualFiltering: filterMode === "server",
enableSortingRemoval,
enableColumnResizing,
columnResizeMode,
onSortingChange: sortingState.onChange,
onPaginationChange: paginationState.onChange,
onColumnFiltersChange: filterState.onChange,
onGlobalFilterChange: globalFilterState.onChange,
onExpandedChange: expandedState.onChange,
onColumnVisibilityChange: setColumnVisibility,
onColumnSizingChange: setColumnSizing,
getCoreRowModel: getCoreRowModel(),
...buildRowModels(sortingMode, paginationMode, expansionGuard),
...buildRowModels(sortingMode, paginationMode, filterMode, expansionGuard),
...(getRowId !== undefined ? { getRowId } : {}),
...(paginationMode === "server" && rowCount !== undefined ? { rowCount } : {}),
};
@ -397,7 +487,8 @@ export function DataTable<TData extends RowData, TValue>(props: DataTableProps<T
const {
isLoading = false,
loadingMessage = "Loading…",
noDataMessage = "No results",
skeletonRowCount = 8,
noDataMessage,
paginationMode = "none",
rowCount,
pageSizeOptions = DEFAULT_PAGE_SIZE_OPTIONS,
@ -443,10 +534,17 @@ export function DataTable<TData extends RowData, TValue>(props: DataTableProps<T
const renderBody = (): React.ReactNode => {
if (isLoading) {
return <MessageRow colSpan={visibleColumnCount}>{loadingMessage}</MessageRow>;
return (
<SkeletonRows
rowCount={skeletonRowCount}
columns={table.getVisibleLeafColumns()}
size={size}
message={loadingMessage}
/>
);
}
if (rows.length === 0) {
return <MessageRow colSpan={visibleColumnCount}>{noDataMessage}</MessageRow>;
return <MessageRow colSpan={visibleColumnCount}>{noDataMessage ?? <DefaultEmptyState />}</MessageRow>;
}
return rows.map((row) => (
<DataTableBodyRow
@ -462,34 +560,38 @@ export function DataTable<TData extends RowData, TValue>(props: DataTableProps<T
));
};
const paginationNode = renderPagination();
return (
<div className="w-full">
{toolbar !== undefined && <div className="w-full">{toolbar(table)}</div>}
<div
className={cn("rounded-lg border border-border", stickyHeader ? "overflow-auto" : "overflow-x-auto")}
style={stickyHeader ? { maxHeight: maxBodyHeight } : undefined}
>
<TableRoot className={enableColumnResizing ? "table-fixed" : ""} style={tableStyle}>
<TableHeader className={stickyHeader ? "sticky top-0 z-20" : ""}>
{table.getHeaderGroups().map((headerGroup) => (
<TableRow key={headerGroup.id} className="bg-muted/50 hover:bg-muted/50">
{headerGroup.headers.map((header) => (
<DataTableHeadCell
key={header.id}
header={header}
size={size}
stickyHeader={stickyHeader}
enableColumnResizing={enableColumnResizing}
/>
))}
</TableRow>
))}
</TableHeader>
<TableBody>{renderBody()}</TableBody>
{footer !== undefined && <TableFooter>{footer(table)}</TableFooter>}
</TableRoot>
<div className="overflow-hidden rounded-lg border border-border">
{toolbar !== undefined && <div className="border-b border-border px-4 py-3">{toolbar(table)}</div>}
<div
className={stickyHeader ? "overflow-auto" : "overflow-x-auto"}
style={stickyHeader ? { maxHeight: maxBodyHeight } : undefined}
>
<TableRoot className={enableColumnResizing ? "table-fixed" : ""} style={tableStyle}>
<TableHeader className={stickyHeader ? "sticky top-0 z-20" : ""}>
{table.getHeaderGroups().map((headerGroup) => (
<TableRow key={headerGroup.id} className="bg-muted/50 hover:bg-muted/50">
{headerGroup.headers.map((header) => (
<DataTableHeadCell
key={header.id}
header={header}
size={size}
stickyHeader={stickyHeader}
enableColumnResizing={enableColumnResizing}
/>
))}
</TableRow>
))}
</TableHeader>
<TableBody>{renderBody()}</TableBody>
{footer !== undefined && <TableFooter>{footer(table)}</TableFooter>}
</TableRoot>
</div>
{paginationNode !== null && <div className="border-t border-border">{paginationNode}</div>}
</div>
{renderPagination()}
</div>
);
}

98
ui/litellm-dashboard/src/components/shared/DataTable/DataTableFilterDrawer.test.tsx Normal file
View file

@ -0,0 +1,98 @@
import type { ColumnDef, ColumnFiltersState } from "@tanstack/react-table";
import { render, screen } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { useState } from "react";
import { describe, expect, it } from "vitest";
import { DataTable } from "./DataTable";
import { DataTableFilterDrawer } from "./DataTableFilterDrawer";
import { DataTableToolbar } from "./DataTableToolbar";
interface Person {
id: string;
name: string;
}
const DATA: Person[] = [
{ id: "a", name: "Alice" },
{ id: "b", name: "Bob" },
{ id: "c", name: "Carol" },
];
const columns: ColumnDef<Person, unknown>[] = [
{
accessorKey: "name",
header: "Name",
meta: { title: "Name" },
filterFn: (row, columnId, value) => row.getValue<string>(columnId) === value,
cell: ({ row }) => <span data-testid="name-cell">{row.original.name}</span>,
},
];
const names = (): (string | null)[] => screen.getAllByTestId("name-cell").map((el) => el.textContent);
function Harness({ initialFilters }: { initialFilters?: ColumnFiltersState }) {
const [open, setOpen] = useState(false);
return (
<DataTable
data={DATA}
columns={columns}
filterMode="client"
defaultColumnFilters={initialFilters}
toolbar={(table) => (
<>
<DataTableToolbar table={table} onOpenFilters={() => setOpen(true)} />
<DataTableFilterDrawer table={table} open={open} onOpenChange={setOpen} title="Filters">
{({ get, set }) => (
<input
aria-label="name filter"
data-testid="draft-name"
value={(get("name") as string | undefined) ?? ""}
onChange={(event) => set("name", event.target.value)}
/>
)}
</DataTableFilterDrawer>
</>
)}
/>
);
}
describe("DataTableFilterDrawer", () => {
it("stages edits and only commits them to the table on Apply", async () => {
const user = userEvent.setup();
render(<Harness />);
expect(names()).toEqual(["Alice", "Bob", "Carol"]);
await user.click(screen.getByTestId("datatable-filters-trigger"));
await user.type(await screen.findByTestId("draft-name"), "Bob");
expect(names()).toEqual(["Alice", "Bob", "Carol"]);
expect(screen.queryByTestId("filter-chip-name")).toBeNull();
await user.click(screen.getByTestId("filter-drawer-apply"));
expect(names()).toEqual(["Bob"]);
expect(screen.getByTestId("filter-chip-name")).toHaveTextContent("Bob");
});
it("seeds the draft from committed filters when opened", async () => {
const user = userEvent.setup();
render(<Harness initialFilters={[{ id: "name", value: "Bob" }]} />);
expect(names()).toEqual(["Bob"]);
await user.click(screen.getByTestId("datatable-filters-trigger"));
expect(await screen.findByTestId("draft-name")).toHaveValue("Bob");
});
it("reset clears the committed filters and the draft", async () => {
const user = userEvent.setup();
render(<Harness initialFilters={[{ id: "name", value: "Bob" }]} />);
await user.click(screen.getByTestId("datatable-filters-trigger"));
await user.click(await screen.findByTestId("filter-drawer-reset"));
expect(names()).toEqual(["Alice", "Bob", "Carol"]);
expect(screen.queryByTestId("filter-chip-name")).toBeNull();
expect(screen.getByTestId("draft-name")).toHaveValue("");
});
});

108
ui/litellm-dashboard/src/components/shared/DataTable/DataTableFilterDrawer.tsx Normal file
View file

@ -0,0 +1,108 @@
"use client";
import type { ColumnFiltersState, Table } from "@tanstack/react-table";
import * as React from "react";
import { Button } from "@/components/ui/button";
import { Label } from "@/components/ui/label";
import { Sheet, SheetContent, SheetDescription, SheetFooter, SheetHeader, SheetTitle } from "@/components/ui/sheet";
export interface FilterDraft {
get: (columnId: string) => unknown;
set: (columnId: string, value: unknown) => void;
}
interface DataTableFilterDrawerProps<TData> {
table: Table<TData>;
open: boolean;
onOpenChange: (open: boolean) => void;
title?: string;
description?: React.ReactNode;
applyLabel?: string;
resetLabel?: string;
children: (draft: FilterDraft) => React.ReactNode;
}
function isEmpty(value: unknown): boolean {
if (Array.isArray(value)) {
return value.length === 0;
}
return value === undefined || value === null || value === "";
}
function toDraft(filters: ColumnFiltersState): Record<string, unknown> {
return Object.fromEntries(filters.map((filter) => [filter.id, filter.value]));
}
function toFilters(draft: Record<string, unknown>): ColumnFiltersState {
return Object.entries(draft)
.filter(([, value]) => !isEmpty(value))
.map(([id, value]) => ({ id, value }));
}
export function DataTableFilterDrawer<TData>({
table,
open,
onOpenChange,
title = "Filters",
description,
applyLabel = "Apply Filters",
resetLabel = "Reset",
children,
}: DataTableFilterDrawerProps<TData>) {
const [draft, setDraft] = React.useState<Record<string, unknown>>(() => toDraft(table.getState().columnFilters));
const [wasOpen, setWasOpen] = React.useState(open);
if (open !== wasOpen) {
setWasOpen(open);
if (open) {
setDraft(toDraft(table.getState().columnFilters));
}
}
const helpers: FilterDraft = {
get: (columnId) => draft[columnId],
set: (columnId, value) => setDraft((previous) => ({ ...previous, [columnId]: value })),
};
const apply = () => {
table.setColumnFilters(toFilters(draft));
onOpenChange(false);
};
const reset = () => {
setDraft({});
table.setColumnFilters([]);
};
return (
<Sheet open={open} onOpenChange={onOpenChange}>
<SheetContent side="right">
<SheetHeader>
<SheetTitle>{title}</SheetTitle>
{description !== undefined && <SheetDescription>{description}</SheetDescription>}
</SheetHeader>
<div className="flex min-h-0 flex-1 flex-col gap-4 overflow-y-auto p-4" data-testid="filter-drawer-body">
{children(helpers)}
</div>
<SheetFooter className="flex-row">
<Button variant="outline" className="flex-1" onClick={reset} data-testid="filter-drawer-reset">
{resetLabel}
</Button>
<Button className="flex-1" onClick={apply} data-testid="filter-drawer-apply">
{applyLabel}
</Button>
</SheetFooter>
</SheetContent>
</Sheet>
);
}
export function DataTableFilterField({ label, children }: { label: string; children: React.ReactNode }) {
return (
<div className="flex flex-col gap-1.5">
<Label>{label}</Label>
{children}
</div>
);
}

5
ui/litellm-dashboard/src/components/shared/DataTable/DataTablePagination.tsx
View file

@ -37,7 +37,7 @@ export function DataTablePagination({
const lastPage = Math.max(pageCount - 1, 0);
return (
<div className={cn("flex flex-wrap items-center justify-between gap-4 px-2 py-2", className)}>
<div className={cn("flex flex-wrap items-center justify-between gap-4 px-4 py-2.5", className)}>
<div className="flex items-center gap-2 text-sm text-muted-foreground">
<span>Rows per page</span>
<Select
@ -65,6 +65,9 @@ export function DataTablePagination({
<span data-testid="pagination-range" className="text-sm text-muted-foreground tabular-nums">
{rowCount === 0 ? "No results" : `Showing ${start}-${end} of ${rowCount}`}
</span>
<span data-testid="pagination-page" className="text-sm text-muted-foreground tabular-nums">
Page {page + 1} of {Math.max(pageCount, 1)}
</span>
<div className="flex items-center gap-1">
<Button
variant="outline"

107
ui/litellm-dashboard/src/components/shared/DataTable/DataTableToolbar.test.tsx
View file

@ -1,35 +1,106 @@
import type { ColumnDef } from "@tanstack/react-table";
import { render, screen } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import type * as React from "react";
import { describe, expect, it, vi } from "vitest";
import { DataTable } from "./DataTable";
import { DataTableToolbar } from "./DataTableToolbar";
interface Person {
id: string;
name: string;
}
const DATA: Person[] = [
{ id: "a", name: "Alice" },
{ id: "b", name: "Bob" },
];
const columns: ColumnDef<Person, unknown>[] = [
{
accessorKey: "name",
header: "Name",
meta: { title: "Name" },
filterFn: (row, columnId, value) => row.getValue<string>(columnId) === value,
cell: ({ row }) => <span data-testid="name-cell">{row.original.name}</span>,
},
];
const names = (): (string | null)[] => screen.getAllByTestId("name-cell").map((el) => el.textContent);
function Harness({
onOpenFilters,
onRefresh,
children,
}: {
onOpenFilters?: () => void;
onRefresh?: () => void;
children?: React.ReactNode;
}) {
return (
<DataTable
data={DATA}
columns={columns}
filterMode="client"
defaultColumnFilters={[{ id: "name", value: "Alice" }]}
toolbar={(table) => (
<DataTableToolbar table={table} onOpenFilters={onOpenFilters} onRefresh={onRefresh}>
{children}
</DataTableToolbar>
)}
/>
);
}
describe("DataTableToolbar", () => {
it("renders a chip for each active filter with its label and value", () => {
render(<Harness />);
expect(names()).toEqual(["Alice"]);
const chip = screen.getByTestId("filter-chip-name");
expect(chip).toHaveTextContent("Name:");
expect(chip).toHaveTextContent("Alice");
});
it("removes a single filter when its chip remove button is clicked", async () => {
const user = userEvent.setup();
render(<Harness />);
await user.click(screen.getByTestId("filter-chip-remove-name"));
expect(screen.queryByTestId("filter-chip-name")).toBeNull();
expect(names()).toEqual(["Alice", "Bob"]);
});
it("clears every filter via Clear all", async () => {
const user = userEvent.setup();
render(<Harness />);
await user.click(screen.getByTestId("datatable-clear-filters"));
expect(screen.queryByTestId("filter-chip-name")).toBeNull();
expect(names()).toEqual(["Alice", "Bob"]);
});
it("shows the active filter count and fires onOpenFilters", async () => {
const user = userEvent.setup();
const onOpenFilters = vi.fn();
render(<Harness onOpenFilters={onOpenFilters} />);
expect(screen.getByTestId("datatable-filter-count")).toHaveTextContent("1");
await user.click(screen.getByTestId("datatable-filters-trigger"));
expect(onOpenFilters).toHaveBeenCalledTimes(1);
});
it("renders slotted action children", () => {
render(
<DataTableToolbar>
<Harness>
<button data-testid="toolbar-action">Action</button>
</DataTableToolbar>,
</Harness>,
);
expect(screen.getByTestId("toolbar-action")).toBeInTheDocument();
});
it("shows the reset button only when there are active filters", async () => {
it("fires onRefresh when the refresh button is clicked", async () => {
const user = userEvent.setup();
const onResetFilters = vi.fn();
const { rerender } = render(<DataTableToolbar onResetFilters={onResetFilters} hasActiveFilters={false} />);
expect(screen.queryByText("Reset Filters")).toBeNull();
rerender(<DataTableToolbar onResetFilters={onResetFilters} hasActiveFilters />);
await user.click(screen.getByText("Reset Filters"));
expect(onResetFilters).toHaveBeenCalledTimes(1);
});
it("wires the filters toggle button", async () => {
const user = userEvent.setup();
const onToggleFilters = vi.fn();
render(<DataTableToolbar onToggleFilters={onToggleFilters} />);
await user.click(screen.getByText("Filters"));
expect(onToggleFilters).toHaveBeenCalledTimes(1);
const onRefresh = vi.fn();
render(<Harness onRefresh={onRefresh} />);
await user.click(screen.getByTestId("datatable-refresh"));
expect(onRefresh).toHaveBeenCalledTimes(1);
});
});

129
ui/litellm-dashboard/src/components/shared/DataTable/DataTableToolbar.tsx
View file

@ -1,55 +1,128 @@
"use client";
import { Search } from "lucide-react";
import type { Table } from "@tanstack/react-table";
import { RefreshCw, Search, SlidersHorizontal, X } from "lucide-react";
import type * as React from "react";
import { FilterInput } from "@/components/common_components/Filters/FilterInput";
import { FiltersButton } from "@/components/common_components/Filters/FiltersButton";
import { ResetFiltersButton } from "@/components/common_components/Filters/ResetFiltersButton";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { cn } from "@/lib/cva.config";
interface DataTableToolbarProps {
import { DataTableViewOptions } from "./DataTableViewOptions";
interface DataTableToolbarProps<TData> {
table: Table<TData>;
searchValue?: string;
onSearchChange?: (value: string) => void;
searchPlaceholder?: string;
filtersActive?: boolean;
hasActiveFilters?: boolean;
onToggleFilters?: () => void;
onResetFilters?: () => void;
onOpenFilters?: () => void;
onRefresh?: () => void;
isRefreshing?: boolean;
filterLabels?: Record<string, string>;
formatFilterValue?: (columnId: string, value: unknown) => string;
showViewOptions?: boolean;
children?: React.ReactNode;
className?: string;
}
export function DataTableToolbar({
function defaultFormatValue(value: unknown): string {
if (Array.isArray(value)) {
return value.join(", ");
}
return String(value);
}
export function DataTableToolbar<TData>({
table,
searchValue,
onSearchChange,
searchPlaceholder = "Search",
filtersActive = false,
hasActiveFilters = false,
onToggleFilters,
onResetFilters,
onOpenFilters,
onRefresh,
isRefreshing = false,
filterLabels,
formatFilterValue,
showViewOptions = true,
children,
className,
}: DataTableToolbarProps) {
const showReset = onResetFilters !== undefined && hasActiveFilters;
}: DataTableToolbarProps<TData>) {
const filters = table.getState().columnFilters;
const labelFor = (columnId: string): string =>
filterLabels?.[columnId] ?? table.getColumn(columnId)?.columnDef.meta?.title ?? columnId;
const valueFor = (columnId: string, value: unknown): string =>
formatFilterValue?.(columnId, value) ?? defaultFormatValue(value);
return (
<div className={cn("flex flex-wrap items-center justify-between gap-2 pb-3", className)}>
<div className="flex flex-wrap items-center gap-2">
<div className={cn("flex flex-wrap items-center justify-between gap-2", className)}>
<div className="flex flex-1 flex-wrap items-center gap-2">
{onSearchChange !== undefined && (
<FilterInput
value={searchValue ?? ""}
onChange={onSearchChange}
placeholder={searchPlaceholder}
icon={Search}
/>
<div className="relative">
<Search className="pointer-events-none absolute top-1/2 left-2.5 size-4 -translate-y-1/2 text-muted-foreground" />
<Input
value={searchValue ?? ""}
onChange={(event) => onSearchChange(event.target.value)}
placeholder={searchPlaceholder}
className="h-8 w-56 pl-8"
data-testid="datatable-search"
/>
</div>
)}
{onToggleFilters !== undefined && (
<FiltersButton onClick={onToggleFilters} active={filtersActive} hasActiveFilters={hasActiveFilters} />
{filters.map((filter) => (
<Badge key={filter.id} variant="outline" className="gap-1 py-1" data-testid={`filter-chip-${filter.id}`}>
<span className="text-muted-foreground">{labelFor(filter.id)}:</span>
{valueFor(filter.id, filter.value)}
<button
type="button"
aria-label={`Remove ${labelFor(filter.id)} filter`}
data-testid={`filter-chip-remove-${filter.id}`}
onClick={() => table.setColumnFilters((previous) => previous.filter((entry) => entry.id !== filter.id))}
className="ml-0.5 rounded-full text-muted-foreground hover:text-foreground"
>
<X className="size-3" />
</button>
</Badge>
))}
{filters.length > 0 && (
<Button
variant="ghost"
size="sm"
onClick={() => table.setColumnFilters([])}
data-testid="datatable-clear-filters"
>
Clear all
</Button>
)}
</div>
<div className="flex flex-wrap items-center gap-2">
{children}
{onRefresh !== undefined && (
<Button
variant="outline"
size="icon-sm"
onClick={onRefresh}
disabled={isRefreshing}
aria-label="Refresh"
title="Refresh"
data-testid="datatable-refresh"
>
<RefreshCw className={isRefreshing ? "animate-spin" : ""} />
</Button>
)}
{showViewOptions && <DataTableViewOptions table={table} label="Columns" />}
{onOpenFilters !== undefined && (
<Button variant="outline" size="sm" onClick={onOpenFilters} data-testid="datatable-filters-trigger">
<SlidersHorizontal />
Filters
{filters.length > 0 && (
<Badge className="ml-1 h-5 min-w-5 justify-center rounded-full px-1" data-testid="datatable-filter-count">
{filters.length}
</Badge>
)}
</Button>
)}
{showReset && <ResetFiltersButton onClick={onResetFilters} />}
</div>
{children !== undefined && <div className="flex flex-wrap items-center gap-2">{children}</div>}
</div>
);
}

7
ui/litellm-dashboard/src/components/shared/DataTable/DataTableViewOptions.tsx
View file

@ -2,7 +2,7 @@
import { Menu } from "@base-ui/react/menu";
import type { Table } from "@tanstack/react-table";
import { Check, SlidersHorizontal } from "lucide-react";
import { Check, Columns3 } from "lucide-react";
import { Button } from "@/components/ui/button";
@ -24,7 +24,7 @@ export function DataTableViewOptions<TData>({ table, label = "View", className }
<Menu.Trigger
render={
<Button variant="outline" size="sm" className={className} data-testid="view-options-trigger">
<SlidersHorizontal />
<Columns3 />
{label}
</Button>
}
@ -44,7 +44,8 @@ export function DataTableViewOptions<TData>({ table, label = "View", className }
<Menu.CheckboxItemIndicator className="absolute left-2 flex size-4 items-center justify-center">
<Check className="size-3.5" />
</Menu.CheckboxItemIndicator>
{column.columnDef.meta?.title ?? column.id}
{column.columnDef.meta?.title ??
(typeof column.columnDef.header === "string" ? column.columnDef.header : column.id)}
</Menu.CheckboxItem>
))}
</Menu.Popup>

3
ui/litellm-dashboard/src/components/shared/DataTable/columnMeta.ts
View file

@ -1,6 +1,6 @@
import type { RowData } from "@tanstack/react-table";
import type { ColumnPinnedSide } from "./types";
import type { ColumnPinnedSide, DataTableSkeletonShape } from "./types";
declare module "@tanstack/react-table" {
interface ColumnMeta<TData extends RowData, TValue> {
@ -9,5 +9,6 @@ declare module "@tanstack/react-table" {
headerClassName?: string;
title?: string;
pinned?: ColumnPinnedSide;
skeleton?: DataTableSkeletonShape;
}
}

2
ui/litellm-dashboard/src/components/shared/DataTable/index.ts
View file

@ -1,6 +1,7 @@
import "./columnMeta";
export { DataTable, DataTableConfigError, validateDataTableConfig } from "./DataTable";
export { DataTableFilterDrawer, DataTableFilterField, type FilterDraft } from "./DataTableFilterDrawer";
export { DataTablePagination, DEFAULT_PAGE_SIZE_OPTIONS } from "./DataTablePagination";
export { DataTableToolbar } from "./DataTableToolbar";
export { DataTableViewOptions } from "./DataTableViewOptions";
@ -11,6 +12,7 @@ export type {
ColumnResizeMode,
DataTableProps,
DataTableSize,
FilterMode,
PaginationMode,
SortingMode,
} from "./types";

12
ui/litellm-dashboard/src/components/shared/DataTable/types.ts
View file

@ -1,5 +1,6 @@
import type {
ColumnDef,
ColumnFiltersState,
ExpandedState,
OnChangeFn,
PaginationState,
@ -13,9 +14,11 @@ import type * as React from "react";
export type SortingMode = "none" | "client" | "server";
export type PaginationMode = "none" | "client" | "server";
export type FilterMode = "none" | "client" | "server";
export type ColumnResizeMode = "onEnd" | "onChange";
export type DataTableSize = "compact" | "default";
export type ColumnPinnedSide = "left" | "right";
export type DataTableSkeletonShape = "text" | "twoLine";
export interface DataTableProps<TData extends RowData, TValue> {
data: TData[];
@ -24,6 +27,7 @@ export interface DataTableProps<TData extends RowData, TValue> {
isLoading?: boolean;
loadingMessage?: string;
skeletonRowCount?: number;
noDataMessage?: React.ReactNode;
sortingMode?: SortingMode;
@ -38,6 +42,14 @@ export interface DataTableProps<TData extends RowData, TValue> {
rowCount?: number;
pageSizeOptions?: number[];
filterMode?: FilterMode;
columnFilters?: ColumnFiltersState;
onColumnFiltersChange?: OnChangeFn<ColumnFiltersState>;
defaultColumnFilters?: ColumnFiltersState;
globalFilter?: string;
onGlobalFilterChange?: OnChangeFn<string>;
enableColumnResizing?: boolean;
columnResizeMode?: ColumnResizeMode;
defaultColumnVisibility?: VisibilityState;

2
ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx
View file

@ -583,7 +583,7 @@ describe("TeamInfoView", () => {
await waitFor(() => {
expect(screen.getByRole("button", { name: "Filters" })).toBeInTheDocument();
});
expect(screen.getByRole("button", { name: "Reset Filters" })).toBeInTheDocument();
expect(screen.getByRole("button", { name: "Columns" })).toBeInTheDocument();
expect(screen.getByTestId("pagination-range")).toHaveTextContent("Showing 1-1 of 1");
expect(screen.getByTestId("pagination-prev")).toBeInTheDocument();
expect(screen.getByTestId("pagination-next")).toBeInTheDocument();

86
ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx
View file

@ -4,8 +4,6 @@ import { beforeEach, describe, expect, it, vi, MockedFunction } from "vitest";
import { renderWithProviders } from "../../../tests/test-utils";
import { TeamVirtualKeysTable } from "./TeamVirtualKeysTable";
import { KeysResponse, useKeys } from "@/app/(dashboard)/hooks/keys/useKeys";
import { fetchTeamFilterOptions } from "../key_team_helpers/filter_helpers";
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
import { KeyResponse } from "../key_team_helpers/key_list";
import { Organization } from "../networking";
@ -13,18 +11,6 @@ vi.mock("@/app/(dashboard)/hooks/keys/useKeys", () => ({
useKeys: vi.fn(),
}));
vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({
default: vi.fn(),
}));
vi.mock("../key_team_helpers/filter_helpers", () => ({
fetchTeamFilterOptions: vi.fn().mockResolvedValue({
keyAliases: [],
organizationIds: [],
userIds: [],
}),
}));
vi.mock("../key_team_helpers/fetch_available_models_team_key", () => ({
getModelDisplayName: vi.fn((model: string) => model),
}));
@ -38,8 +24,12 @@ vi.mock("../templates/key_info_view", () => ({
)),
}));
// Resolve the debounced search synchronously so typed input lands in the useKeys query within the test tick.
vi.mock("@tanstack/react-pacer/debouncer", () => ({
useDebouncedValue: (value: unknown) => [value, { cancel: vi.fn(), flush: vi.fn() }],
}));
const mockUseKeys = useKeys as MockedFunction<typeof useKeys>;
const mockUseAuthorized = useAuthorized as MockedFunction<typeof useAuthorized>;
const createMockKey = (overrides: Partial<KeyResponse> = {}): KeyResponse =>
({
@ -85,7 +75,6 @@ describe("TeamVirtualKeysTable", () => {
beforeEach(() => {
vi.clearAllMocks();
mockUseAuthorized.mockReturnValue({ accessToken: "test-token" } as any);
mockUseKeys.mockReturnValue({
data: { keys: [], total_count: 0, current_page: 1, total_pages: 1 } as KeysResponse,
isPending: false,
@ -262,30 +251,48 @@ describe("TeamVirtualKeysTable", () => {
await waitFor(() => expect(mockUseKeys).toHaveBeenLastCalledWith(1, 50, expect.anything()));
});
it("resets the sort order to the default when filters are reset", async () => {
it("maps the User ID drawer filter to a server-side useKeys query and clears it", async () => {
const user = userEvent.setup();
const result = {
mockUseKeys.mockReturnValue({
data: { keys: [createMockKey()], total_count: 1, current_page: 1, total_pages: 1 },
isPending: false,
isFetching: false,
refetch: vi.fn(),
} as unknown as ReturnType<typeof useKeys>;
mockUseKeys.mockReturnValue(result);
} as unknown as ReturnType<typeof useKeys>);
renderWithProviders(<TeamVirtualKeysTable {...defaultProps} />);
await user.click(await screen.findByTestId("sort-header-created_at"));
await user.click(await screen.findByTestId("datatable-filters-trigger"));
const drawerBody = await screen.findByTestId("filter-drawer-body");
const userInput = drawerBody.querySelector("input") as HTMLElement;
await user.type(userInput, "user-42");
await user.click(screen.getByTestId("filter-drawer-apply"));
await waitFor(() =>
expect(mockUseKeys).toHaveBeenLastCalledWith(1, 50, expect.objectContaining({ sortOrder: "asc" })),
expect(mockUseKeys).toHaveBeenLastCalledWith(1, 50, expect.objectContaining({ userID: "user-42" })),
);
await user.click(screen.getByRole("button", { name: "Reset Filters" }));
await user.click(screen.getByTestId("datatable-clear-filters"));
await waitFor(() =>
expect(mockUseKeys).toHaveBeenLastCalledWith(
1,
50,
expect.objectContaining({ sortBy: "created_at", sortOrder: "desc" }),
),
expect(mockUseKeys).toHaveBeenLastCalledWith(1, 50, expect.objectContaining({ userID: undefined })),
);
});
it("maps the search box to a server-side key-alias query", async () => {
const user = userEvent.setup();
mockUseKeys.mockReturnValue({
data: { keys: [createMockKey()], total_count: 1, current_page: 1, total_pages: 1 },
isPending: false,
isFetching: false,
refetch: vi.fn(),
} as unknown as ReturnType<typeof useKeys>);
renderWithProviders(<TeamVirtualKeysTable {...defaultProps} />);
await user.type(await screen.findByTestId("datatable-search"), "check-002");
await waitFor(() =>
expect(mockUseKeys).toHaveBeenLastCalledWith(1, 50, expect.objectContaining({ selectedKeyAlias: "check-002" })),
);
});
@ -304,7 +311,7 @@ describe("TeamVirtualKeysTable", () => {
});
});
it("should show No keys found when keys array is empty", async () => {
it("should show the empty state when keys array is empty", async () => {
mockUseKeys.mockReturnValue({
data: { keys: [], total_count: 0, current_page: 1, total_pages: 1 } as KeysResponse,
isPending: false,
@ -315,26 +322,7 @@ describe("TeamVirtualKeysTable", () => {
renderWithProviders(<TeamVirtualKeysTable {...defaultProps} />);
await waitFor(() => {
expect(screen.getByText("No keys found")).toBeInTheDocument();
});
});
it("should fetch team-scoped filter options for Key Alias, Organization ID, and User ID", async () => {
const mockFetchTeamFilterOptions = vi.mocked(fetchTeamFilterOptions);
mockFetchTeamFilterOptions.mockResolvedValue({
keyAliases: ["alice_key_team1", "charlie_key_team1"],
organizationIds: ["org-123"],
userIds: [
{ id: "user-1", email: "alice@example.com" },
{ id: "user-2", email: "charlie@example.com" },
],
});
// Use unique teamId to avoid cache hit from previous tests (refetchOnMount: false)
renderWithProviders(<TeamVirtualKeysTable {...defaultProps} teamId="team-filter-options-test" />);
await waitFor(() => {
expect(mockFetchTeamFilterOptions).toHaveBeenCalledWith("test-token", "team-filter-options-test");
expect(screen.getByText("No rows match your search or filters.")).toBeInTheDocument();
});
});

184
ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx
View file

@ -1,21 +1,25 @@
"use client";
import { useKeys } from "@/app/(dashboard)/hooks/keys/useKeys";
import { DateCell, IdCell, MoneyCell } from "@/components/shared/table_cells";
import { DataTable, DataTablePagination, DataTableSortHeader } from "@/components/shared/DataTable";
import {
DataTable,
DataTableFilterDrawer,
DataTableFilterField,
DataTableSortHeader,
DataTableToolbar,
} from "@/components/shared/DataTable";
import { Input } from "@/components/ui/input";
import { ChevronDownIcon, ChevronRightIcon } from "@heroicons/react/outline";
import { ColumnDef, PaginationState, SortingState } from "@tanstack/react-table";
import { useDebouncedValue } from "@tanstack/react-pacer/debouncer";
import { ColumnDef, ColumnFiltersState, OnChangeFn, PaginationState, SortingState } from "@tanstack/react-table";
import { Badge, Icon, Text } from "@tremor/react";
import { Popover, Tooltip, Typography } from "antd";
import DefaultProxyAdminTag from "../common_components/DefaultProxyAdminTag";
import React, { useCallback, useEffect, useMemo, useState } from "react";
import { getModelDisplayName } from "../key_team_helpers/fetch_available_models_team_key";
import { KeyResponse, Team } from "../key_team_helpers/key_list";
import FilterComponent, { FilterOption } from "../molecules/filter";
import { Organization } from "../networking";
import KeyInfoView from "../templates/key_info_view";
import { useQuery } from "@tanstack/react-query";
import { fetchTeamFilterOptions } from "../key_team_helpers/filter_helpers";
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
interface TeamVirtualKeysTableProps {
teamId: string;
@ -30,18 +34,29 @@ interface TeamVirtualKeysTableProps {
const DEFAULT_SORTING: SortingState = [{ id: "created_at", desc: true }];
export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVirtualKeysTableProps) {
const { accessToken } = useAuthorized();
const [selectedKey, setSelectedKey] = useState<KeyResponse | null>(null);
const [sorting, setSorting] = useState<SortingState>(DEFAULT_SORTING);
const [tablePagination, setTablePagination] = useState<PaginationState>({
pageIndex: 0,
pageSize: 50,
});
const [filters, setFilters] = useState<Record<string, string>>({
"Organization ID": "",
"Key Alias": "",
"User ID": "",
});
const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>([]);
const [filtersOpen, setFiltersOpen] = useState(false);
const [searchInput, setSearchInput] = useState("");
const [searchQuery] = useDebouncedValue(searchInput, { wait: 300 });
const handleSearchChange = useCallback((value: string) => {
setSearchInput(value);
setTablePagination((prev) => ({ ...prev, pageIndex: 0 }));
}, []);
const getFilterValue = useCallback(
(columnId: string): string | undefined => {
const entry = columnFilters.find((filter) => filter.id === columnId);
return typeof entry?.value === "string" && entry.value.trim() ? entry.value.trim() : undefined;
},
[columnFilters],
);
const sortBy = sorting.length > 0 ? sorting[0].id : "created_at";
const sortOrder = sorting.length > 0 ? (sorting[0].desc ? "desc" : "asc") : "desc";
@ -56,9 +71,8 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi
refetch,
} = useKeys(pageIndex + 1, pageSize, {
teamID: teamId,
organizationID: filters["Organization ID"]?.trim() || undefined,
selectedKeyAlias: filters["Key Alias"]?.trim() || undefined,
userID: filters["User ID"]?.trim() || undefined,
selectedKeyAlias: searchQuery.trim() || undefined,
userID: getFilterValue("user_id"),
sortBy: sortBy || undefined,
sortOrder: sortOrder || undefined,
expand: "user",
@ -95,18 +109,6 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi
[teamId, teamAlias, organization],
);
const teamFilterOptionsQuery = useQuery({
queryKey: ["teamFilterOptions", teamId, accessToken],
queryFn: async () => fetchTeamFilterOptions(accessToken, teamId),
enabled: !!accessToken && !!teamId,
staleTime: 30000, // 30 seconds - align with useKeys
});
const teamFilterOptions = teamFilterOptionsQuery.data || {
keyAliases: [],
organizationIds: [],
userIds: [],
};
const handleStorageChange = useCallback(() => {
refetch?.();
}, [refetch]);
@ -116,76 +118,17 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi
return () => window.removeEventListener("storage", handleStorageChange);
}, [handleStorageChange]);
const handleFilterChange = useCallback((newFilters: Record<string, string>) => {
setFilters((prev) => ({
...prev,
"Organization ID": newFilters["Organization ID"] ?? prev["Organization ID"],
"Key Alias": newFilters["Key Alias"] ?? prev["Key Alias"],
"User ID": newFilters["User ID"] ?? prev["User ID"],
}));
const handleColumnFiltersChange = useCallback<OnChangeFn<ColumnFiltersState>>((updaterOrValue) => {
setColumnFilters(updaterOrValue);
setTablePagination((prev) => ({ ...prev, pageIndex: 0 }));
}, []);
const handleFilterReset = useCallback(() => {
setFilters({
"Organization ID": "",
"Key Alias": "",
"User ID": "",
});
setSorting(DEFAULT_SORTING);
setTablePagination((prev) => ({ ...prev, pageIndex: 0 }));
}, []);
const filterOptions: FilterOption[] = useMemo(
() => [
{
name: "Organization ID",
label: "Organization ID",
isSearchable: true,
searchFn: async (searchText: string) => {
const { organizationIds } = teamFilterOptions;
if (!organizationIds.length) return [];
const lower = searchText.toLowerCase();
const filtered = lower ? organizationIds.filter((id) => id.toLowerCase().includes(lower)) : organizationIds;
return filtered.map((id) => ({ label: id, value: id }));
},
},
{
name: "Key Alias",
label: "Key Alias",
isSearchable: true,
searchFn: async (searchText: string) => {
const { keyAliases } = teamFilterOptions;
const lower = searchText.toLowerCase();
const filtered = lower ? keyAliases.filter((alias) => alias.toLowerCase().includes(lower)) : keyAliases;
return filtered.map((alias) => ({ label: alias, value: alias }));
},
},
{
name: "User ID",
label: "User ID",
isSearchable: true,
searchFn: async (searchText: string) => {
const { userIds } = teamFilterOptions;
const lower = searchText.toLowerCase();
const filtered = lower
? userIds.filter((u) => u.id.toLowerCase().includes(lower) || u.email.toLowerCase().includes(lower))
: userIds;
return filtered.map((u) => ({
label: u.email ? `${u.id} (${u.email})` : u.id,
value: u.id,
}));
},
},
],
[teamFilterOptions],
);
const columns: ColumnDef<KeyResponse>[] = useMemo(
() => [
{
id: "token",
accessorKey: "token",
meta: { title: "Key ID" },
header: ({ column }) => <DataTableSortHeader column={column} title="Key ID" variant="header-cycle" />,
size: 120,
enableSorting: true,
@ -196,6 +139,7 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi
{
id: "key_alias",
accessorKey: "key_alias",
meta: { title: "Key Alias" },
header: ({ column }) => <DataTableSortHeader column={column} title="Key Alias" variant="header-cycle" />,
size: 150,
enableSorting: true,
@ -268,6 +212,7 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi
{
id: "created_at",
accessorKey: "created_at",
meta: { title: "Created At" },
header: ({ column }) => <DataTableSortHeader column={column} title="Created At" variant="header-cycle" />,
size: 120,
enableSorting: true,
@ -335,6 +280,7 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi
{
id: "updated_at",
accessorKey: "updated_at",
meta: { title: "Updated At" },
header: ({ column }) => <DataTableSortHeader column={column} title="Updated At" variant="header-cycle" />,
size: 120,
enableSorting: true,
@ -359,6 +305,7 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi
{
id: "spend",
accessorKey: "spend",
meta: { title: "Spend (USD)" },
header: ({ column }) => <DataTableSortHeader column={column} title="Spend (USD)" variant="header-cycle" />,
size: 100,
enableSorting: true,
@ -367,6 +314,7 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi
{
id: "max_budget",
accessorKey: "max_budget",
meta: { title: "Budget (USD)" },
header: ({ column }) => <DataTableSortHeader column={column} title="Budget (USD)" variant="header-cycle" />,
size: 110,
enableSorting: true,
@ -503,27 +451,7 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi
onDelete={refetch}
/>
) : (
<div className="border-b py-4 flex-1 overflow-hidden">
<div className="w-full mb-6">
<FilterComponent
options={filterOptions}
onApplyFilters={handleFilterChange}
initialValues={filters}
onResetFilters={handleFilterReset}
/>
</div>
<div className="w-full mb-4">
<DataTablePagination
page={pageIndex}
pageSize={pageSize}
rowCount={rowCount}
onPageChange={(nextPage) => setTablePagination((prev) => ({ ...prev, pageIndex: nextPage }))}
onPageSizeChange={(nextSize) => setTablePagination({ pageIndex: 0, pageSize: nextSize })}
isLoading={isLoading || isFetching}
/>
</div>
<div className="py-4 flex-1 overflow-hidden">
<DataTable
data={displayKeys}
columns={columns}
@ -534,14 +462,46 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi
pagination={tablePagination}
onPaginationChange={setTablePagination}
rowCount={rowCount}
paginationSlot={() => null}
filterMode="server"
columnFilters={columnFilters}
onColumnFiltersChange={handleColumnFiltersChange}
enableColumnResizing
columnResizeMode="onChange"
isLoading={isLoading || isFetching}
loadingMessage="Loading keys..."
noDataMessage="No keys found"
maxBodyHeight="75vh"
size="compact"
toolbar={(table) => (
<>
<DataTableToolbar
table={table}
searchValue={searchInput}
onSearchChange={handleSearchChange}
searchPlaceholder="Search by key alias…"
onRefresh={() => refetch?.()}
isRefreshing={isFetching}
onOpenFilters={() => setFiltersOpen(true)}
filterLabels={{ user_id: "User ID" }}
/>
<DataTableFilterDrawer
table={table}
open={filtersOpen}
onOpenChange={setFiltersOpen}
title="Filters"
description={`Narrow down keys for ${teamAlias ?? "this team"}`}
>
{({ get, set }) => (
<DataTableFilterField label="User ID">
<Input
value={(get("user_id") as string) ?? ""}
onChange={(event) => set("user_id", event.target.value)}
placeholder="Filter by user ID…"
/>
</DataTableFilterField>
)}
</DataTableFilterDrawer>
</>
)}
/>
</div>
)}

100
ui/litellm-dashboard/src/components/ui/sheet.tsx Normal file
View file

@ -0,0 +1,100 @@
"use client";
import * as React from "react";
import { Dialog as SheetPrimitive } from "@base-ui/react/dialog";
import { cn } from "@/lib/cva.config";
import { Button } from "@/components/ui/button";
import { XIcon } from "lucide-react";
function Sheet({ ...props }: SheetPrimitive.Root.Props) {
return <SheetPrimitive.Root data-slot="sheet" {...props} />;
}
function SheetTrigger({ ...props }: SheetPrimitive.Trigger.Props) {
return <SheetPrimitive.Trigger data-slot="sheet-trigger" {...props} />;
}
function SheetClose({ ...props }: SheetPrimitive.Close.Props) {
return <SheetPrimitive.Close data-slot="sheet-close" {...props} />;
}
function SheetPortal({ ...props }: SheetPrimitive.Portal.Props) {
return <SheetPrimitive.Portal data-slot="sheet-portal" {...props} />;
}
function SheetOverlay({ className, ...props }: SheetPrimitive.Backdrop.Props) {
return (
<SheetPrimitive.Backdrop
data-slot="sheet-overlay"
className={cn(
"fixed inset-0 z-50 bg-black/10 transition-opacity duration-150 data-ending-style:opacity-0 data-starting-style:opacity-0 supports-backdrop-filter:backdrop-blur-xs",
className,
)}
{...props}
/>
);
}
function SheetContent({
className,
children,
side = "right",
showCloseButton = true,
...props
}: SheetPrimitive.Popup.Props & {
side?: "top" | "right" | "bottom" | "left";
showCloseButton?: boolean;
}) {
return (
<SheetPortal>
<SheetOverlay />
<SheetPrimitive.Popup
data-slot="sheet-content"
data-side={side}
className={cn(
"fixed z-50 flex flex-col gap-4 bg-popover bg-clip-padding text-sm text-popover-foreground shadow-lg transition duration-200 ease-in-out data-ending-style:opacity-0 data-starting-style:opacity-0 data-[side=bottom]:inset-x-0 data-[side=bottom]:bottom-0 data-[side=bottom]:h-auto data-[side=bottom]:border-t data-[side=bottom]:data-ending-style:translate-y-[2.5rem] data-[side=bottom]:data-starting-style:translate-y-[2.5rem] data-[side=left]:inset-y-0 data-[side=left]:left-0 data-[side=left]:h-full data-[side=left]:w-3/4 data-[side=left]:border-r data-[side=left]:data-ending-style:translate-x-[-2.5rem] data-[side=left]:data-starting-style:translate-x-[-2.5rem] data-[side=right]:inset-y-0 data-[side=right]:right-0 data-[side=right]:h-full data-[side=right]:w-3/4 data-[side=right]:border-l data-[side=right]:data-ending-style:translate-x-[2.5rem] data-[side=right]:data-starting-style:translate-x-[2.5rem] data-[side=top]:inset-x-0 data-[side=top]:top-0 data-[side=top]:h-auto data-[side=top]:border-b data-[side=top]:data-ending-style:translate-y-[-2.5rem] data-[side=top]:data-starting-style:translate-y-[-2.5rem] data-[side=left]:sm:max-w-sm data-[side=right]:sm:max-w-sm",
className,
)}
{...props}
>
{children}
{showCloseButton && (
<SheetPrimitive.Close
data-slot="sheet-close"
render={<Button variant="ghost" className="absolute top-4 right-4" size="icon-sm" />}
>
<XIcon />
<span className="sr-only">Close</span>
</SheetPrimitive.Close>
)}
</SheetPrimitive.Popup>
</SheetPortal>
);
}
function SheetHeader({ className, ...props }: React.ComponentProps<"div">) {
return <div data-slot="sheet-header" className={cn("flex flex-col gap-1.5 p-4", className)} {...props} />;
}
function SheetFooter({ className, ...props }: React.ComponentProps<"div">) {
return <div data-slot="sheet-footer" className={cn("mt-auto flex flex-col gap-2 p-4", className)} {...props} />;
}
function SheetTitle({ className, ...props }: SheetPrimitive.Title.Props) {
return (
<SheetPrimitive.Title data-slot="sheet-title" className={cn("font-medium text-foreground", className)} {...props} />
);
}
function SheetDescription({ className, ...props }: SheetPrimitive.Description.Props) {
return (
<SheetPrimitive.Description
data-slot="sheet-description"
className={cn("text-sm text-muted-foreground", className)}
{...props}
/>
);
}
export { Sheet, SheetTrigger, SheetClose, SheetContent, SheetHeader, SheetFooter, SheetTitle, SheetDescription };

104
ui/litellm-dashboard/src/lib/http/api.test.ts Normal file
View file

@ -0,0 +1,104 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { fetchClient } from "./api";
import {
registerAuthHeaderNameGetter,
registerAuthTokenGetter,
registerBaseUrlGetter,
registerErrorHandler,
} from "./runtime";
const jsonResponse = (status: number, body: unknown): Response =>
new Response(JSON.stringify(body), { status, headers: { "Content-Type": "application/json" } });
const capturingFetch = (response: Response) => {
const requests: Request[] = [];
const fetch = vi.fn(async (request: Request) => {
requests.push(request);
return response;
});
return { fetch, requests };
};
describe("typed api client middleware", () => {
beforeEach(() => {
registerBaseUrlGetter(() => "");
registerAuthHeaderNameGetter(() => "Authorization");
registerErrorHandler(() => {});
registerAuthTokenGetter(() => null);
});
afterEach(() => {
vi.restoreAllMocks();
});
it("injects the bearer token under the registered auth header name", async () => {
registerAuthTokenGetter(() => "sk-test");
registerAuthHeaderNameGetter(() => "x-litellm-key");
const { fetch, requests } = capturingFetch(jsonResponse(200, { data: [] }));
await fetchClient.GET("/model_group/info", { fetch });
expect(requests[0].headers.get("x-litellm-key")).toBe("Bearer sk-test");
expect(requests[0].headers.get("Authorization")).toBeNull();
});
it("omits the auth header when no token is set", async () => {
const { fetch, requests } = capturingFetch(jsonResponse(200, { data: [] }));
await fetchClient.GET("/model_group/info", { fetch });
expect(requests[0].headers.get("Authorization")).toBeNull();
});
it("rebases the request onto the registered base url, preserving path and query", async () => {
registerBaseUrlGetter(() => "https://proxy.example.com/");
const { fetch, requests } = capturingFetch(jsonResponse(200, { data: [] }));
await fetchClient.GET("/model_group/info", { fetch, params: { query: { model_group: "gpt-4o" } } });
const url = new URL(requests[0].url);
expect(url.origin).toBe("https://proxy.example.com");
expect(url.pathname).toBe("/model_group/info");
expect(url.searchParams.get("model_group")).toBe("gpt-4o");
});
it("maps a non-2xx response to an ApiError carrying status and the derived message", async () => {
const { fetch } = capturingFetch(jsonResponse(403, { error: { message: "no access" } }));
await expect(fetchClient.GET("/model_group/info", { fetch })).rejects.toMatchObject({
name: "ApiError",
status: 403,
message: "no access",
});
});
it("returns the parsed body on a successful response", async () => {
const body = { data: [{ model_group: "gpt-4o" }] };
const { fetch } = capturingFetch(jsonResponse(200, body));
const { data, error } = await fetchClient.GET("/model_group/info", { fetch });
expect(error).toBeUndefined();
expect(data).toEqual(body);
});
it("reports the derived message to the registered error handler on a non-2xx response", async () => {
const onError = vi.fn();
registerErrorHandler(onError);
const { fetch } = capturingFetch(jsonResponse(401, { error: { message: "Authentication Error - Expired Key" } }));
await expect(fetchClient.GET("/model_group/info", { fetch })).rejects.toBeInstanceOf(Error);
expect(onError).toHaveBeenCalledWith("Authentication Error - Expired Key");
});
it("does not call the error handler on a successful response", async () => {
const onError = vi.fn();
registerErrorHandler(onError);
const { fetch } = capturingFetch(jsonResponse(200, { data: [] }));
await fetchClient.GET("/model_group/info", { fetch });
expect(onError).not.toHaveBeenCalled();
});
});

48
ui/litellm-dashboard/src/lib/http/api.ts Normal file
View file

@ -0,0 +1,48 @@
import createFetchClient, { type Middleware } from "openapi-fetch";
import type { paths } from "./schema";
import { ApiError, deriveErrorMessage } from "./client";
import { getAuthHeaderName, getAuthToken, getRequestBaseUrl, reportError } from "./runtime";
const rebaseUrl = (requestUrl: string, base: string): string => {
const { pathname, search } = new URL(requestUrl);
return `${base.replace(/\/+$/, "")}${pathname}${search}`;
};
const middleware: Middleware = {
onRequest({ request }) {
const base = getRequestBaseUrl();
const next = new Request(base ? rebaseUrl(request.url, base) : request.url, request);
const token = getAuthToken();
if (token) {
next.headers.set(getAuthHeaderName(), `Bearer ${token}`);
}
return next;
},
async onResponse({ response }) {
if (response.ok) return response;
const raw = await response.clone().text();
let body: unknown = raw;
let message: string;
try {
body = JSON.parse(raw);
message = deriveErrorMessage(body);
} catch {
message = raw || `HTTP ${response.status}`;
}
reportError(message);
throw new ApiError(message, response.status, body);
},
};
/**
* The typed, schema-bound HTTP client. Use it inside TanStack Query hooks
* (`fetchClient.GET("/path", { params })`) and for imperative calls; path
* params, query params, and request bodies are inferred from schema.d.ts.
*
* The creation-time base is the current origin so request URLs are absolute; the
* middleware rebases each call onto the runtime base when one is registered (a
* split-origin proxy or worker URL), injects the auth header, and maps non-2xx
* responses to ApiError so query functions can just read `.data`.
*/
export const fetchClient = createFetchClient<paths>({ baseUrl: globalThis.location?.origin ?? "" });
fetchClient.use(middleware);

22
ui/litellm-dashboard/src/lib/http/runtime.test.ts Normal file
View file

@ -0,0 +1,22 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { getAuthHeaderName, getRequestBaseUrl } from "./runtime";
describe("runtime request config defaults", () => {
afterEach(() => {
vi.unstubAllEnvs();
});
it("resolves the default base URL from NEXT_PUBLIC_BASE_URL before a getter is registered", () => {
vi.stubEnv("NEXT_PUBLIC_BASE_URL", "https://proxy.example.com/");
expect(getRequestBaseUrl()).toBe("https://proxy.example.com");
});
it("defaults the base URL to same-origin when NEXT_PUBLIC_BASE_URL is unset", () => {
vi.stubEnv("NEXT_PUBLIC_BASE_URL", "");
expect(getRequestBaseUrl()).toBe("");
});
it("defaults the auth header name to Authorization", () => {
expect(getAuthHeaderName()).toBe("Authorization");
});
});

46
ui/litellm-dashboard/src/lib/http/runtime.ts Normal file
View file

@ -0,0 +1,46 @@
import { resolveApiBase } from "./resolveApiBase";
/**
* Runtime request config the typed client reads on every call. The values are
* mutable at runtime (base URL can switch to a worker origin; the auth header
* name and token come from the logged-in session), and they are owned outside
* this module: networking.tsx registers the base URL / header-name / token
* getters and the error handler. The token getter reads the session cookie, the
* same source useAuthorized decodes, so the client's token and the gate that
* enables a query cannot diverge. Keeping the seam here (not importing from the
* component tree) lets api.ts stay in lib/http without a layering inversion.
*
* The base URL default resolves from NEXT_PUBLIC_BASE_URL so a request still
* hits the right origin if it fires before networking registers its fuller
* getter (which additionally folds in the server root path from the live UI
* config). The auth header name has no build-time source, so it defaults to
* "Authorization" until the session's JWT supplies a custom one.
*/
type Getter<T> = () => T;
let baseUrlGetter: Getter<string> = () => resolveApiBase({ explicitBase: process.env.NEXT_PUBLIC_BASE_URL });
let authHeaderNameGetter: Getter<string> = () => "Authorization";
let authTokenGetter: Getter<string | null> = () => null;
let errorHandler: (message: string) => void = () => {};
export const registerBaseUrlGetter = (getter: Getter<string>): void => {
baseUrlGetter = getter;
};
export const registerAuthHeaderNameGetter = (getter: Getter<string>): void => {
authHeaderNameGetter = getter;
};
export const registerAuthTokenGetter = (getter: Getter<string | null>): void => {
authTokenGetter = getter;
};
export const registerErrorHandler = (handler: (message: string) => void): void => {
errorHandler = handler;
};
export const getRequestBaseUrl = (): string => baseUrlGetter();
export const getAuthHeaderName = (): string => authHeaderNameGetter();
export const getAuthToken = (): string | null => authTokenGetter();
export const reportError = (message: string): void => errorHandler(message);

49
uv.lock generated
View file

@ -9,7 +9,7 @@ resolution-markers = [
]
[options]
exclude-newer = "2026-07-06T18:05:33.611729Z"
exclude-newer = "2026-07-08T17:10:15.525149Z"
exclude-newer-span = "P3D"
[manifest]
@ -2252,14 +2252,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/b0/03/84359833f7e1d49a883e92777637c592306030e30cee5e2b1e6476f95c88/greenlet-3.5.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:29ea813b2e1f45fa9649a17853b2b5465c4072fbcb072e5af6cd3a288216574a", size = 283502, upload-time = "2026-04-27T12:20:55.213Z" },
{ url = "https://files.pythonhosted.org/packages/25/ce/6f9f008266273aa14a2e011945797ac5802b97b8b40efe7afe1ee6c1afc9/greenlet-3.5.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:804a70b328e706b785c6ef16187051c394a63dd1a906d89be24b6ad77759f13f", size = 600508, upload-time = "2026-04-27T12:52:37.876Z" },
{ url = "https://files.pythonhosted.org/packages/e0/6d/b0f3272c2368ea2c1aa19a5ad70db0be8f8dff6e6d3d1eb82efa00cbcf19/greenlet-3.5.0-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:884f649de075b84739713d41dd4dfd41e2b910bfb769c4a3ea02ec1da52cd9bb", size = 613283, upload-time = "2026-04-27T12:59:37.957Z" },
{ url = "https://files.pythonhosted.org/packages/e5/ae/1db979ff6ae7958d80b288f63d5f6c30df96682700ea9fc340ce994d94a1/greenlet-3.5.0-cp310-cp310-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4d0eadc7e4d9ffb2af4247b606cae307be8e448911e5a0d0b16d72fc3d224cfd", size = 619894, upload-time = "2026-04-27T13:02:35.13Z" },
{ url = "https://files.pythonhosted.org/packages/ed/ac/0b509b6fb93551ce5a01612ee1acda7f7dda4bbb66c99aeb2ab403d205dc/greenlet-3.5.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b28037cb07768933c54d81bfe47a85f9f402f57d7d69743b991a713b63954eb", size = 613418, upload-time = "2026-04-27T12:25:23.852Z" },
{ url = "https://files.pythonhosted.org/packages/ce/94/b0590e3d1978f02419f30502341c40d72f77eb0a2198119fe27df47714ee/greenlet-3.5.0-cp310-cp310-manylinux_2_39_riscv64.whl", hash = "sha256:f8c30c2225f40dd76c50790f0eb3b5c7c18431efb299e2782083e1981feed243", size = 415681, upload-time = "2026-04-27T13:05:11.494Z" },
{ url = "https://files.pythonhosted.org/packages/03/03/2b2b680ec87aaa97998fb5b8d76658d4d3560386864f17efab33ba7c2e24/greenlet-3.5.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cda05425526240807408156b6960a17a79a0c760b813573b67027823be760977", size = 1572229, upload-time = "2026-04-27T12:53:23.509Z" },
{ url = "https://files.pythonhosted.org/packages/61/e4/42b259e7a19aff1a270a4bd82caf6353109ed6860c9454e18f37162b83ae/greenlet-3.5.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9c615f869163e14bb1ced20322d8038fb680b08236521ac3f30cd4c1288785a0", size = 1639886, upload-time = "2026-04-27T12:25:22.325Z" },
{ url = "https://files.pythonhosted.org/packages/6f/b4/733ca47b883b67c57f90d3ecb21055c9ec753597d10754ac201644061f9d/greenlet-3.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:ba8f0bdc2fae6ce915dfd0c16d2d00bca7e4247c1eae4416e06430e522137858", size = 237795, upload-time = "2026-04-27T12:21:40.118Z" },
{ url = "https://files.pythonhosted.org/packages/8b/0f/a91f143f356523ff682309732b175765a9bc2836fd7c081c2c67fedc1ad4/greenlet-3.5.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:8f1cc966c126639cd152fdaa52624d2655f492faa79e013fea161de3e6dda082", size = 284726, upload-time = "2026-04-27T12:20:51.402Z" },
{ url = "https://files.pythonhosted.org/packages/95/82/800646c7ffc5dbabd75ddd2f6b519bb898c0c9c969e5d0473bfe5d20bcce/greenlet-3.5.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:362624e6a8e5bca3b8233e45eef33903a100e9539a2b995c364d595dbc4018b3", size = 604264, upload-time = "2026-04-27T12:52:39.494Z" },
{ url = "https://files.pythonhosted.org/packages/ca/ac/354867c0bba812fc33b15bc55aedafedd0aee3c7dd91dfca22444157dc0c/greenlet-3.5.0-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5ecd83806b0f4c2f53b1018e0005cd82269ea01d42befc0368730028d850ed1c", size = 616099, upload-time = "2026-04-27T12:59:39.623Z" },
{ url = "https://files.pythonhosted.org/packages/c9/ab/192090c4a5b30df148c22bf4b8895457d739a7c7c5a7b9c41e5dd7f537f2/greenlet-3.5.0-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa94cb2288681e3a11645958f1871d48ee9211bd2f66628fdace505927d6e564", size = 623976, upload-time = "2026-04-27T13:02:37.363Z" },
{ url = "https://files.pythonhosted.org/packages/ff/b0/815bece7399e01cadb69014219eebd0042339875c59a59b0820a46ece356/greenlet-3.5.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0ff251e9a0279522e62f6176412869395a64ddf2b5c5f782ff609a8216a4e662", size = 615198, upload-time = "2026-04-27T12:25:25.928Z" },
{ url = "https://files.pythonhosted.org/packages/24/11/05eb2b9b188c6df7d68a89c99134d644a7af616a40b9808e8e6ced315d5d/greenlet-3.5.0-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:64d6ac45f7271f48e45f67c95b54ef73534c52ec041fcda8edf520c6d811f4bc", size = 418379, upload-time = "2026-04-27T13:05:12.755Z" },
{ url = "https://files.pythonhosted.org/packages/10/80/3b2c0a895d6698f6ddb31b07942ebfa982f3e30888bc5546a5b5990de8b2/greenlet-3.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d874e79afd41a96e11ff4c5d0bc90a80973e476fda1c2c64985667397df432b", size = 1574927, upload-time = "2026-04-27T12:53:25.81Z" },
{ url = "https://files.pythonhosted.org/packages/44/0e/f354af514a4c61454dbc68e44d47544a5a4d6317e30b77ddfa3a09f4c5f3/greenlet-3.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0ed006e4b86c59de7467eb2601cd1b77b5a7d657d1ee55e30fe30d76451edba4", size = 1642683, upload-time = "2026-04-27T12:25:23.9Z" },
{ url = "https://files.pythonhosted.org/packages/fa/6a/87f38255201e993a1915265ebb80cd7c2c78b04a45744995abbf6b259fd8/greenlet-3.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:703cb211b820dbffbbc55a16bfc6e4583a6e6e990f33a119d2cc8b83211119c8", size = 238115, upload-time = "2026-04-27T12:21:48.845Z" },
@ -2267,7 +2271,9 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/ef/32/f2ce6d4cac3e55bc6173f92dbe627e782e1850f89d986c3606feb63aafa7/greenlet-3.5.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:db2910d3c809444e0a20147361f343fe2798e106af8d9d8506f5305302655a9f", size = 286228, upload-time = "2026-04-27T12:20:34.421Z" },
{ url = "https://files.pythonhosted.org/packages/b7/aa/caed9e5adf742315fc7be2a84196373aab4816e540e38ba0d76cb7584d68/greenlet-3.5.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ec9ea74e7268ace7f9aab1b1a4e730193fc661b39a993cd91c606c32d4a3628", size = 601775, upload-time = "2026-04-27T12:52:41.045Z" },
{ url = "https://files.pythonhosted.org/packages/c7/af/90ae08497400a941595d12774447f752d3dfe0fbb012e35b76bc5c0ff37e/greenlet-3.5.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54d243512da35485fc7a6bf3c178fdda6327a9d6506fcdd62b1abd1e41b2927b", size = 614436, upload-time = "2026-04-27T12:59:41.595Z" },
{ url = "https://files.pythonhosted.org/packages/3f/e9/4eeadf8cb3403ac274245ba75f07844abc7fa5f6787583fc9156ba741e0f/greenlet-3.5.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:41353ec2ecedf7aa8f682753a41919f8718031a6edac46b8d3dc7ed9e1ceb136", size = 620610, upload-time = "2026-04-27T13:02:39.194Z" },
{ url = "https://files.pythonhosted.org/packages/2b/e0/2e13df68f367e2f9960616927d60857dd7e56aaadd59a47c644216b2f920/greenlet-3.5.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d280a7f5c331622c69f97eb167f33577ff2d1df282c41cd15907fc0a3ca198c", size = 611388, upload-time = "2026-04-27T12:25:28.008Z" },
{ url = "https://files.pythonhosted.org/packages/ee/ef/f913b3c0eb7d26d86a2401c5e1546c9d46b657efee724b06f6f4ac5d8824/greenlet-3.5.0-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:58c1c374fe2b3d852f9b6b11a7dff4c85404e51b9a596fd9e89cf904eb09866d", size = 422775, upload-time = "2026-04-27T13:05:14.261Z" },
{ url = "https://files.pythonhosted.org/packages/82/f7/393c64055132ac0d488ef6be549253b7e6274194863967ddc0bc8f5b87b8/greenlet-3.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1eb67d5adefb5bd2e182d42678a328979a209e4e82eb93575708185d31d1f588", size = 1570768, upload-time = "2026-04-27T12:53:28.099Z" },
{ url = "https://files.pythonhosted.org/packages/b8/4b/eaf7735253522cf56d1b74d672a58f54fc114702ceaf05def59aae72f6e1/greenlet-3.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2628d6c86f6cb0cb45e0c3c54058bbec559f57eaae699447748cb3928150577e", size = 1635983, upload-time = "2026-04-27T12:25:26.903Z" },
{ url = "https://files.pythonhosted.org/packages/4c/fe/4fb3a0805bd5165da5ebf858da7cc01cce8061674106d2cf5bdab32cbfde/greenlet-3.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:d4d9f0624c775f2dfc56ba54d515a8c771044346852a918b405914f6b19d7fd8", size = 238840, upload-time = "2026-04-27T12:23:54.806Z" },
@ -2275,7 +2281,9 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/0c/58/fc576f99037ce19c5aa16628e4c3226b6d1419f72a62c79f5f40576e6eb3/greenlet-3.5.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:5a5ed18de6a0f6cc7087f1563f6bd93fc7df1c19165ca01e9bde5a5dc281d106", size = 285066, upload-time = "2026-04-27T12:23:05.033Z" },
{ url = "https://files.pythonhosted.org/packages/4a/ba/b28ddbe6bfad6a8ac196ef0e8cff37bc65b79735995b9e410923fffeeb70/greenlet-3.5.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a717fbc46d8a354fa675f7c1e813485b6ba3885f9bef0cd56e5ba27d758ff5b", size = 604414, upload-time = "2026-04-27T12:52:42.358Z" },
{ url = "https://files.pythonhosted.org/packages/09/06/4b69f8f0b67603a8be2790e55107a190b376f2627fe0eaf5695d85ffb3cd/greenlet-3.5.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ddc090c5c1792b10246a78e8c2163ebbe04cf877f9d785c230a7b27b39ad038e", size = 617349, upload-time = "2026-04-27T12:59:43.32Z" },
{ url = "https://files.pythonhosted.org/packages/6a/15/a643b4ecd09969e30b8a150d5919960caae0abe4f5af75ab040b1ab85e78/greenlet-3.5.0-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4964101b8585c144cbda5532b1aa644255126c08a265dae90c16e7a0e63aaa9d", size = 623234, upload-time = "2026-04-27T13:02:40.611Z" },
{ url = "https://files.pythonhosted.org/packages/8a/17/a3918541fd0ddefe024a69de6d16aa7b46d36ac19562adaa63c7fa180eff/greenlet-3.5.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2094acd54b272cb6eae8c03dd87b3fa1820a4cef18d6889c378d503500a1dc13", size = 613927, upload-time = "2026-04-27T12:25:30.28Z" },
{ url = "https://files.pythonhosted.org/packages/77/18/3b13d5ef1275b0ffaf933b05efa21408ac4ca95823c7411d79682e4fdcff/greenlet-3.5.0-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:7022615368890680e67b9965d33f5773aade330d5343bbe25560135aaa849eae", size = 425243, upload-time = "2026-04-27T13:05:15.689Z" },
{ url = "https://files.pythonhosted.org/packages/ee/e1/bd0af6213c7dd33175d8a462d4c1fe1175124ebed4855bc1475a5b5242c2/greenlet-3.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5e05ba267789ea87b5a155cf0e810b1ab88bf18e9e8740813945ceb8ee4350ba", size = 1570893, upload-time = "2026-04-27T12:53:29.483Z" },
{ url = "https://files.pythonhosted.org/packages/9b/2a/0789702f864f5382cb476b93d7a9c823c10472658102ccd65f415747d2e2/greenlet-3.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0ecec963079cd58cbd14723582384f11f166fd58883c15dcbfb342e0bc9b5846", size = 1636060, upload-time = "2026-04-27T12:25:28.845Z" },
{ url = "https://files.pythonhosted.org/packages/b2/8f/22bf9df92bbff0eb07842b60f7e63bf7675a9742df628437a9f02d09137f/greenlet-3.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:728d9667d8f2f586644b748dbd9bb67e50d6a9381767d1357714ea6825bb3bf5", size = 238740, upload-time = "2026-04-27T12:24:01.341Z" },
@ -3453,6 +3461,10 @@ dev = [
{ name = "types-setuptools" },
{ name = "vcrpy" },
]
e2e-dev = [
{ name = "playwright" },
{ name = "websockets" },
]
healthcheck = [
{ name = "httpx" },
{ name = "pyyaml" },
@ -3622,6 +3634,10 @@ dev = [
{ name = "types-setuptools", specifier = "==75.8.0.20250225" },
{ name = "vcrpy", specifier = "==8.2.1" },
]
e2e-dev = [
{ name = "playwright", specifier = "==1.61.0" },
{ name = "websockets", specifier = ">=15.0.1,<16.0" },
]
healthcheck = [
{ name = "httpx", specifier = "==0.28.1" },
{ name = "pyyaml", specifier = "==6.0.3" },
@ -5343,6 +5359,25 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/75/a6/a0a304dc33b49145b21f4808d763822111e67d1c3a32b524a1baf947b6e1/platformdirs-4.9.6-py3-none-any.whl", hash = "sha256:e61adb1d5e5cb3441b4b7710bea7e4c12250ca49439228cc1021c00dcfac0917", size = 21348, upload-time = "2026-04-09T00:04:09.463Z" },
]
[[package]]
name = "playwright"
version = "1.61.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "greenlet" },
{ name = "pyee" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/44/ee/31e4e0db36588b817a10b299a0285082545fde7d36543c2abe498bb3d61a/playwright-1.61.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:ff138c3a604f69911e9d42fd036e55c2a171e5616edf04c1e7f60a2a285540b0", size = 43421877, upload-time = "2026-06-29T10:32:48.428Z" },
{ url = "https://files.pythonhosted.org/packages/42/35/71395dd3ecc798965be4a3ef8c443217d4abca168e7cb34536304f9489e6/playwright-1.61.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:009588c2a7e499bc5a8b425b61fa65490968bbda9cd69e0cf2cff10f8304659a", size = 42205016, upload-time = "2026-06-29T10:32:52.104Z" },
{ url = "https://files.pythonhosted.org/packages/f4/44/323164cf5cd1647bdefce76ffce27651aadb959d089b48f53ea40918276e/playwright-1.61.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:9f7de4536088d12037c13a52b7ea34b59270b78926bb56935070597ffac6b1af", size = 43421884, upload-time = "2026-06-29T10:32:55.773Z" },
{ url = "https://files.pythonhosted.org/packages/ab/f8/a35bf179e4ba2522c1893635094a64e407572547bd61528820fc0abc87fe/playwright-1.61.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:54f3b39f6eab832e33458c1dd7da0b5682aedab3b09ae731b5c59fa12fd2024e", size = 47421381, upload-time = "2026-06-29T10:32:59.903Z" },
{ url = "https://files.pythonhosted.org/packages/b7/eb/e3f922348ec17c315f98c463f72faa1181a1c3de0bfe31a8d2edf6561723/playwright-1.61.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:93454322ade8c11d5d6c211bfd91bdfb9ffb4810e3e026371bcbc4bec1b7ee4c", size = 47120545, upload-time = "2026-06-29T10:33:03.574Z" },
{ url = "https://files.pythonhosted.org/packages/c2/a6/5be4e52b40a9c0c8a073e7c5b0785c05cf5a9ea8f8a7b5b260e32d970342/playwright-1.61.0-py3-none-win32.whl", hash = "sha256:372d55a6f1248fa1dd47599686980cb8fb5bbe6fcda59eab793eb657c11d8a9b", size = 37844841, upload-time = "2026-06-29T10:33:07.361Z" },
{ url = "https://files.pythonhosted.org/packages/6c/fd/2b78036e5fbe9d5f5645bbe08a1eac7160c51243c0093963edbcf67c35d9/playwright-1.61.0-py3-none-win_amd64.whl", hash = "sha256:35c6cc4589a5d00964a59d7b3e59641e0aac0c02f15479a7af77d20f6bc79597", size = 37844846, upload-time = "2026-06-29T10:33:10.637Z" },
{ url = "https://files.pythonhosted.org/packages/27/0d/1b0f3c4ee4eb0514bc805b5c2f9a223e5b6de4f11a926f5235d51d0fc81b/playwright-1.61.0-py3-none-win_arm64.whl", hash = "sha256:e9fcbffcf557a8620fdedd92491eb59a32d18e23d6f3b4f6214b952be324fe51", size = 33955127, upload-time = "2026-06-29T10:33:14.008Z" },
]
[[package]]
name = "pluggy"
version = "1.6.0"
@ -5915,6 +5950,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/77/c1/6e422f34e569cf8e18df68d1939c81c099d2b61e4f7d9621c8a77560799c/pydantic_settings-2.14.2-py3-none-any.whl", hash = "sha256:a20c97b37910b6550d5ea50fbcc2d4187defe58cd57070b73863d069419c9440", size = 61715, upload-time = "2026-06-19T13:44:55.02Z" },
]
[[package]]
name = "pyee"
version = "13.0.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/a0/c4/b4d4827c93ef43c01f599ef31453ccc1c132b353284fc6c87d535c233129/pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228", size = 15659, upload-time = "2026-02-14T21:12:26.263Z" },
]
[[package]]
name = "pyflakes"
version = "3.4.0"