diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index c2fc3453261..0b5b0e9b976 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -48,7 +48,7 @@ jobs: - name: Install dependencies run: | - uv sync --frozen --group proxy-dev + uv sync --frozen --group proxy-dev --group e2e-dev # basedpyright resolves Prisma's generated client (litellm/proxy/schema.prisma) # only after `prisma generate` writes prisma/client.py et al. Without this the @@ -107,6 +107,16 @@ jobs: run: | (uv run --no-sync basedpyright --outputjson || true) | uv run --no-sync python scripts/type_check_gate.py --base "$BASE_SHA" + - name: Check tests/e2e basedpyright (zero errors) + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + if git diff --name-only --diff-filter=ACMRD "$BASE_SHA"...HEAD -- 'tests/e2e/**/*.py' | grep -q .; then + uv run --no-sync basedpyright tests/e2e + else + echo "No changed tests/e2e Python files; skipping." + fi + - name: Check for circular imports run: | cd litellm diff --git a/Makefile b/Makefile index f2753b09ff5..965a3254616 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \ test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \ info lint lint-dev lint-checks format \ - lint-basedpyright lint-basedpyright-budget-update lint-type-discipline lint-type-discipline-budget-update \ + lint-basedpyright lint-e2e-basedpyright lint-basedpyright-budget-update lint-type-discipline lint-type-discipline-budget-update \ lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-gate \ install-dev install-proxy-dev install-test-deps install-hooks \ install-helm-unittest check-circular-imports check-import-safety pre-commit \ @@ -27,6 +27,7 @@ help: @echo " make lint - Run all linting (Ruff, basedpyright, format check, circular imports, import safety)" @echo " make lint-ruff - Run Ruff linting only" @echo " make lint-basedpyright - Run basedpyright strict, gated by per-rule error counts" + @echo " make lint-e2e-basedpyright - Run basedpyright over tests/e2e (zero errors allowed)" @echo " make lint-basedpyright-budget-update - Ratchet basedpyright limits down by what this branch fixed" @echo " make lint-format - Check ruff format formatting (matches CI)" @echo " make lint-ruff-budget - Gate the codebase total of each strict ruff rule against its limit" @@ -54,6 +55,7 @@ UV := uv UV_RUN := $(UV) run --no-sync LINT_DEP_INSTALL ?= install-dev +LINT_E2E_DEP_INSTALL ?= lint-install LINT_DEP_BASE ?= lint-fetch-base LINT_JOBS := $(shell sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 4) LINT_OUTPUT_SYNC := $(if $(filter output-sync,$(.FEATURES)),--output-sync=target,) @@ -111,7 +113,7 @@ lint-fetch-base: # CI's). --inexact tops up the venv instead of pruning the proxy extras gen:api and the # running proxy need. lint-install: - $(UV) sync --inexact --frozen --group proxy-dev + $(UV) sync --inexact --frozen --group proxy-dev --group e2e-dev $(UV_RUN) python scripts/prisma_generate_if_needed.py # Diff-scoped format check, identical to test-linting.yml's "Check ruff format" step: @@ -164,6 +166,9 @@ lint-ruff-FULL-dev: install-dev lint-basedpyright: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE) ($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --base origin/litellm_internal_staging +lint-e2e-basedpyright: $(LINT_E2E_DEP_INSTALL) + $(UV_RUN) basedpyright tests/e2e + # Type-discipline budget (mutable collections / casts / type guards / kwargs / # unexplained suppressions), the test-linting.yml step `make lint` used to omit. lint-type-discipline: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE) @@ -208,9 +213,9 @@ check-import-safety: $(LINT_DEP_INSTALL) # base fetch) runs once up front; the checks themselves are independent, so a sub-make # fans them out with -j and the fast ones finish under basedpyright's shadow. lint: lint-install lint-fetch-base - $(MAKE) -j $(LINT_JOBS) $(LINT_OUTPUT_SYNC) LINT_DEP_INSTALL= LINT_DEP_BASE= lint-checks + $(MAKE) -j $(LINT_JOBS) $(LINT_OUTPUT_SYNC) LINT_DEP_INSTALL= LINT_E2E_DEP_INSTALL= LINT_DEP_BASE= lint-checks -lint-checks: lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-basedpyright check-circular-imports check-import-safety +lint-checks: lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-basedpyright lint-e2e-basedpyright check-circular-imports check-import-safety # Faster linting for local development (only checks changed code) lint-dev: lint-format-changed check-circular-imports check-import-safety diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index cb3427bed4d..edfb3536ad3 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -57,7 +57,7 @@ "limit": 5900 }, "reportMissingTypeArgument": { - "limit": 15918 + "limit": 15903 }, "reportMissingTypeStubs": { "limit": 41 @@ -105,13 +105,13 @@ "limit": 113 }, "reportUnknownMemberType": { - "limit": 40541 + "limit": 40539 }, "reportUnknownParameterType": { - "limit": 20418 + "limit": 20403 }, "reportUnknownVariableType": { - "limit": 32151 + "limit": 32141 }, "reportUnnecessaryCast": { "limit": 177 @@ -123,7 +123,7 @@ "limit": 7 }, "reportUnnecessaryIsInstance": { - "limit": 1212 + "limit": 1209 }, "reportUntypedBaseClass": { "limit": 165 diff --git a/litellm/litellm_core_utils/fallback_generalizations.py b/litellm/litellm_core_utils/fallback_generalizations.py index 8b5a797cbd7..410bb9623fe 100644 --- a/litellm/litellm_core_utils/fallback_generalizations.py +++ b/litellm/litellm_core_utils/fallback_generalizations.py @@ -3,55 +3,69 @@ Declarative fallback generalizations for unknown / newly-released models. The ``fallback_generalizations`` block in ``model_prices_and_context_window.json`` holds an ordered list of rules. Each rule pairs a single case-insensitive regex -with the metadata to apply when a model name has no exact entry in the cost map. -The metadata is a partial cost-map entry: ``litellm_provider`` drives provider -routing, and the remaining fields (``mode``, ``supports_*``, context window, -pricing, ...) drive ``get_model_info`` / ``supports_*``. +with a ``model_info`` dict, and the structure of ``model_info`` decides which of +two kinds the rule is. -Precedence: rules are evaluated in file order and the first match wins. Callers -with extra constraints (model-info resolution checks the provider) use -``match_all_fallback_generalizations`` to skip inapplicable earlier rules instead -of discarding the model name. Rules are consulted only after exact and -case-insensitive lookups miss, so an exact entry always takes precedence over a -rule. +A ROUTING rule carries exactly one ``model_info`` key, ``litellm_provider``. It is +consumed only by ``get_llm_provider`` bare-id inference: the first routing rule +whose regex matches decides the provider. Routing rules never contribute to model +info. + +A CAPABILITY rule carries any ``model_info`` keys except ``litellm_provider`` +(``mode``, ``supports_*``, context window, pricing, ...). It is consumed by +``get_model_info`` fallback resolution: the ``model_info`` of ALL capability rules +whose regex matches is unioned in file order, with later rules overriding earlier +ones on key conflicts, and the caller backfills ``litellm_provider`` with the +provider it requested. If no capability rule matches, model-info resolution misses +as if no rules existed. + +LEGACY-SCHEMA SHIM (temporary, until the new-schema JSON reaches main): released +proxies fetch this JSON remotely from main, whose block still ships the old schema +where a rule mixes ``litellm_provider`` with capability keys and may inherit a +parent's ``model_info`` via ``extends``. Such a legacy rule is tolerated rather +than skipped: ``extends`` is resolved once at install time (single level, against +raw parents), and the resolved rule acts as BOTH kinds, a routing rule (its +``litellm_provider`` participates in first-hit inference) and a capability rule +(its full ``model_info``, provider included, participates in the union). New-schema +rules never mix the two and never use ``extends``. A rule whose +``litellm_provider`` is not a string is invalid and is warned about and skipped +(a warning rather than a crash, for the same remote-fetch reason). + +Rules are only consulted after exact and case-insensitive lookups miss, so an +exact cost-map entry always takes precedence over any rule. Patterns are matched case-insensitively with ``re.search`` and are not implicitly -anchored: a rule must include ``^`` and ``$`` (as the shipped rules do) to bind to -the whole model name, otherwise it matches as a substring. Keeping anchoring in the -regex makes the rule the single, self-contained source of truth for what it matches. - -A rule may set ``extends`` to the ``name`` of another rule to inherit that rule's -``model_info``; the rule's own ``model_info`` overrides the inherited keys, so a -narrow rule (for example a version-gated capability flag) carries only its delta -instead of duplicating the parent's pricing block. Inheritance is resolved once, -at install time, against each rule's raw (unresolved) ``model_info``; it is a -single level (a parent that itself extends is not chained). +anchored: a rule must include ``^`` and ``$`` to bind to the whole model name, +otherwise it matches as a substring. Keeping anchoring in the regex makes the rule +the single, self-contained source of truth for what it matches. Any other keys on a rule (for example a free-text ``description`` documenting what the regex matches) are ignored by the engine and exist only for the reader. -The compiled-regex list is built once and cached. ``match_fallback_generalization`` -is O(number of rules); callers must only invoke it on a cache miss. +Rules are compiled and classified once, at install time. The match functions are +O(number of rules); callers must only invoke them on a cache miss. """ import re -from typing import Optional +from dataclasses import dataclass +from typing import Optional, Union from litellm._logging import verbose_logger NAME_FIELD = "name" PATTERN_FIELD = "pattern" MODEL_INFO_FIELD = "model_info" -EXTENDS_FIELD = "extends" +PROVIDER_KEY = "litellm_provider" +LEGACY_EXTENDS_FIELD = "extends" -def _resolve_extends(rules: list) -> list: - """Expand ``extends`` inheritance so each rule's ``model_info`` is self-contained. +def _resolve_legacy_extends(rules: list) -> list: + """Expand legacy ``extends`` inheritance so each rule's ``model_info`` is self-contained. - A rule with ``extends: `` is rewritten with ``model_info`` set to the parent's - ``model_info`` overlaid by its own. Resolution is single-level and uses each rule's - raw ``model_info`` as the parent source. Non-dict rules and dangling parents are - passed through unchanged. + Compatibility shim for the old remote schema: single level, resolved against each + parent's raw ``model_info``, with the child's own keys winning on conflict. Non-dict + rules and dangling parents pass through unchanged; new-schema rules carry no + ``extends`` and are untouched. """ base_by_name = { rule[NAME_FIELD]: rule[MODEL_INFO_FIELD] @@ -61,93 +75,138 @@ def _resolve_extends(rules: list) -> list: and isinstance(rule.get(MODEL_INFO_FIELD), dict) } - def resolved(rule: dict) -> dict: - parent_name = rule.get(EXTENDS_FIELD) + def resolved(rule: object) -> object: + if not isinstance(rule, dict): + return rule + parent_name = rule.get(LEGACY_EXTENDS_FIELD) own_info = rule.get(MODEL_INFO_FIELD) parent_info = base_by_name.get(parent_name) if isinstance(parent_name, str) else None if parent_info is None or not isinstance(own_info, dict): return rule return {**rule, MODEL_INFO_FIELD: {**parent_info, **own_info}} - return [resolved(rule) if isinstance(rule, dict) else rule for rule in rules] + return [resolved(rule) for rule in rules] + + +@dataclass(frozen=True, slots=True) +class _RoutingRule: + pattern: re.Pattern + provider: str + + +@dataclass(frozen=True, slots=True) +class _CapabilityRule: + pattern: re.Pattern + model_info: dict + + +_CompiledRule = Union[_RoutingRule, _CapabilityRule] + + +def _compile_rule(rule: object) -> tuple[_CompiledRule, ...]: + if not isinstance(rule, dict): + return () + pattern = rule.get(PATTERN_FIELD) + model_info = rule.get(MODEL_INFO_FIELD) + if not isinstance(pattern, str) or not isinstance(model_info, dict): + verbose_logger.warning( + "LiteLLM: skipping malformed fallback generalization rule %s (needs string '%s' and dict '%s').", + rule.get(NAME_FIELD, pattern), + PATTERN_FIELD, + MODEL_INFO_FIELD, + ) + return () + try: + compiled = re.compile(pattern, re.IGNORECASE) + except re.error as e: + verbose_logger.warning( + "LiteLLM: skipping fallback generalization rule with invalid regex %r: %s", + pattern, + e, + ) + return () + if PROVIDER_KEY not in model_info: + return (_CapabilityRule(pattern=compiled, model_info=model_info),) + provider = model_info[PROVIDER_KEY] + if not isinstance(provider, str): + verbose_logger.warning( + "LiteLLM: skipping invalid fallback generalization rule %s: '%s' in '%s' must be a string.", + rule.get(NAME_FIELD, pattern), + PROVIDER_KEY, + MODEL_INFO_FIELD, + ) + return () + if len(model_info) == 1: + return (_RoutingRule(pattern=compiled, provider=provider),) + return ( + _RoutingRule(pattern=compiled, provider=provider), + _CapabilityRule(pattern=compiled, model_info=model_info), + ) class _FallbackGeneralizations: - """Holds the active rule list and its lazily-compiled regex cache.""" + """Holds the raw rule list and its install-time-compiled routing and capability rules.""" def __init__(self) -> None: - self.rules: list[dict] = [] - self._compiled: Optional[list[tuple[re.Pattern, dict]]] = None + self.rules: list = [] + self.routing_rules: tuple = () + self.capability_rules: tuple = () - def set_rules(self, rules: Optional[list[dict]]) -> None: - self.rules = rules if isinstance(rules, list) else [] - self._compiled = None + def set_rules(self, rules: Optional[list]) -> None: + installed = rules if isinstance(rules, list) else [] + compiled = tuple(kind for rule in _resolve_legacy_extends(installed) for kind in _compile_rule(rule)) + self.rules = installed + self.routing_rules = tuple(rule for rule in compiled if isinstance(rule, _RoutingRule)) + self.capability_rules = tuple(rule for rule in compiled if isinstance(rule, _CapabilityRule)) - def _compile(self) -> list[tuple[re.Pattern, dict]]: - compiled: list[tuple[re.Pattern, dict]] = [] - for rule in self.rules: - if not isinstance(rule, dict): - continue - pattern = rule.get(PATTERN_FIELD) - model_info = rule.get(MODEL_INFO_FIELD) - if not isinstance(pattern, str) or not isinstance(model_info, dict): - verbose_logger.warning( - "LiteLLM: skipping malformed fallback generalization rule %s (needs string '%s' and dict '%s').", - rule.get("name", pattern), - PATTERN_FIELD, - MODEL_INFO_FIELD, - ) - continue - try: - compiled.append((re.compile(pattern, re.IGNORECASE), model_info)) - except re.error as e: - verbose_logger.warning( - "LiteLLM: skipping fallback generalization rule with invalid regex %r: %s", - pattern, - e, - ) - return compiled - - def matches(self, model: str) -> list[dict]: + def match_routing(self, model: str) -> Optional[str]: if not model: - return [] - if self._compiled is None: - self._compiled = self._compile() - return [dict(model_info) for pattern, model_info in self._compiled if pattern.search(model) is not None] + return None + return next( + (rule.provider for rule in self.routing_rules if rule.pattern.search(model) is not None), + None, + ) - def match(self, model: str) -> Optional[dict]: - return next(iter(self.matches(model)), None) + def match_capabilities(self, model: str) -> Optional[dict]: + if not model: + return None + matched = tuple(rule.model_info for rule in self.capability_rules if rule.pattern.search(model) is not None) + if not matched: + return None + return {key: value for model_info in matched for key, value in model_info.items()} _registry = _FallbackGeneralizations() -def set_fallback_generalizations(rules: Optional[list[dict]]) -> None: - """Install the active rule list and invalidate the compiled-regex cache. +def set_fallback_generalizations(rules: Optional[list]) -> None: + """Install the active rule list, compiling and classifying each rule. - ``extends`` inheritance is resolved here, once, before the rules are stored. - Called once when the model cost map is loaded (and again on any reload). + Legacy ``extends`` inheritance is resolved here, once, before classification; + a legacy rule mixing ``litellm_provider`` with capability keys installs as both + kinds. Malformed and invalid-regex rules are warned about and skipped. Called + once when the model cost map is loaded (and again on any reload). """ - _registry.set_rules(_resolve_extends(rules) if isinstance(rules, list) else rules) + _registry.set_rules(rules) -def get_fallback_generalization_rules() -> list[dict]: +def get_fallback_generalization_rules() -> list: """Return the raw rule list (read-only view for callers/tests).""" return _registry.rules -def match_fallback_generalization(model: str) -> Optional[dict]: - """Return the ``model_info`` of the first rule whose regex matches ``model``. +def match_routing_generalization(model: str) -> Optional[str]: + """Return the provider of the first routing rule whose regex matches ``model``. O(number of rules). Only call this once exact lookups have missed. """ - return _registry.match(model) + return _registry.match_routing(model) -def match_all_fallback_generalizations(model: str) -> list[dict]: - """Return the ``model_info`` of every rule whose regex matches ``model``, in rule order. +def match_capability_generalizations(model: str) -> Optional[dict]: + """Return the union of the ``model_info`` of every capability rule matching ``model``. - Lets a caller with extra constraints (e.g. a provider match) skip an - inapplicable earlier rule instead of discarding the whole candidate. + Later rules override earlier ones on key conflicts. Returns ``None`` when no + capability rule matches. O(number of rules); only call once exact lookups have missed. """ - return _registry.matches(model) + return _registry.match_capabilities(model) diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index fa8ffe1ba1d..487a7b7e25f 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -4,7 +4,7 @@ from urllib.parse import urlparse import litellm from litellm.constants import REPLICATE_MODEL_NAME_WITH_ID_LENGTH from litellm.litellm_core_utils.fallback_generalizations import ( - match_fallback_generalization, + match_routing_generalization, ) from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.secret_managers.main import get_secret, get_secret_str @@ -474,12 +474,10 @@ def get_llm_provider( custom_llm_provider = "sap" # Last resort for an otherwise-unknown model: a declarative - # fallback-generalization rule (e.g. routes future claude-* to anthropic). + # fallback-generalization routing rule (e.g. routes future claude-* to anthropic). # Exact provider matches above always win; this only runs on a miss. if not custom_llm_provider: - generalization = match_fallback_generalization(model) - if generalization is not None: - custom_llm_provider = generalization.get("litellm_provider") or None + custom_llm_provider = match_routing_generalization(model) if not custom_llm_provider: if litellm.suppress_debug_info is False: diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b42ba23fa78..b4ae842c4e6 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1359,6 +1359,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1393,6 +1394,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1427,6 +1429,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1461,6 +1464,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1661,6 +1665,7 @@ "jp.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1743,6 +1748,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1777,6 +1783,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1811,6 +1818,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1845,6 +1853,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1879,6 +1888,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1913,6 +1923,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -45039,31 +45050,26 @@ "fallback_generalizations": { "rules": [ { - "name": "bedrock-anthropic-claude-mid-conversation-system", - "pattern": "anthropic\\.claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", - "description": "Bedrock Invoke ids for Claude 4.8 or higher: anthropic.claude- with minor 4.8 through 4.99, any 5.x or later major-minor, or a bare 5+ major, which also admits new families such as fable. These models accept mid-conversation role system messages in place (verified live on Opus 4.8, Sonnet 5 and Fable 5), so unmapped future Bedrock Claudes keep the cache-preserving in-place handling instead of the hoist-all default. Listed first so bare-id provider inference, which takes the first pattern hit, resolves these Bedrock ids to bedrock; model-info resolution skips provider-mismatched rules either way.", - "extends": "anthropic-claude", + "name": "bedrock-claude-ids", + "pattern": "anthropic\\.claude-", + "description": "Any Bedrock-syntax Claude id: the dotted anthropic.claude- segment appears in bare (anthropic.claude-...), region-prefixed (us./eu./au./jp./apac.) and global.-prefixed ids, for every version. Routes these to bedrock before the bare-id Anthropic rule is consulted.", "model_info": { - "litellm_provider": "bedrock", - "supports_adaptive_thinking": true, - "supports_mid_conversation_system": true + "litellm_provider": "bedrock" } }, { - "name": "anthropic-claude-adaptive-thinking", - "pattern": "(?:opus|sonnet|haiku)[-._](?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d{1,})[-._]\\d{1,2}(?!\\d))", - "description": "Claude opus/sonnet/haiku at version 4.6 or higher: 4.6 through 4.99, then any 5.x, 6.x or later major. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new families with no code change.", - "extends": "anthropic-claude", + "name": "anthropic-claude-ids", + "pattern": "^claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?$", + "description": "A bare Claude family-major id with an optional minor and an optional 8-digit date suffix, anchored to the whole name, so claude-newfamily-5 routes like claude-newfamily-5-1 does. Routes an unmapped Claude id that carries no provider namespace to the Anthropic API.", "model_info": { - "supports_adaptive_thinking": true + "litellm_provider": "anthropic" } }, { - "name": "anthropic-claude", - "pattern": "^claude-[a-z]+-\\d+[-.]\\d+(?:-\\d{8})?$", - "description": "Any Claude family-major-minor id, optionally with an 8-digit date suffix, anchored to the whole name. Version-neutral fallback that gives an unmapped Claude provider routing and baseline capabilities; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.", + "name": "claude-family-baseline", + "pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?", + "description": "Any Claude family-major id with an optional minor and an optional 8-digit date suffix, under any provider namespace (bare, bedrock-dotted, vertex, databricks, ...), so bare majors like claude-newfamily-5 get the same baseline as claude-newfamily-5-1. Carries the model-family facts every Claude shares; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.", "model_info": { - "litellm_provider": "anthropic", "mode": "chat", "max_input_tokens": 200000, "max_output_tokens": 64000, @@ -45079,6 +45085,22 @@ "supports_pdf_input": true, "supports_system_messages": true } + }, + { + "name": "claude-adaptive-thinking", + "pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", + "description": "Claude at version 4.6 or higher, in any id shape that contains claude--: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new versions and new families with no code change.", + "model_info": { + "supports_adaptive_thinking": true + } + }, + { + "name": "claude-mid-conversation-system", + "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", + "description": "Claude at version 4.8 or higher, in any id shape that contains claude--: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.", + "model_info": { + "supports_mid_conversation_system": true + } } ] } diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index 9689a7cb047..ebceb320906 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -820,6 +820,22 @@ class _PersistedDcrCredentials(BaseModel): client_id: Optional[str] = None client_secret: Optional[str] = None token_endpoint_auth_method: Optional[str] = None + redirect_uris: Optional[list[str]] = None + + +def _redirect_uri_not_registered(credentials: _PersistedDcrCredentials, current_redirect_uri: str) -> bool: + """Whether a persisted DCR client is positively known NOT to cover the current callback. + + A DCR client is bound to the redirect_uris it was registered with; if the proxy's + resolved public origin has since changed, every authorize built for it will be + rejected by the IdP. Clients persisted before ``redirect_uris`` was recorded (and + admin-configured clients, which never get a recording) return False so they are + grandfathered rather than re-registered, because re-minting a client_id orphans + every user's refresh tokens for that server.""" + recorded = credentials.redirect_uris + if not recorded: + return False + return current_redirect_uri not in recorded def _get_persisted_dcr_credentials(credentials: object) -> Optional[_PersistedDcrCredentials]: @@ -886,11 +902,23 @@ async def _get_persisted_mcp_server_with_dcr_client_id( return persisted_mcp_server, credentials -async def _reuse_persisted_dcr_client_if_available(mcp_server: MCPServer) -> bool: +async def _reuse_persisted_dcr_client_if_available( + mcp_server: MCPServer, current_redirect_uri: Optional[str] = None +) -> bool: persisted = await _get_persisted_mcp_server_with_dcr_client_id(mcp_server) if persisted is None: return False persisted_mcp_server, credentials = persisted + if current_redirect_uri is not None and _redirect_uri_not_registered(credentials, current_redirect_uri): + verbose_logger.debug( + "register_client_with_server: not reusing persisted DCR client for server_id=%s; its registered " + "redirect_uris=%s do not include the current callback %s. The operator-facing warning for this " + "re-registration event is emitted once by _persisted_dcr_redirect_uri_is_stale.", + mcp_server.server_id, + credentials.redirect_uris, + current_redirect_uri, + ) + return False if not _apply_persisted_dcr_credentials(mcp_server, credentials): return False @@ -909,11 +937,36 @@ async def _reuse_persisted_dcr_client_if_available(mcp_server: MCPServer) -> boo return bool(mcp_server.client_id) +async def _persisted_dcr_redirect_uri_is_stale(mcp_server: MCPServer, current_redirect_uri: str) -> bool: + """Whether the server's persisted DCR client is bound to redirect_uris that no longer + cover the current proxy callback, meaning authorize is guaranteed to fail IdP-side. + + Consulted when the in-memory server already carries a hydrated client_id, which + otherwise short-circuits registration before any redirect check can run. Servers + without a persisted DCR recording (admin-configured client_id, or registered before + redirect_uris were recorded) are never reported stale.""" + persisted = await _get_persisted_mcp_server_with_dcr_client_id(mcp_server) + if persisted is None: + return False + _, credentials = persisted + if not _redirect_uri_not_registered(credentials, current_redirect_uri): + return False + verbose_logger.warning( + "register_client_with_server: persisted DCR client for server_id=%s is registered with redirect_uris=%s " + "which do not include the current callback %s (proxy origin changed); registering a replacement client. " + "Users previously signed in to this server will need to re-authenticate.", + mcp_server.server_id, + credentials.redirect_uris, + current_redirect_uri, + ) + return True + + DcrRegistrationPersistenceResult = Literal["persisted", "reused", "skipped", "failed"] async def _persist_dcr_client_registration( - mcp_server: MCPServer, registration_response: object + mcp_server: MCPServer, registration_response: object, current_redirect_uri: str ) -> DcrRegistrationPersistenceResult: """Persist the dynamically registered OAuth client (RFC 7591) onto the MCP server row. @@ -929,6 +982,13 @@ async def _persist_dcr_client_registration( client identity for these servers. Persisting here would stamp ``oauth2_flow`` and a ``client_id`` onto a server whose mode promises the gateway stores nothing, making a fresh pass-through server read as gateway-authorized. + + ``redirect_uris`` records what the client is bound to so a later origin change can be + detected as a positive mismatch and trigger re-registration instead of stranding the + server on IdP-side redirect_uri rejections. ``client_secret`` and + ``token_endpoint_auth_method`` are written explicitly (None when absent) because + ``update_mcp_server`` merges credential blobs: a re-registered public client must not + inherit the previous client's secret or auth method. """ if mcp_server.is_true_passthrough or mcp_server.is_oauth_delegate: return "skipped" @@ -944,17 +1004,16 @@ async def _persist_dcr_client_registration( ) return "failed" - if await _reuse_persisted_dcr_client_if_available(mcp_server): + if await _reuse_persisted_dcr_client_if_available(mcp_server, current_redirect_uri=current_redirect_uri): return "reused" credentials: MCPCredentials = { "client_id": registration.client_id, - **({"client_secret": registration.client_secret} if registration.client_secret is not None else {}), - **( - {"token_endpoint_auth_method": "client_secret_basic"} - if registration.token_endpoint_auth_method == "client_secret_basic" - else {} + "client_secret": registration.client_secret, + "token_endpoint_auth_method": ( + "client_secret_basic" if registration.token_endpoint_auth_method == "client_secret_basic" else None ), + "redirect_uris": [current_redirect_uri], } from litellm.proxy._experimental.mcp_server.db import update_mcp_server # noqa: PLC0415 @@ -1017,16 +1076,24 @@ async def register_client_with_server( ): _raise_if_not_oauth2(mcp_server) request_base_url = get_request_base_url(request) + current_redirect_uri = f"{request_base_url}/callback" dummy_return = { "client_id": fallback_client_id or mcp_server.server_name, "client_secret": "dummy", - "redirect_uris": [f"{request_base_url}/callback"], + "redirect_uris": [current_redirect_uri], } - if mcp_server.client_id: + if mcp_server.client_id and not ( + persist_credentials + and mcp_server.registration_url + and await _persisted_dcr_redirect_uri_is_stale(mcp_server, current_redirect_uri) + ): return dummy_return - if await _reuse_persisted_dcr_client_if_available(mcp_server): + if await _reuse_persisted_dcr_client_if_available( + mcp_server, + current_redirect_uri=current_redirect_uri if persist_credentials else None, + ): return dummy_return if mcp_server.authorization_url is None: @@ -1044,7 +1111,7 @@ async def register_client_with_server( register_data = { "client_name": client_name, - "redirect_uris": client_redirect_uris if bridge_relay else [f"{request_base_url}/callback"], + "redirect_uris": client_redirect_uris if bridge_relay else [current_redirect_uri], "grant_types": grant_types or (["authorization_code", "refresh_token"] if bridge_relay else []), "response_types": response_types or (["code"] if bridge_relay else []), "token_endpoint_auth_method": token_endpoint_auth_method or ("none" if bridge_relay else ""), @@ -1072,7 +1139,7 @@ async def register_client_with_server( token_response = response.json() if persist_credentials and not bridge_relay: - persistence_result = await _persist_dcr_client_registration(mcp_server, token_response) + persistence_result = await _persist_dcr_client_registration(mcp_server, token_response, current_redirect_uri) if persistence_result == "reused": return dummy_return diff --git a/litellm/proxy/_experimental/mcp_server/oauth_utils.py b/litellm/proxy/_experimental/mcp_server/oauth_utils.py index 4d5813dbc5b..6edb22dd858 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth_utils.py +++ b/litellm/proxy/_experimental/mcp_server/oauth_utils.py @@ -129,7 +129,7 @@ def get_request_base_url(request: Request) -> str: if x_forwarded_port and ":" not in netloc: netloc = f"{netloc}:{x_forwarded_port}" - return urlunparse((scheme, netloc, parsed.path, "", "", "")) + return urlunparse((scheme, _strip_default_port(scheme, netloc), parsed.path, "", "", "")) def validate_loopback_redirect_uri(redirect_uri: str) -> None: diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 7aedb74f2ea..d60c17c744f 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -244,6 +244,10 @@ TPM_RESERVED_SCOPES_KEY = "_litellm_tpm_reserved_scopes" # does not double-refund. TPM_RESERVATION_RELEASED_KEY = "_litellm_tpm_reservation_released" RATE_LIMIT_DESCRIPTORS_KEY = "_litellm_rate_limit_descriptors" +# Pre-call RateLimitResponse stashed here so streaming success logging can +# mirror ``x-ratelimit-*`` headers into the SLP. Streaming exits +# common_request_processing before ``async_post_call_success_hook`` runs. +RATE_LIMIT_RESPONSE_KEY = "_litellm_proxy_rate_limit_response" # Stash keys live ONLY in metadata channels — never at the top level of the # request body. Top-level keys are forwarded as body params to upstream # providers, which reject unknown fields with 400/429 errors. @@ -253,6 +257,7 @@ _LITELLM_STASH_KEYS: Tuple[str, ...] = ( TPM_RESERVED_SCOPES_KEY, TPM_RESERVATION_RELEASED_KEY, RATE_LIMIT_DESCRIPTORS_KEY, + RATE_LIMIT_RESPONSE_KEY, ) @@ -2037,6 +2042,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): else: # add descriptors to request headers data["litellm_proxy_rate_limit_response"] = response + # Mirror into metadata so streaming success logging can find + # it via ``kwargs["litellm_params"]["metadata"]``. + self._stash_value_in_metadata_channels( + data=data, + key=RATE_LIMIT_RESPONSE_KEY, + value=response, + ) # ---------------------------------------------------------------- # TPM token reservation @@ -2133,6 +2145,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): stored_response.setdefault("statuses", []).extend(tpm_response["statuses"]) elif tpm_response["statuses"]: data["litellm_proxy_rate_limit_response"] = tpm_response + # Keep the metadata stash in sync when this is the + # first snapshot written. + self._stash_value_in_metadata_channels( + data=data, + key=RATE_LIMIT_RESPONSE_KEY, + value=tpm_response, + ) verbose_proxy_logger.debug(f"TPM tokens reserved: {estimated_tokens} for model {requested_model}") @@ -2318,6 +2337,23 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): return "total" # default to total return specified_rate_limit_type + @staticmethod + def _merge_ratelimit_statuses_into_additional_headers( + additional_headers: Dict[str, Any], + statuses: List[RateLimitStatus], + ) -> Dict[str, Any]: + """ + Return ``additional_headers`` extended with + ``x-ratelimit-{descriptor_key}-{remaining|limit}-{rate_limit_type}`` + entries. Non-mutating so callers pick their own target dict. + """ + merged: Dict[str, Any] = dict(additional_headers) + for status in statuses: + prefix = f"x-ratelimit-{status['descriptor_key']}" + merged[f"{prefix}-remaining-{status['rate_limit_type']}"] = status["limit_remaining"] + merged[f"{prefix}-limit-{status['rate_limit_type']}"] = status["current_limit"] + return merged + @staticmethod def _stash_value_in_metadata_channels( data: Dict[str, Any], @@ -2698,6 +2734,112 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): except Exception as e: verbose_proxy_logger.exception(f"Error in rate limit success event: {str(e)}") + async def async_logging_hook( + self, + kwargs: dict, + result: Any, + call_type: str, + ) -> Tuple[dict, Any]: + """ + Mirror the pre-call rate-limit snapshot into the SLP so streaming + success callbacks see the same ``x-ratelimit-*`` headers the + non-streaming path writes via ``async_post_call_success_hook``. + Runs in the earlier of the two callback loops inside + ``async_success_handler`` so downstream callbacks see the values + regardless of registration order. Idempotent for non-streaming. + """ + self._mirror_ratelimit_response_into_logging_payload( + kwargs=kwargs, + response_obj=result, + ) + return kwargs, result + + def _mirror_ratelimit_response_into_logging_payload( + self, + kwargs: Any, + response_obj: Any, + ) -> None: + """ + Copy the stashed ``RateLimitResponse`` into the SLP's + ``hidden_params.additional_headers`` and the response object's + ``_hidden_params.additional_headers`` (when the latter is a dict). + """ + if not isinstance(kwargs, dict): + return + + standard_logging_object = kwargs.get("standard_logging_object") + standard_logging_metadata: Optional[Dict[str, Any]] = None + if isinstance(standard_logging_object, dict): + slp_metadata = standard_logging_object.get("metadata") + if isinstance(slp_metadata, dict): + standard_logging_metadata = slp_metadata + + statuses = self._narrow_ratelimit_statuses( + self._lookup_stashed_value( + kwargs=kwargs, + standard_logging_metadata=standard_logging_metadata, + key=RATE_LIMIT_RESPONSE_KEY, + ) + ) + if not statuses: + return + + if isinstance(standard_logging_object, dict): + hidden_params = standard_logging_object.get("hidden_params") + if not isinstance(hidden_params, dict): + hidden_params = {} + existing = hidden_params.get("additional_headers") + hidden_params["additional_headers"] = self._merge_ratelimit_statuses_into_additional_headers( + additional_headers=existing if isinstance(existing, dict) else {}, + statuses=statuses, + ) + standard_logging_object["hidden_params"] = hidden_params + + response_hidden = getattr(response_obj, "_hidden_params", None) + if isinstance(response_hidden, dict): + existing = response_hidden.get("additional_headers") + response_hidden["additional_headers"] = self._merge_ratelimit_statuses_into_additional_headers( + additional_headers=existing if isinstance(existing, dict) else {}, + statuses=statuses, + ) + + @staticmethod + def _narrow_ratelimit_statuses(stashed: Any) -> List[RateLimitStatus]: + """ + Narrow a stashed ``RateLimitResponse``-shaped dict to a typed + ``statuses`` list. Entries missing any header-write field are dropped; + an empty list means "nothing to mirror". + """ + if not isinstance(stashed, dict): + return [] + raw_statuses = stashed.get("statuses") + if not isinstance(raw_statuses, list): + return [] + narrowed: List[RateLimitStatus] = [] + for entry in raw_statuses: + if not isinstance(entry, dict): + continue + descriptor_key = entry.get("descriptor_key") + rate_limit_type = entry.get("rate_limit_type") + current_limit = entry.get("current_limit") + limit_remaining = entry.get("limit_remaining") + if ( + isinstance(descriptor_key, str) + and rate_limit_type in ("requests", "tokens", "max_parallel_requests") + and isinstance(current_limit, int) + and isinstance(limit_remaining, int) + ): + narrowed.append( + RateLimitStatus( + code=entry.get("code", "OK") if isinstance(entry.get("code"), str) else "OK", + current_limit=current_limit, + limit_remaining=limit_remaining, + rate_limit_type=rate_limit_type, + descriptor_key=descriptor_key, + ) + ) + return narrowed + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): """ On failure: decrement max_parallel_requests and refund the upfront @@ -2838,15 +2980,10 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if isinstance(_hidden_params, BaseModel): _hidden_params = _hidden_params.model_dump() - _additional_headers = _hidden_params.get("additional_headers", {}) or {} - - # Add rate limit headers - for status in litellm_proxy_rate_limit_response["statuses"]: - prefix = f"x-ratelimit-{status['descriptor_key']}" - _additional_headers[f"{prefix}-remaining-{status['rate_limit_type']}"] = status[ - "limit_remaining" - ] - _additional_headers[f"{prefix}-limit-{status['rate_limit_type']}"] = status["current_limit"] + _additional_headers = self._merge_ratelimit_statuses_into_additional_headers( + additional_headers=_hidden_params.get("additional_headers", {}) or {}, + statuses=litellm_proxy_rate_limit_response["statuses"], + ) setattr( response, diff --git a/litellm/types/mcp.py b/litellm/types/mcp.py index d273e8ec4db..ac411ad9d9a 100644 --- a/litellm/types/mcp.py +++ b/litellm/types/mcp.py @@ -165,6 +165,15 @@ class MCPCredentials(TypedDict, total=False): sends HTTP Basic; defaults to "client_secret_post" when unset. """ + redirect_uris: Optional[List[str]] + """ + The redirect URIs a dynamically registered (RFC 7591) OAuth client was bound to at + registration time. Lets a later registration detect that the proxy's public origin no + longer matches the registered callback and re-register instead of reusing a client the + IdP will reject. Absent for admin-configured clients and for clients registered before + this field existed. Not a secret; stored unencrypted. + """ + token_exchange_profile: Optional[str] """ Token exchange wire dialect: "rfc8693" (default, the standard token-exchange grant) or diff --git a/litellm/utils.py b/litellm/utils.py index 731be992af0..18b89ee0d13 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -61,7 +61,7 @@ from litellm._lazy_imports import ( ) from litellm._uuid import uuid from litellm.litellm_core_utils.fallback_generalizations import ( - match_all_fallback_generalizations, + match_capability_generalizations, ) from litellm.constants import ( DEFAULT_CHAT_COMPLETION_PARAM_VALUES, @@ -2651,6 +2651,26 @@ def _resolve_builtin_model_cost_entry(key: str, provider: str) -> Optional[Dict[ return None +def _get_builtin_model_info_for_registration(model: str) -> Optional[ModelInfo]: + """Resolve ``model`` to its built-in cost-map entry for registration merging. + + Returns ``None`` when the lookup raises or when it resolved via a + fallback-generalization capability rule, detected as the resolved key missing + ``litellm.model_cost`` while matching a capability rule. A rule-derived entry + carries no pricing, so treating it as a hit would skip the built-in + cache-pricing inheritance for prefix-mangled keys. + """ + try: + info = get_model_info(model=model) + except Exception: + return None + if info["key"] in litellm.model_cost: + return info + if match_capability_generalizations(info["key"]) is None: + return info + return None + + def register_model(model_cost: Union[str, dict]): """ Register new / Override existing models (and their pricing) to specific providers. @@ -2691,10 +2711,11 @@ def register_model(model_cost: Union[str, dict]): existing_model = litellm.model_cost.get(key, {}) model_cost_key = key else: - try: - existing_model = cast(dict, get_model_info(model=key)) + builtin_model_info = _get_builtin_model_info_for_registration(model=_key_str) + if builtin_model_info is not None: + existing_model = cast(dict, builtin_model_info) model_cost_key = existing_model["key"] - except Exception: + else: existing_model = {} model_cost_key = key builtin_entry = _resolve_builtin_model_cost_entry(key=_key_str, provider=provider) @@ -5043,25 +5064,35 @@ def _get_model_info_from_generalization( potential_model_names: PotentialModelNamesAndCustomLLMProvider, custom_llm_provider: Optional[str], ) -> Optional[tuple[str, dict]]: - """Resolve an unmapped model via a declarative fallback-generalization rule. + """Resolve an unmapped model via the declarative capability generalization rules. Tries the same name candidates as the exact lookups, in the same order, and - returns ``(matched_name, model_info)`` for the first matching rule that also - satisfies the provider constraint; a rule scoped to another provider is - skipped in favor of later rules rather than discarding the candidate. - O(number of rules); only call after the exact lookups have missed. + returns ``(matched_name, model_info)`` for the first candidate matched by at + least one capability rule, with ``litellm_provider`` backfilled from the + provider the caller requested. Rules lose to exact entries: if ANY candidate is + an exact ``litellm.model_cost`` key (necessarily provider-mismatched, or the + exact lookups would have returned it), the model is known rather than unmapped, + and resolving it from rules would hand an unpriced rule-derived entry to + callers whose fallback ladder (e.g. the cost calculator's model-name variants) + still had a priced exact name to try. O(number of rules); only call after the + exact lookups have missed. """ - candidates = [ + candidates = ( potential_model_names["combined_model_name"], model, potential_model_names["split_model"], potential_model_names["combined_stripped_model_name"], potential_model_names["stripped_model_name"], - ] + ) + if any(_get_model_cost_key(candidate) is not None for candidate in candidates): + return None for candidate in candidates: - for generalized_info in match_all_fallback_generalizations(candidate): - if _check_provider_match(model_info=generalized_info, custom_llm_provider=custom_llm_provider): - return candidate, generalized_info + generalized_info = match_capability_generalizations(candidate) + if generalized_info is None: + continue + if custom_llm_provider is None: + return candidate, generalized_info + return candidate, {**generalized_info, "litellm_provider": custom_llm_provider} return None diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 4920f12e428..94d9f6496bd 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1359,6 +1359,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1393,6 +1394,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1427,6 +1429,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1461,6 +1464,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1661,6 +1665,7 @@ "jp.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1743,6 +1748,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1777,6 +1783,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1811,6 +1818,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1845,6 +1853,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1879,6 +1888,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1913,6 +1923,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -45272,31 +45283,26 @@ "fallback_generalizations": { "rules": [ { - "name": "bedrock-anthropic-claude-mid-conversation-system", - "pattern": "anthropic\\.claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", - "description": "Bedrock Invoke ids for Claude 4.8 or higher: anthropic.claude- with minor 4.8 through 4.99, any 5.x or later major-minor, or a bare 5+ major, which also admits new families such as fable. These models accept mid-conversation role system messages in place (verified live on Opus 4.8, Sonnet 5 and Fable 5), so unmapped future Bedrock Claudes keep the cache-preserving in-place handling instead of the hoist-all default. Listed first so bare-id provider inference, which takes the first pattern hit, resolves these Bedrock ids to bedrock; model-info resolution skips provider-mismatched rules either way.", - "extends": "anthropic-claude", + "name": "bedrock-claude-ids", + "pattern": "anthropic\\.claude-", + "description": "Any Bedrock-syntax Claude id: the dotted anthropic.claude- segment appears in bare (anthropic.claude-...), region-prefixed (us./eu./au./jp./apac.) and global.-prefixed ids, for every version. Routes these to bedrock before the bare-id Anthropic rule is consulted.", "model_info": { - "litellm_provider": "bedrock", - "supports_adaptive_thinking": true, - "supports_mid_conversation_system": true + "litellm_provider": "bedrock" } }, { - "name": "anthropic-claude-adaptive-thinking", - "pattern": "(?:opus|sonnet|haiku)[-._](?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d{1,})[-._]\\d{1,2}(?!\\d))", - "description": "Claude opus/sonnet/haiku at version 4.6 or higher: 4.6 through 4.99, then any 5.x, 6.x or later major. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new families with no code change.", - "extends": "anthropic-claude", + "name": "anthropic-claude-ids", + "pattern": "^claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?$", + "description": "A bare Claude family-major id with an optional minor and an optional 8-digit date suffix, anchored to the whole name, so claude-newfamily-5 routes like claude-newfamily-5-1 does. Routes an unmapped Claude id that carries no provider namespace to the Anthropic API.", "model_info": { - "supports_adaptive_thinking": true + "litellm_provider": "anthropic" } }, { - "name": "anthropic-claude", - "pattern": "^claude-[a-z]+-\\d+[-.]\\d+(?:-\\d{8})?$", - "description": "Any Claude family-major-minor id, optionally with an 8-digit date suffix, anchored to the whole name. Version-neutral fallback that gives an unmapped Claude provider routing and baseline capabilities; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.", + "name": "claude-family-baseline", + "pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?", + "description": "Any Claude family-major id with an optional minor and an optional 8-digit date suffix, under any provider namespace (bare, bedrock-dotted, vertex, databricks, ...), so bare majors like claude-newfamily-5 get the same baseline as claude-newfamily-5-1. Carries the model-family facts every Claude shares; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.", "model_info": { - "litellm_provider": "anthropic", "mode": "chat", "max_input_tokens": 200000, "max_output_tokens": 64000, @@ -45312,6 +45318,22 @@ "supports_pdf_input": true, "supports_system_messages": true } + }, + { + "name": "claude-adaptive-thinking", + "pattern": "claude-[a-z]+-(?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", + "description": "Claude at version 4.6 or higher, in any id shape that contains claude--: minors 4.6 through 4.99, any later major-minor, and bare 5+ majors so a new family shaped like claude-fable-5 matches. Requiring the claude- prefix keeps non-Claude names such as team-sonnet-5-1 out. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new versions and new families with no code change.", + "model_info": { + "supports_adaptive_thinking": true + } + }, + { + "name": "claude-mid-conversation-system", + "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", + "description": "Claude at version 4.8 or higher, in any id shape that contains claude--: minors 4.8 through 4.99, any later major-minor, and bare 5+ majors so a new family like claude-fable-5 matches. Anthropic introduced mid-conversation system messages with Opus 4.8 and every newer Claude keeps them; 4.7 and below reject the system role inside messages.", + "model_info": { + "supports_mid_conversation_system": true + } } ] } diff --git a/pyproject.toml b/pyproject.toml index 4c70bf66031..effcd0c675f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -184,6 +184,10 @@ dev = [ "vcrpy==8.2.1", "pytest-recording==0.13.4", ] +e2e-dev = [ + "playwright==1.61.0", + "websockets>=15.0.1,<16.0", +] proxy-dev = [ "prisma==0.11.0", "hypercorn==0.17.3", diff --git a/scripts/pre_commit_lint.sh b/scripts/pre_commit_lint.sh index bd416ab814d..54d488b2440 100755 --- a/scripts/pre_commit_lint.sh +++ b/scripts/pre_commit_lint.sh @@ -4,6 +4,7 @@ # before `git commit`; it inspects your staged files and runs only the matching # gating CI checks, so a clean run means a green CI lint: # - litellm/ Python staged -> `make lint` (test-linting.yml's lint job) +# - tests/e2e Python staged -> `make lint-e2e-basedpyright` (test-linting.yml's e2e type-check step) # - dashboard staged -> prettier + eslint + lint budgets (test-litellm-ui-build.yml's frontend-lint) # - proxy/types staged -> regenerate dashboard API types and fail on drift (check-ui-api-types.yml) # @@ -25,6 +26,7 @@ staged_match() { printf '%s\n' "$staged" | grep -E "$1" || true; } # scripts-only commit can't turn it red; scope the trigger there to skip the slow # make lint when it couldn't catch anything. litellm_py_files=$(staged_match '^litellm/.*\.py$') +e2e_py_files=$(staged_match '^tests/e2e/.*\.py$') # ruff format (and CI's format step) skip enterprise; the rest of make lint covers it. fmt_files=$(printf '%s\n' "$litellm_py_files" | grep -v '^litellm/enterprise/' || true) # check-ui-api-types.yml triggers on any file under litellm/proxy or litellm/types @@ -107,6 +109,11 @@ if [ -n "$litellm_py_files" ]; then fi fi +if [ -n "$e2e_py_files" ] && [ -z "$litellm_py_files" ]; then + echo "pre-commit: type-checking tests/e2e (make lint-e2e-basedpyright)" + make lint-e2e-basedpyright || { echo "✗ tests/e2e basedpyright failed. Fix the errors above, then re-run make pre-commit." >&2; status=1; } +fi + if [ -n "$ui_prettier_files" ] || [ -n "$ui_eslint_files" ]; then echo "pre-commit: linting dashboard (prettier + eslint + lint budgets)" lint_dashboard || { echo "✗ Dashboard lint failed. See above; format with: (cd ui/litellm-dashboard && npm run format)." >&2; status=1; } diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 502c881c764..13bd0ea0cec 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -57,7 +57,7 @@ Mark live tests with `@pytest.mark.e2e` (on the class or the module). Pure cover ## Typing -The harness is fully typed and new code must not add `Any` or widen the basedpyright budgets. When a response field is untyped, model it in `models.py` (just the fields you read) and let pydantic validate it, rather than threading a `dict` or `Any` through the test +The harness is fully typed with no error budget: `make lint-e2e-basedpyright` must report zero basedpyright errors, and CI enforces that on any PR touching `tests/e2e/**/*.py`. When a response field is untyped, model it in `models.py` (just the fields you read) and let pydantic validate it, rather than threading a `dict` or `Any` through the test ## Coverage registry diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 4395d299c5b..2082f2c9de4 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -34,10 +34,10 @@ The suites run against a live proxy, so bring one up first. `docker-compose.yml` uv run pytest tests/e2e/llm_translation/ -v ``` - The browser tests in the `management/` suite drive the dashboard the proxy serves at `/ui` through playwright, an optional dependency behind `importorskip` (the suite's API tests run without it). Install it once into your environment along with its browser: + The browser tests in the `management/` suite drive the dashboard the proxy serves at `/ui` through playwright, an optional dependency behind `importorskip` (the suite's API tests run without it). It lives in the `e2e-dev` dependency group; install it along with its browser: ```bash - uv pip install playwright + uv sync --inexact --group e2e-dev uv run playwright install chromium ``` @@ -140,7 +140,7 @@ Mark live tests with `@pytest.mark.e2e` (on the class or the module). Pure cover Before you push -1. Run basedpyright over your changes; the harness is fully typed and new code must not add `Any` or widen the budgets +1. Run `make lint-e2e-basedpyright` (or `make pre-commit` with your changes staged); the harness is fully typed and the gate allows zero basedpyright errors, enforced in CI on any PR touching `tests/e2e/**/*.py` 2. Add the models your test needs to the inline config in `docker-compose.yml` diff --git a/tests/e2e/budgets/budget_client.py b/tests/e2e/budgets/budget_client.py index fe862b5c119..7b37c3af98e 100644 --- a/tests/e2e/budgets/budget_client.py +++ b/tests/e2e/budgets/budget_client.py @@ -342,7 +342,7 @@ class BudgetClient: case _: time.sleep(_TEAM_READY_SLEEP_SECONDS) assert last is not None - _ = unwrap(last) + raise AssertionError(last) def add_team_member(self, team_id: str, user_id: str, *, max_budget_in_team: float | None = None) -> None: last_body = "" diff --git a/tests/e2e/budgets/test_budget_fallback_e2e.py b/tests/e2e/budgets/test_budget_fallback_e2e.py index 6114bfe6fc9..adbda5ed767 100644 --- a/tests/e2e/budgets/test_budget_fallback_e2e.py +++ b/tests/e2e/budgets/test_budget_fallback_e2e.py @@ -4,7 +4,6 @@ blocking the caller with a `budget_exceeded` error. Coverage for the budget_fallbacks feature in litellm/proxy/hooks/model_max_budget_limiter.py. """ -import json import time import pytest @@ -12,6 +11,7 @@ import pytest from budget_client import BudgetClient, model_budget from e2e_config import unique_marker from lifecycle import ResourceManager +from models import AnthropicMessagesResponse pytestmark = pytest.mark.e2e @@ -42,8 +42,8 @@ def test_budget_fallback_reroutes_anthropic_messages_to_openai( "budget_fallbacks must reroute transparently, never surface a " f"block; status={result.status_code} body={result.body[:300]}" ) - served_by = json.loads(result.body)["model"] - if FALLBACK_MODEL in served_by: + served_by = AnthropicMessagesResponse.model_validate_json(result.body).model + if served_by is not None and FALLBACK_MODEL in served_by: break time.sleep(1) assert served_by is not None and FALLBACK_MODEL in served_by, ( diff --git a/tests/e2e/budgets/test_spend_counter_reseed_e2e.py b/tests/e2e/budgets/test_spend_counter_reseed_e2e.py index de868f4fd3c..132f99e9cdf 100644 --- a/tests/e2e/budgets/test_spend_counter_reseed_e2e.py +++ b/tests/e2e/budgets/test_spend_counter_reseed_e2e.py @@ -23,6 +23,7 @@ from threading import Barrier from typing import TYPE_CHECKING import pytest +from pydantic import TypeAdapter, ValidationError from budget_client import BudgetClient from e2e_config import unique_marker @@ -42,6 +43,8 @@ BURST = 6 # expires the counter; this waits out both. COLD_WAIT_SECONDS = 80 +_JSON_FLOAT: TypeAdapter[float] = TypeAdapter(float) + def _redis() -> "redis.Redis[str] | RedisCluster[str]": """The proxy's Redis. The deployed runner sets REDIS_HOST to the serverless @@ -75,12 +78,11 @@ def _parse_counter(raw: object) -> float | None: try: return float(text) except ValueError: - try: - import json - - return float(json.loads(text)) - except Exception: - return None + pass + try: + return _JSON_FLOAT.validate_json(text) + except ValidationError: + return None def _spend_counter(rds: "redis.Redis[str] | RedisCluster[str]", key: str) -> float | None: diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 82f2604d492..4c4c8fe735b 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -1,9 +1,9 @@ """Shared fixtures for all live e2e suites under tests/e2e/. -Design rule: skip on environment, fail on behavior. Live tests (marked `e2e`) -skip when no proxy answers; once a request reaches the proxy, behavior is -asserted. Pure unit coverage of the harness itself carries no `e2e` marker and -runs regardless of whether a proxy is up. +Design rule: hard failures only. Live tests (marked `e2e`) fail when no proxy +answers or when credentials/env are missing; they never skip. Pure unit coverage +of the harness itself carries no `e2e` marker and runs regardless of whether a +proxy is up. Lifecycle: the `resources` fixture maps the init -> run -> teardown contract (lifecycle.E2ECase) onto pytest - setup is init(), the test body is run(), and @@ -40,7 +40,7 @@ def pytest_configure(config: pytest.Config) -> None: def _liveness_reason(label: str, base_url: str) -> str | None: - """None if `base_url` answers its liveness probe, else a skip reason.""" + """None if `base_url` answers its liveness probe, else a failure reason.""" try: resp = requests.get(f"{base_url}/health/liveliness", timeout=5) except requests.RequestException as exc: @@ -51,10 +51,10 @@ def _liveness_reason(label: str, base_url: str) -> str | None: @functools.lru_cache(maxsize=1) -def _proxy_skip_reason() -> str | None: - """Probe the proxy once per session. None if it answers, else a skip reason. In - a split deployment the management/admin control plane is a separate service, so - require it too (when it differs) - else its tests would fail rather than skip.""" +def _proxy_fail_reason() -> str | None: + """Probe the proxy once per session. None if it answers, else a failure reason. + In a split deployment the management/admin control plane is a separate service, + so require it too when it differs.""" reason = _liveness_reason("proxy", PROXY_BASE_URL) if reason is not None: return reason @@ -64,19 +64,19 @@ def _proxy_skip_reason() -> str | None: def pytest_runtest_setup(item: pytest.Item) -> None: - """Skip `e2e`-marked tests unless a proxy answers its liveness probe. Unmarked - tests (unit coverage of the harness) don't touch the proxy, so they run even - when none is up.""" + """Hard-fail `e2e`-marked tests unless a proxy answers its liveness probe. + Unmarked tests (unit coverage of the harness) don't touch the proxy, so they + run even when none is up. Never skip for a missing proxy.""" if item.get_closest_marker("e2e") is None: return - reason = _proxy_skip_reason() + reason = _proxy_fail_reason() if reason is not None: - pytest.skip(reason) + pytest.fail(reason) def pytest_runtest_call(item: pytest.Item) -> None: - """Mark that an e2e test body actually ran (not skipped at setup). Skipped - sessions never reach this hook, so the session-finish cleanup can use it as a + """Mark that an e2e test body actually ran (setup passed). Sessions that fail + setup never reach this hook, so the session-finish cleanup can use it as a guard before truncating the spend-log DB. Tests under `tests/e2e/` without the `e2e` marker (pure unit coverage for the harness itself) never hit the proxy, so they must not arm the destructive DB truncate.""" @@ -87,12 +87,12 @@ def pytest_runtest_call(item: pytest.Item) -> None: def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: """Once the whole e2e session is done (all suites), truncate the spend logs so - the DB doesn't accumulate test rows. Skipped sessions (no live proxy, no test - actually executed) leave the DB alone so a `DATABASE_URL` pointing at a shared - instance is never wiped without an e2e run. Best-effort: a cleanup failure (no - DB reachable) must not fail the run. The spend_tracking dir goes on sys.path - only for this import and is removed after, so a broader `pytest tests/` run is - not left with a mutated path.""" + the DB doesn't accumulate test rows. Sessions where no e2e test body ran leave + the DB alone so a `DATABASE_URL` pointing at a shared instance is never wiped + without an e2e run. Best-effort: a cleanup failure (no DB reachable) must not + fail the run. The spend_tracking dir goes on sys.path only for this import and + is removed after, so a broader `pytest tests/` run is not left with a mutated + path.""" if not session.stash.get(_E2E_TEST_RAN, False): return spend_dir = str(Path(__file__).parent / "spend_tracking") @@ -102,7 +102,7 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: reset_spend_logs() except Exception as exc: # noqa: BLE001 - cleanup is best-effort - print(f"spend-log cleanup skipped: {exc}") + print(f"spend-log cleanup best-effort failed: {exc}") finally: if spend_dir in sys.path: sys.path.remove(spend_dir) @@ -112,7 +112,7 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: remediate(session) except Exception as exc: # noqa: BLE001 - remediation is best-effort - print(f"devin remediation skipped: {exc}") + print(f"devin remediation best-effort failed: {exc}") @pytest.fixture diff --git a/tests/e2e/coverage_registry/collector.py b/tests/e2e/coverage_registry/collector.py index f6e59ca4a88..50ef23bcb1a 100644 --- a/tests/e2e/coverage_registry/collector.py +++ b/tests/e2e/coverage_registry/collector.py @@ -17,8 +17,10 @@ import sys from argparse import ArgumentParser from dataclasses import dataclass from pathlib import Path +from typing import Literal import pytest +from pydantic import BaseModel from .registry import load_registry from .schema import MODULE_ORDER, Cell, Tier, dashboard_module, loki_module_label @@ -35,12 +37,13 @@ class _CoversSink: self.collection_errors: tuple[str, ...] = () def pytest_collection_finish(self, session: pytest.Session) -> None: - self.covered_ids = frozenset( - arg + marker_args: tuple[tuple[object, ...], ...] = tuple( + marker.args for item in session.items for marker in item.iter_markers(name="covers") - for arg in marker.args - if isinstance(arg, str) + ) + self.covered_ids = frozenset( + arg for args in marker_args for arg in args if isinstance(arg, str) ) def pytest_collectreport(self, report: pytest.CollectReport) -> None: @@ -257,6 +260,12 @@ def render_loki(report: CoverageReport) -> str: return "\n".join(lines) +class _CliArgs(BaseModel): + format: Literal["text", "json", "prometheus", "loki"] + strict: bool + fail_on_collection_errors: bool + + def main() -> int: parser = ArgumentParser() parser.add_argument( @@ -275,7 +284,7 @@ def main() -> int: action="store_true", help="Exit non-zero if pytest collection errors are found.", ) - args = parser.parse_args() + args = _CliArgs.model_validate(vars(parser.parse_args())) cells = load_registry() covered, errors = collect_covered_ids() report = compute_coverage(cells, covered, errors) diff --git a/tests/e2e/coverage_registry/registry.py b/tests/e2e/coverage_registry/registry.py index 7a472cfbf2f..baeea080a9d 100644 --- a/tests/e2e/coverage_registry/registry.py +++ b/tests/e2e/coverage_registry/registry.py @@ -6,20 +6,23 @@ from collections import Counter from pathlib import Path import yaml +from pydantic import TypeAdapter -from .schema import CELL_ADAPTER, Cell +from .schema import Cell REGISTRY_DIR = Path(__file__).resolve().parent +_CELLS_ADAPTER: TypeAdapter[tuple[Cell, ...]] = TypeAdapter(tuple[Cell, ...]) + + +def _load_cells(path: Path) -> tuple[Cell, ...]: + return _CELLS_ADAPTER.validate_python(yaml.safe_load(path.read_text()) or ()) + def load_registry(registry_dir: Path = REGISTRY_DIR) -> tuple[Cell, ...]: """Every cell across every `*.yaml`, validated. Raises on a schema violation or a duplicate id, since either would corrupt the coverage denominator.""" - cells = tuple( - CELL_ADAPTER.validate_python(row) - for path in sorted(registry_dir.glob("*.yaml")) - for row in (yaml.safe_load(path.read_text()) or ()) - ) + cells = tuple(cell for path in sorted(registry_dir.glob("*.yaml")) for cell in _load_cells(path)) duplicates = sorted(cid for cid, n in Counter(c.id for c in cells).items() if n > 1) if duplicates: raise ValueError(f"duplicate cell ids in registry: {duplicates}") diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py index 7b8d3045d3b..32005faed4b 100644 --- a/tests/e2e/e2e_http.py +++ b/tests/e2e/e2e_http.py @@ -107,13 +107,15 @@ class ProbeResult(BaseModel): class StreamingResponse(BaseModel): """Raw outcome for calls whose body is provider-native or streamed: status, the - x-litellm-call-id header (== SpendLogs.request_id), the content-type (which - tells streaming `text/event-stream` from non-streaming `application/json`), and - the body. Used by passthrough and streaming, where one validated JSON model - does not fit.""" + x-litellm-call-id header, the x-litellm-response-cost header (StandardLogging + response_cost), the content-type (which tells streaming `text/event-stream` from + non-streaming `application/json`), and the body. SpendLogs.request_id is the + completion body id, not call_id. Used by passthrough and streaming, where one + validated JSON model does not fit.""" status_code: int call_id: str | None = None # x-litellm-call-id header + response_cost: float | None = None # x-litellm-response-cost header content_type: str | None = None body: str chunks: int = 0 # streamed events (0 for non-streaming) @@ -260,13 +262,25 @@ def probe( return ProbeResult(status_code=resp.status_code, body=resp.text) +def _parse_response_cost(resp: requests.Response) -> float | None: + raw = _hdr(resp, "x-litellm-response-cost") + if raw is None or raw == "": + return None + try: + return float(raw) + except ValueError: + return None + + def _streaming_outcome(resp: requests.Response, stream: bool) -> StreamingResponse: call_id = _hdr(resp, "x-litellm-call-id") + response_cost = _parse_response_cost(resp) content_type = _hdr(resp, "content-type") if not stream or not (200 <= resp.status_code < 300): return StreamingResponse( status_code=resp.status_code, call_id=call_id, + response_cost=response_cost, content_type=content_type, body=resp.text, ) @@ -275,6 +289,7 @@ def _streaming_outcome(resp: requests.Response, stream: bool) -> StreamingRespon return StreamingResponse( status_code=resp.status_code, call_id=call_id, + response_cost=response_cost, content_type=content_type, body="", chunks=chunks, diff --git a/tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md b/tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md index ffec7176f10..4795c3b9f54 100644 --- a/tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md +++ b/tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md @@ -17,14 +17,15 @@ transcript, and that `response.done` carries normalized usage. normalized `response.function_call_arguments.done` with valid JSON arguments and a matching `function_call` output item, the test sends a `function_call_output` back, and the follow-up response incorporates the result (the temperature 72 -appears). +appears). That raw-websocket tool path is the source of truth for tool calling. -`test_realtime_pipecat_e2e` is a realism layer that drives the same providers -through pipecat's GA `OpenAIRealtimeLLMService` (base_url pointed at the proxy) -rather than speaking the protocol by hand. Its assertions are coarse (the tool -callback fired, assistant text was produced); the raw-websocket suite is the -source of truth. It skips unless `pipecat-ai` is installed -(`uv pip install "pipecat-ai[openai]"`). +`test_pipecat_tool_smoke` is a realism layer through pipecat for openai, azure, +and gemini only (not vertex_ai: native-audio live is flaky under pipecat tool +calling while raw-ws tools pass; see pipecat-ai/pipecat#2544). Assertions are +coarse; raw-ws remains authoritative. Requires `pipecat-ai`. + +Pipecat audio coverage lives in `test_realtime_pipecat_audio_e2e.py` (VAD / audio +I/O). ## Provisioning @@ -43,7 +44,7 @@ at call time. The provider table below is the source of truth; edit `PROVIDERS` | gemini | `gemini-realtime` | `gemini/gemini-3.1-flash-live-preview` | | vertex_ai | `vertex-realtime` | `vertex_ai/gemini-live-2.5-flash-preview-native-audio-09-2025` | -Bedrock and xai (`xai/grok-4-1-fast`) are supported by the proxy but +Bedrock and xai (`xai/grok-4-1-fast-non-reasoning`) are supported by the proxy but kept commented out in `PROVIDERS` until they pass end-to-end here; re-enable them by uncommenting their entry. diff --git a/tests/e2e/llm_translation/realtime/realtime_client.py b/tests/e2e/llm_translation/realtime/realtime_client.py index 51052abe227..b50160e3538 100644 --- a/tests/e2e/llm_translation/realtime/realtime_client.py +++ b/tests/e2e/llm_translation/realtime/realtime_client.py @@ -14,7 +14,7 @@ import time from collections.abc import Generator, Mapping from contextlib import contextmanager from dataclasses import dataclass -from typing import Any, TypeVar +from typing import TypeVar from urllib.parse import urlencode from pydantic import BaseModel, ConfigDict @@ -28,7 +28,7 @@ from models import LiteLLMParamsBody _M = TypeVar("_M", bound=BaseModel) -def _ws_base_url() -> str: +def ws_base_url() -> str: for scheme, ws_scheme in (("https://", "wss://"), ("http://", "ws://")): if PROXY_BASE_URL.startswith(scheme): return ws_scheme + PROXY_BASE_URL[len(scheme) :] @@ -36,7 +36,7 @@ def _ws_base_url() -> str: def realtime_ws_url(model: str) -> str: - return f"{_ws_base_url()}/v1/realtime?{urlencode({'model': model})}" + return f"{ws_base_url()}/v1/realtime?{urlencode({'model': model})}" @dataclass(frozen=True, slots=True) @@ -217,7 +217,7 @@ class OutputItemDone(BaseModel): class ResponsePayload(BaseModel): model_config = ConfigDict(extra="allow") - usage: dict[str, Any] | None = None + usage: dict[str, object] | None = None output: list[OutputItem] | None = None diff --git a/tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py index 0d290f6f64b..78955974cd5 100644 --- a/tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py +++ b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py @@ -30,7 +30,7 @@ import pytest from realtime_client import ( PROVIDERS, RealtimeProvider, - _ws_base_url, + ws_base_url, realtime_model, ) @@ -154,7 +154,7 @@ async def _run_pipeline( llm = LiteLLMRealtimeLLMService( api_key=key, - base_url=f"{_ws_base_url()}/v1/realtime", + base_url=f"{ws_base_url()}/v1/realtime", settings=OpenAIRealtimeLLMService.Settings( model=model, system_instruction=( @@ -276,7 +276,7 @@ async def _run_audio_input_pipeline( llm = LiteLLMRealtimeLLMService( api_key=key, - base_url=f"{_ws_base_url()}/v1/realtime", + base_url=f"{ws_base_url()}/v1/realtime", settings=OpenAIRealtimeLLMService.Settings( model=model, system_instruction=( diff --git a/tests/e2e/llm_translation/realtime/test_realtime_pipecat_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_e2e.py index 799958ef4e3..16628fd257a 100644 --- a/tests/e2e/llm_translation/realtime/test_realtime_pipecat_e2e.py +++ b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_e2e.py @@ -28,7 +28,7 @@ import pytest from realtime_client import ( PROVIDERS, RealtimeProvider, - _ws_base_url, + ws_base_url, realtime_model, ) @@ -60,7 +60,12 @@ from pipecat.services.llm_service import FunctionCallParams # noqa: E402 from pipecat_service import LiteLLMRealtimeLLMService # noqa: E402 -PROVIDER_PARAMS = [pytest.param(p, id=p.id) for p in PROVIDERS] +# Vertex native-audio live is flaky through pipecat tool calling (upstream +# pipecat-ai/pipecat#2544); raw-ws tool_call_round_trip[vertex_ai] is the +# source of truth for that provider. Keep openai/azure/gemini here. +PROVIDER_PARAMS = [ + pytest.param(p, id=p.id) for p in PROVIDERS if p.id != "vertex_ai" +] WEATHER_TOOL = ToolsSchema( standard_tools=[ @@ -94,7 +99,7 @@ async def _run_pipeline(key: str, model: str) -> tuple[bool, bool]: await params.result_callback({"city": "Paris", "temperature_f": 72}) llm = LiteLLMRealtimeLLMService( - api_key=key, base_url=f"{_ws_base_url()}/v1/realtime", model=model + api_key=key, base_url=f"{ws_base_url()}/v1/realtime", model=model ) llm.register_function("get_weather", get_weather) diff --git a/tests/e2e/llm_translation/test_cache_control.py b/tests/e2e/llm_translation/test_cache_control.py index fda05cdc950..51ae6ccbc1d 100644 --- a/tests/e2e/llm_translation/test_cache_control.py +++ b/tests/e2e/llm_translation/test_cache_control.py @@ -111,12 +111,11 @@ def _assert_cache_read_on_second_call( first = unwrap(_cache_chat(client, key, model, prefix)) assert first.choices, f"{model}: first cache-priming call returned no choices: {first}" - read_tokens = 0 deadline = time.monotonic() + 30.0 - while time.monotonic() < deadline: + while True: second = unwrap(_cache_chat(client, key, model, prefix)) read_tokens = _cached_read_tokens(second.usage) - if read_tokens > 0: + if read_tokens > 0 or time.monotonic() >= deadline: break time.sleep(3.0) diff --git a/tests/e2e/logging/conftest.py b/tests/e2e/logging/conftest.py index 40c19aefca7..567355f23ac 100644 --- a/tests/e2e/logging/conftest.py +++ b/tests/e2e/logging/conftest.py @@ -1,37 +1,43 @@ -"""Fixtures for the Datadog logging suite. +"""Fixtures for the logging e2e suite. -These tests drive the Datadog batch-send path (#25663) directly against the real -Datadog logs intake with synthetic events - no LLM calls, no proxy, no log -read-back - so they need only the shipping credentials DD_API_KEY + DD_SITE -(DD_SERVICE is an optional tag). No Datadog Application key is required, and they -skip when the shipping credentials are absent from the environment. +Missing proxy, provider keys, or integration credentials are hard failures. +Never pytest.skip from this suite for environment gaps. """ +from __future__ import annotations + import os import pytest -from logging_client import LoggingClient, build_logging_client +from logging_client import LangfuseCreds, LoggingClient, build_logging_client, load_langfuse_creds def pytest_configure(config: pytest.Config) -> None: config.addinivalue_line( "markers", - "covers: registry cell a test covers, e.g. logging.datadog.success.writes_object", + "covers: registry cell a test covers, e.g. logging.langfuse.success.logs_spend", ) @pytest.fixture(scope="session") def client() -> LoggingClient: """The logging suite's client: holds the shared Gateway so `resources` / - `scoped_key` clean up keys, and adds `/metrics` scraping.""" + `scoped_key` clean up keys and teams, and adds `/metrics` scraping plus + Langfuse read-back.""" return build_logging_client() @pytest.fixture def datadog_creds() -> None: - """Gate the suite on the Datadog shipping credentials. The DataDogLogger is built - inside each async test, not here, because its __init__ schedules a periodic-flush - task via asyncio.create_task and so needs a running event loop.""" + """Require Datadog shipping credentials. Hard-fail when absent; never skip.""" if not (os.getenv("DD_API_KEY") and os.getenv("DD_SITE")): - pytest.skip("set DD_API_KEY and DD_SITE to run the Datadog logging suite") + pytest.fail( + "Datadog e2e requires DD_API_KEY and DD_SITE; missing credentials is a hard failure, not a skip" + ) + + +@pytest.fixture(scope="session") +def langfuse_creds() -> LangfuseCreds: + """Require real Langfuse cloud credentials for team callback + trace poll.""" + return load_langfuse_creds() diff --git a/tests/e2e/logging/logging_client.py b/tests/e2e/logging/logging_client.py index a3213fbdb00..06b219fc6e2 100644 --- a/tests/e2e/logging/logging_client.py +++ b/tests/e2e/logging/logging_client.py @@ -1,48 +1,571 @@ -"""Client for the logging e2e suite: drive traffic and scrape the proxy's -Prometheus ``/metrics`` endpoint. +"""Client for the logging e2e suite: team/key/org-scoped Langfuse OTEL callbacks, +chat (including tools), Prometheus scrape, and Langfuse observation read-back. -Holds the shared Gateway so the ``resources`` fixture cleans up keys it creates. -``/metrics`` is exposed as plaintext (not a typed JSON body), so scraping goes -through ``transport.probe`` and returns the raw exposition text for a Prometheus -parser to read. +Holds the shared Gateway so the ``resources`` fixture cleans up keys, teams, +users, orgs, and models it creates. External Langfuse reads go through +``e2e_http`` (the only module allowed to call ``requests.*``). + +Uses the ``langfuse_otel`` callback (OTLP to ``{host}/api/public/otel``), not +the classic ``langfuse`` SDK callback. OTEL generations land as name +``litellm_request``; correlate by unique prompt marker and ``user_api_key_alias`` +in metadata. Spend is on ``calculatedTotalCost`` (StandardLogging response_cost). """ from __future__ import annotations +import base64 +import json +import os +import time from dataclasses import dataclass +from typing import Literal +import pytest +from pydantic import BaseModel, ConfigDict, Field + +from e2e_config import POLL_INTERVAL, POLL_TIMEOUT from e2e_gateway import Gateway, build_gateway -from e2e_http import NoBody, unwrap -from models import ChatBody, ChatMessage, ChatResponse, KeyGenerateBody +from e2e_http import ( + URL, + AuthHeaders, + NoBody, + StreamingResponse, + Success, + get, + unwrap, +) +from models import ( + ChatBody, + ChatMessage, + ChatResponse, + ChatTool, + ChatToolFunction, + KeyGenerateBody, + KeyLoggingCallback, + KeyLoggingCallbackVars, + KeyMetadata, + LiteLLMParamsBody, + OrgDeleteBody, + OrgNewBody, + OrgNewResponse, + SpendLogRow, + TeamDeleteBody, + TeamNewBody, + TeamNewResponse, + UserDeleteBody, + UserNewBody, + UserNewResponse, +) + +# Deliberately invalid *upstream provider* key for failure-path tests. +# Not a LiteLLM virtual key; OpenAI must reject it after the proxy accepts the call. +INVALID_UPSTREAM_API_KEY = "sk-upstream-invalid-for-langfuse-e2e-only" + +WEATHER_TOOL = ChatTool( + type="function", + function=ChatToolFunction( + name="get_weather", + description="Get the current weather for a city", + parameters={ + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + ), +) + + +class TeamCallbackBody(BaseModel): + callback_name: Literal["langfuse_otel", "langfuse", "langsmith", "gcs"] + callback_type: Literal["success", "failure", "success_and_failure"] + callback_vars: dict[str, str] + + +class TeamCallbackResponse(BaseModel): + model_config = ConfigDict(extra="ignore") + + status: str + + +class GuardrailLitellmParams(BaseModel): + guardrail: str + mode: str + default_on: bool = False + rules: list[dict[str, object]] | None = None + default_action: str | None = None + on_disallowed_action: str | None = None + + +class GuardrailSpec(BaseModel): + guardrail_name: str + litellm_params: GuardrailLitellmParams + + +class CreateGuardrailBody(BaseModel): + guardrail: GuardrailSpec + + +class CreateGuardrailResponse(BaseModel): + model_config = ConfigDict(extra="ignore") + + guardrail_id: str | None = None + guardrail_name: str | None = None + + +class LangfuseObservation(BaseModel): + model_config = ConfigDict(extra="ignore", populate_by_name=True) + + id: str + trace_id: str | None = Field(default=None, alias="traceId") + name: str | None = None + type: str | None = None + calculated_total_cost: float | None = Field(default=None, alias="calculatedTotalCost") + level: str | None = None + input: object | None = None + output: object | None = None + metadata: object | None = None + usage: object | None = None + usage_details: object | None = Field(default=None, alias="usageDetails") + model: str | None = None + + +class LangfuseObservationList(BaseModel): + model_config = ConfigDict(extra="ignore") + + data: list[LangfuseObservation] = [] + + +class LangfuseListParams(BaseModel): + model_config = ConfigDict(populate_by_name=True) + + limit: int = 100 + trace_id: str | None = Field(default=None, alias="traceId") + name: str | None = None + from_start_time: str | None = Field(default=None, alias="fromStartTime") + + +@dataclass(frozen=True, slots=True) +class LangfuseCreds: + public_key: str + secret_key: str + host: str + + @property + def auth_headers(self) -> AuthHeaders: + token = base64.b64encode(f"{self.public_key}:{self.secret_key}".encode()).decode() + return AuthHeaders(authorization=f"Basic {token}") + + def callback_vars(self) -> dict[str, str]: + return { + "langfuse_public_key": self.public_key, + "langfuse_secret_key": self.secret_key, + "langfuse_host": self.host, + } + + def key_logging_metadata(self) -> KeyMetadata: + return KeyMetadata( + logging=[ + KeyLoggingCallback( + callback_name="langfuse_otel", + callback_type="success_and_failure", + callback_vars=KeyLoggingCallbackVars( + langfuse_public_key=self.public_key, + langfuse_secret_key=self.secret_key, + langfuse_host=self.host, + ), + ) + ] + ) + + +def load_langfuse_creds() -> LangfuseCreds: + public_key = os.getenv("LANGFUSE_PUBLIC_KEY") + secret_key = os.getenv("LANGFUSE_SECRET_KEY") + host = (os.getenv("LANGFUSE_BASE_URL") or os.getenv("LANGFUSE_HOST") or "").rstrip("/") + if not (public_key and secret_key and host): + pytest.fail( + "Langfuse e2e requires LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY, and " + "LANGFUSE_BASE_URL (or LANGFUSE_HOST); missing credentials is a hard failure, not a skip" + ) + return LangfuseCreds(public_key=public_key, secret_key=secret_key, host=host) + + +def observation_spend(obs: LangfuseObservation) -> float | None: + """Langfuse calculatedTotalCost is populated from StandardLogging response_cost.""" + return obs.calculated_total_cost + + +def costs_agree(expected: float, actual: float, *, rel_tol: float = 0.05) -> bool: + """Costs agree within 5% relative (or 1e-9 absolute for near-zero).""" + return abs(expected - actual) <= max(1e-9, abs(expected) * rel_tol) + + +def completion_response_id(body: str) -> str | None: + """SpendLogs.request_id is the chat completion body id, not x-litellm-call-id.""" + if not body or body == "": + return None + try: + parsed = json.loads(body) + except json.JSONDecodeError: + return None + if not isinstance(parsed, dict): + return None + raw = parsed.get("id") + return raw if isinstance(raw, str) and raw else None + + +def _matches_run(obs: LangfuseObservation, *, key_alias: str, prompt_marker: str) -> bool: + """Match a Langfuse generation for this run. + + langfuse_otel names generations ``litellm_request`` (not ``litellm:{alias}``). + Prefer the unique prompt marker in input; fall back to key alias in metadata + (user_api_key_alias) or the classic SDK generation name. + """ + if prompt_marker and prompt_marker in json.dumps(obs.input, default=str): + return True + meta_blob = json.dumps(obs.metadata, default=str) if obs.metadata is not None else "" + if key_alias and key_alias in meta_blob: + return True + if obs.name == f"litellm:{key_alias}": + return True + return False + + +def observation_mentions_tool(obs: LangfuseObservation, tool_name: str) -> bool: + blob = json.dumps( + {"input": obs.input, "output": obs.output, "metadata": obs.metadata}, + default=str, + ) + return tool_name in blob + + +def observation_has_guardrail(obs: LangfuseObservation, *, guardrail_name: str) -> bool: + blob = json.dumps(obs.metadata, default=str) if obs.metadata is not None else "" + if guardrail_name in blob or "guardrail" in blob.lower(): + return True + if obs.name is not None and "guardrail" in obs.name.lower(): + return True + return False @dataclass(frozen=True, slots=True) class LoggingClient: gateway: Gateway - def key_with_alias(self, alias: str, *, models: list[str]) -> str: + def key_with_alias( + self, + alias: str, + *, + models: list[str], + team_id: str | None = None, + user_id: str | None = None, + organization_id: str | None = None, + metadata: KeyMetadata | None = None, + ) -> str: return self.gateway.generate_key( - KeyGenerateBody(key_alias=alias, models=models, user_id=f"e2e-{alias}") + KeyGenerateBody( + key_alias=alias, + models=models, + user_id=user_id or f"e2e-{alias}", + team_id=team_id, + organization_id=organization_id, + metadata=metadata, + ) ) def delete_key(self, key: str) -> None: self.gateway.delete_key(key) + def create_team( + self, + alias: str, + *, + models: list[str], + organization_id: str | None = None, + ) -> str: + return unwrap( + self.gateway.transport.post( + "/team/new", + headers=self.gateway.transport.master, + json=TeamNewBody( + team_alias=alias, + models=models, + organization_id=organization_id, + ), + response_type=TeamNewResponse, + ) + ).team_id + + def delete_team(self, team_id: str) -> None: + _ = self.gateway.transport.post( + "/team/delete", + headers=self.gateway.transport.master, + json=TeamDeleteBody(team_ids=[team_id]), + response_type=NoBody, + ) + + def create_user(self, *, user_email: str, user_id: str | None = None) -> str: + return unwrap( + self.gateway.transport.post( + "/user/new", + headers=self.gateway.transport.master, + json=UserNewBody( + user_email=user_email, + user_role="internal_user", + user_id=user_id, + ), + response_type=UserNewResponse, + ) + ).user_id + + def delete_user(self, user_id: str) -> None: + _ = self.gateway.transport.post( + "/user/delete", + headers=self.gateway.transport.master, + json=UserDeleteBody(user_ids=[user_id]), + response_type=NoBody, + ) + + def create_org(self, alias: str, *, models: list[str]) -> str: + return unwrap( + self.gateway.transport.post( + "/organization/new", + headers=self.gateway.transport.master, + json=OrgNewBody(organization_alias=alias, models=models), + response_type=OrgNewResponse, + ) + ).organization_id + + def delete_org(self, organization_id: str) -> None: + _ = self.gateway.transport.delete( + "/organization/delete", + headers=self.gateway.transport.master, + json=OrgDeleteBody(organization_ids=[organization_id]), + response_type=NoBody, + ) + + def add_team_langfuse_callback( + self, + team_id: str, + creds: LangfuseCreds, + *, + callback_type: Literal["success", "failure", "success_and_failure"] = "success_and_failure", + ) -> None: + response = unwrap( + self.gateway.transport.post( + f"/team/{team_id}/callback", + headers=self.gateway.transport.master, + json=TeamCallbackBody( + callback_name="langfuse_otel", + callback_type=callback_type, + callback_vars=creds.callback_vars(), + ), + response_type=TeamCallbackResponse, + ) + ) + assert response.status == "success", ( + f"POST /team/{team_id}/callback must return status=success; got {response.status!r}" + ) + + def create_tool_permission_guardrail(self, name: str, *, allowed_tool: str) -> str: + """Register a tool_permission guardrail that allows one tool and denies the rest.""" + response = unwrap( + self.gateway.transport.post( + "/guardrails", + headers=self.gateway.transport.master, + json=CreateGuardrailBody( + guardrail=GuardrailSpec( + guardrail_name=name, + litellm_params=GuardrailLitellmParams( + guardrail="tool_permission", + mode="post_call", + default_on=False, + default_action="deny", + on_disallowed_action="block", + rules=[ + { + "id": "allow-named-tool", + "tool_name": allowed_tool, + "decision": "allow", + } + ], + ), + ) + ), + response_type=CreateGuardrailResponse, + ) + ) + guardrail_id = response.guardrail_id + assert guardrail_id, f"create guardrail returned no id: {response!r}" + return guardrail_id + + def delete_guardrail(self, guardrail_id: str) -> None: + _ = self.gateway.transport.delete( + f"/guardrails/{guardrail_id}", + headers=self.gateway.transport.master, + json=NoBody(), + response_type=NoBody, + ) + + def create_model(self, model_name: str, litellm_params: LiteLLMParamsBody) -> str: + return self.gateway.create_model(model_name, litellm_params) + + def delete_model(self, model_id: str) -> None: + self.gateway.delete_model(model_id) + def chat(self, key: str, model: str, text: str) -> ChatResponse: return unwrap( self.gateway.chat( key, ChatBody( model=model, - messages=[ChatMessage(role="user", content=text)], - max_tokens=64, + messages=[ChatMessage(role="user", content=text)], + max_tokens=64, ), ) ) + def chat_raw( + self, + key: str, + model: str, + text: str, + *, + stream: bool = False, + tools: list[ChatTool] | None = None, + tool_choice: str | None = None, + guardrails: list[str] | None = None, + max_tokens: int = 64, + ) -> StreamingResponse: + body = ChatBody( + model=model, + messages=[ChatMessage(role="user", content=text)], + max_tokens=max_tokens, + stream=stream, + tools=tools, + tool_choice=tool_choice, + guardrails=guardrails, + ) + if stream: + return self.gateway.chat_stream(key, body) + return self.gateway.transport.send( + "/chat/completions", + headers=self.gateway.transport.bearer(key), + json=body, + ) + def scrape_metrics(self) -> str: return self.gateway.probe("/metrics", params=NoBody()).body + def poll_proxy_spend_for_key( + self, + key: str, + *, + response_id: str | None = None, + require_positive_spend: bool = True, + ) -> SpendLogRow | None: + """Poll /spend/logs by virtual key. + + When ``response_id`` is set, only that SpendLogs.request_id may match. + When unset, any positive-spend row for the key is accepted. Never falls + back to an unmatched row; missing match returns None. + """ + + def _matches(row: SpendLogRow) -> bool: + if response_id is not None and row.request_id != response_id: + return False + if require_positive_spend and not (row.spend is not None and row.spend > 0): + return False + return True + + rows = self.gateway.poll_logs_for_key( + key, min_rows=1, predicate=lambda rs: any(_matches(r) for r in rs) + ) + for row in rows: + if _matches(row): + return row + return None + + def list_langfuse_observations( + self, + creds: LangfuseCreds, + *, + trace_id: str | None = None, + name: str | None = None, + from_start_time: str | None = None, + ) -> list[LangfuseObservation]: + result = get( + URL(f"{creds.host}/api/public/observations"), + headers=creds.auth_headers, + params=LangfuseListParams( + limit=100, + trace_id=trace_id, + name=name, + from_start_time=from_start_time, + ), + response_type=LangfuseObservationList, + timeout=30.0, + ) + match result: + case Success(data=page): + return page.data + case _: + return [] + + def find_langfuse_observation( + self, + creds: LangfuseCreds, + *, + key_alias: str, + prompt_marker: str, + ) -> LangfuseObservation | None: + # langfuse_otel generations are named litellm_request; classic SDK used + # litellm:{key_alias}. Search both, then a recent unfiltered page. + for name in ("litellm_request", f"litellm:{key_alias}"): + for obs in self.list_langfuse_observations(creds, name=name): + if _matches_run(obs, key_alias=key_alias, prompt_marker=prompt_marker): + return obs + for obs in self.list_langfuse_observations(creds): + if _matches_run(obs, key_alias=key_alias, prompt_marker=prompt_marker): + return obs + return None + + def poll_langfuse_observation( + self, + creds: LangfuseCreds, + *, + key_alias: str, + prompt_marker: str, + require_positive_cost: bool = False, + ) -> LangfuseObservation | None: + deadline = time.monotonic() + POLL_TIMEOUT + last: LangfuseObservation | None = None + while time.monotonic() < deadline: + last = self.find_langfuse_observation( + creds, key_alias=key_alias, prompt_marker=prompt_marker + ) + if last is not None: + cost = observation_spend(last) + if not require_positive_cost or (cost is not None and cost > 0): + return last + time.sleep(POLL_INTERVAL) + return last + + def poll_langfuse_trace_observations( + self, + creds: LangfuseCreds, + *, + key_alias: str, + prompt_marker: str, + ) -> list[LangfuseObservation]: + """Generation plus any sibling/child observations (guardrail spans, etc.).""" + gen = self.poll_langfuse_observation( + creds, key_alias=key_alias, prompt_marker=prompt_marker + ) + if gen is None or not gen.trace_id: + return [] if gen is None else [gen] + return self.list_langfuse_observations(creds, trace_id=gen.trace_id) or [gen] + def build_logging_client() -> LoggingClient: return LoggingClient(gateway=build_gateway()) diff --git a/tests/e2e/logging/test_langfuse_e2e.py b/tests/e2e/logging/test_langfuse_e2e.py new file mode 100644 index 00000000000..d014b5d8291 --- /dev/null +++ b/tests/e2e/logging/test_langfuse_e2e.py @@ -0,0 +1,534 @@ +"""Live e2e: Langfuse OTEL logs_spend for registry cells in logging.yaml P0. + +Registry cells: +- logging.langfuse.success.logs_spend (exercised_on chat_completions, messages, embeddings) +- logging.langfuse.failure.logs_spend (exercised_on chat_completions, messages) +- logging.langfuse.stream.logs_spend (exercised_on chat_completions, messages) + +Integration under test is ``langfuse_otel`` (OTLP to Langfuse), not the classic +``langfuse`` SDK callback. StandardLoggingPayload.response_cost is the spend +source of truth. Generations are named ``litellm_request``; correlate by unique +prompt marker and user_api_key_alias in metadata. + +Dynamic credentials by product surface: +- team: POST /team/{id}/callback with callback_name=langfuse_otel +- user/key: key metadata.logging with callback_name=langfuse_otel +- org: organization + team under it + team callback (no org-level callback API) + +Extra success paths assert tool calls and applied guardrails land on the trace. +""" + +from __future__ import annotations + +import json + +import pytest + +from e2e_config import unique_marker +from e2e_http import StreamingResponse, require_successful_call +from lifecycle import ResourceManager +from logging_client import ( + INVALID_UPSTREAM_API_KEY, + WEATHER_TOOL, + LangfuseCreds, + LoggingClient, + completion_response_id, + costs_agree, + observation_has_guardrail, + observation_mentions_tool, + observation_spend, +) +from models import LiteLLMParamsBody + +pytestmark = pytest.mark.e2e + +DRIVER_MODEL = "gemini-2.5-flash" +FAIL_BACKEND = "openai/gpt-4o-mini" + + +def _json_blob(value: object) -> str: + return json.dumps(value, default=str) + + +def _assert_logs_spend( + client: LoggingClient, + *, + key: str, + outcome: StreamingResponse, + obs_cost: float | None, + scope: str, + require_positive: bool = True, +) -> None: + """logs_spend: Langfuse cost matches StandardLogging response_cost and proxy spend. + + Non-stream responses expose response_cost on x-litellm-response-cost. Streaming + sends headers before final cost is known, so stream paths rely on /spend/logs. + """ + if not require_positive: + assert obs_cost is not None, ( + f"{scope}: failure path must still track spend (0 is fine); cost={obs_cost!r}" + ) + return + + assert obs_cost is not None and obs_cost > 0, ( + f"{scope}: Langfuse must log positive spend; calculatedTotalCost={obs_cost!r}" + ) + # Stream responses send headers before final cost is known, so the cost header + # is often absent; non-stream must always expose x-litellm-response-cost. + if not outcome.is_streaming: + assert outcome.response_cost is not None and outcome.response_cost > 0, ( + f"{scope}: proxy must return positive x-litellm-response-cost; " + f"got {outcome.response_cost!r}" + ) + assert costs_agree(outcome.response_cost, obs_cost), ( + f"{scope}: Langfuse cost {obs_cost!r} disagrees with " + f"x-litellm-response-cost {outcome.response_cost!r}" + ) + elif outcome.response_cost is not None and outcome.response_cost > 0: + assert costs_agree(outcome.response_cost, obs_cost), ( + f"{scope}: Langfuse cost {obs_cost!r} disagrees with " + f"x-litellm-response-cost {outcome.response_cost!r}" + ) + spend_row = client.poll_proxy_spend_for_key( + key, + response_id=completion_response_id(outcome.body), + require_positive_spend=True, + ) + assert spend_row is not None and spend_row.spend is not None and spend_row.spend > 0, ( + f"{scope}: proxy /spend/logs never produced a positive spend row for key" + ) + assert costs_agree(spend_row.spend, obs_cost), ( + f"{scope}: Langfuse cost {obs_cost!r} disagrees with proxy spend " + f"{spend_row.spend!r} (request_id={spend_row.request_id!r})" + ) + + +class TestLangfuseTeamLogging: + """Team-scoped callback via POST /team/{id}/callback.""" + + def _team_key( + self, + client: LoggingClient, + resources: ResourceManager, + creds: LangfuseCreds, + *, + models: list[str], + organization_id: str | None = None, + ) -> tuple[str, str, str]: + marker = unique_marker() + key_alias = f"e2e-lf-team-key-{marker}" + team_id = client.create_team( + f"e2e-lf-team-{marker}", + models=models, + organization_id=organization_id, + ) + resources.defer(lambda: client.delete_team(team_id)) + client.add_team_langfuse_callback(team_id, creds) + key = client.key_with_alias(key_alias, models=models, team_id=team_id) + resources.defer(lambda: client.delete_key(key)) + return team_id, key, key_alias + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["chat_completions"]) + def test_success_logs_spend( + self, + client: LoggingClient, + resources: ResourceManager, + langfuse_creds: LangfuseCreds, + ) -> None: + _, key, key_alias = self._team_key( + client, resources, langfuse_creds, models=[DRIVER_MODEL] + ) + prompt_marker = unique_marker() + outcome = client.chat_raw( + key, DRIVER_MODEL, f"reply with one word only {prompt_marker}" + ) + require_successful_call(outcome) + + obs = client.poll_langfuse_observation( + langfuse_creds, + key_alias=key_alias, + prompt_marker=prompt_marker, + require_positive_cost=True, + ) + assert obs is not None, ( + f"team scope: Langfuse never received generation for key_alias={key_alias!r}" + ) + _assert_logs_spend( + client, + key=key, + outcome=outcome, + obs_cost=observation_spend(obs), + scope="team-success", + ) + + @pytest.mark.covers("logging.langfuse.failure.logs_spend", exercised_on=["chat_completions"]) + def test_failure_logs_spend( + self, + client: LoggingClient, + resources: ResourceManager, + langfuse_creds: LangfuseCreds, + ) -> None: + """Provider-auth failure still ships a Langfuse observation with spend tracked. + + Uses a throwaway deployment whose upstream OpenAI key is + INVALID_UPSTREAM_API_KEY (not a LiteLLM virtual key). + """ + prompt_marker = unique_marker() + model_name = f"e2e-lf-fail-{prompt_marker}" + model_id = client.create_model( + model_name, + LiteLLMParamsBody(model=FAIL_BACKEND, api_key=INVALID_UPSTREAM_API_KEY), + ) + resources.defer(lambda: client.delete_model(model_id)) + + _, key, key_alias = self._team_key( + client, resources, langfuse_creds, models=[model_name] + ) + outcome = client.chat_raw(key, model_name, f"this must fail {prompt_marker}") + assert not outcome.ok, ( + f"expected upstream provider failure for {INVALID_UPSTREAM_API_KEY!r}, " + f"got {outcome.status_code}: {outcome.body[:200]}" + ) + + obs = client.poll_langfuse_observation( + langfuse_creds, + key_alias=key_alias, + prompt_marker=prompt_marker, + require_positive_cost=False, + ) + assert obs is not None, ( + f"team failure path: Langfuse never received generation for key_alias={key_alias!r}" + ) + _assert_logs_spend( + client, + key=key, + outcome=outcome, + obs_cost=observation_spend(obs), + scope="team-failure", + require_positive=False, + ) + + @pytest.mark.covers("logging.langfuse.stream.logs_spend", exercised_on=["chat_completions"]) + def test_stream_logs_spend( + self, + client: LoggingClient, + resources: ResourceManager, + langfuse_creds: LangfuseCreds, + ) -> None: + _, key, key_alias = self._team_key( + client, resources, langfuse_creds, models=[DRIVER_MODEL] + ) + prompt_marker = unique_marker() + outcome = client.chat_raw( + key, DRIVER_MODEL, f"reply with one word only {prompt_marker}", stream=True + ) + require_successful_call(outcome) + assert outcome.is_streaming + assert outcome.chunks > 0 + + obs = client.poll_langfuse_observation( + langfuse_creds, + key_alias=key_alias, + prompt_marker=prompt_marker, + require_positive_cost=True, + ) + assert obs is not None + # Streamed body is elided; correlate cost via header + key spend row. + _assert_logs_spend( + client, + key=key, + outcome=outcome, + obs_cost=observation_spend(obs), + scope="team-stream", + ) + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["chat_completions"]) + def test_tool_calls_logged_with_cost( + self, + client: LoggingClient, + resources: ResourceManager, + langfuse_creds: LangfuseCreds, + ) -> None: + _, key, key_alias = self._team_key( + client, resources, langfuse_creds, models=[DRIVER_MODEL] + ) + prompt_marker = unique_marker() + outcome = client.chat_raw( + key, + DRIVER_MODEL, + f"Use get_weather for Paris. marker={prompt_marker}", + tools=[WEATHER_TOOL], + tool_choice="required", + max_tokens=128, + ) + require_successful_call(outcome) + assert "get_weather" in outcome.body or "tool_calls" in outcome.body, ( + f"gateway response must include a tool call; body={outcome.body[:300]}" + ) + + obs = client.poll_langfuse_observation( + langfuse_creds, + key_alias=key_alias, + prompt_marker=prompt_marker, + require_positive_cost=True, + ) + assert obs is not None + assert observation_mentions_tool(obs, "get_weather"), ( + f"Langfuse generation must record the tool; name={obs.name!r} " + f"input={str(obs.input)[:200]} output={str(obs.output)[:200]}" + ) + _assert_logs_spend( + client, + key=key, + outcome=outcome, + obs_cost=observation_spend(obs), + scope="team-tools", + ) + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["chat_completions"]) + def test_tool_permission_guardrail_logged( + self, + client: LoggingClient, + resources: ResourceManager, + langfuse_creds: LangfuseCreds, + ) -> None: + """tool_permission post_call guardrail must appear on the Langfuse trace + (StandardLogging guardrail_information -> Langfuse guardrail span).""" + marker = unique_marker() + guardrail_name = f"e2e-lf-tool-perm-{marker}" + guardrail_id = client.create_tool_permission_guardrail( + guardrail_name, allowed_tool="get_weather" + ) + resources.defer(lambda: client.delete_guardrail(guardrail_id)) + + _, key, key_alias = self._team_key( + client, resources, langfuse_creds, models=[DRIVER_MODEL] + ) + prompt_marker = unique_marker() + outcome = client.chat_raw( + key, + DRIVER_MODEL, + f"Use get_weather for Berlin. marker={prompt_marker}", + tools=[WEATHER_TOOL], + tool_choice="required", + guardrails=[guardrail_name], + max_tokens=128, + ) + require_successful_call(outcome) + + observations = client.poll_langfuse_trace_observations( + langfuse_creds, key_alias=key_alias, prompt_marker=prompt_marker + ) + assert observations, ( + f"team+guardrail: no Langfuse observations for key_alias={key_alias!r}" + ) + gen = next( + ( + o + for o in observations + if prompt_marker in _json_blob(o.input) + or key_alias in _json_blob(o.metadata) + or o.name in (f"litellm:{key_alias}", "litellm_request") + ), + observations[0], + ) + _assert_logs_spend( + client, + key=key, + outcome=outcome, + obs_cost=observation_spend(gen), + scope="team-guardrail", + ) + assert any( + observation_has_guardrail(o, guardrail_name=guardrail_name) + or (o.name is not None and "guardrail" in o.name.lower()) + for o in observations + ), ( + f"Langfuse trace must include applied guardrail {guardrail_name!r}; " + f"observation names={[o.name for o in observations]}" + ) + + +class TestLangfuseUserKeyLogging: + """User-owned key with metadata.logging (key-level dynamic Langfuse credentials). + + Product surface: key metadata.logging on /key/generate, not a separate + /user/.../callback route. The key is bound to a real /user/new user_id. + """ + + def _user_key( + self, + client: LoggingClient, + resources: ResourceManager, + creds: LangfuseCreds, + *, + models: list[str], + ) -> tuple[str, str, str]: + marker = unique_marker() + key_alias = f"e2e-lf-user-key-{marker}" + user_id = client.create_user( + user_email=f"e2e-lf-user-{marker}@example.com", + user_id=f"e2e-lf-user-{marker}", + ) + resources.defer(lambda: client.delete_user(user_id)) + key = client.key_with_alias( + key_alias, + models=models, + user_id=user_id, + metadata=creds.key_logging_metadata(), + ) + resources.defer(lambda: client.delete_key(key)) + return user_id, key, key_alias + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["chat_completions"]) + def test_success_logs_spend( + self, + client: LoggingClient, + resources: ResourceManager, + langfuse_creds: LangfuseCreds, + ) -> None: + user_id, key, key_alias = self._user_key( + client, resources, langfuse_creds, models=[DRIVER_MODEL] + ) + prompt_marker = unique_marker() + outcome = client.chat_raw( + key, DRIVER_MODEL, f"reply with one word only {prompt_marker}" + ) + require_successful_call(outcome) + + obs = client.poll_langfuse_observation( + langfuse_creds, + key_alias=key_alias, + prompt_marker=prompt_marker, + require_positive_cost=True, + ) + assert obs is not None, ( + f"user/key scope: Langfuse never received generation for key_alias={key_alias!r}" + ) + meta_blob = _json_blob(obs.metadata) + assert user_id in meta_blob or key_alias in (obs.name or ""), ( + f"user/key scope should attribute the user or key; metadata={meta_blob[:300]}" + ) + _assert_logs_spend( + client, + key=key, + outcome=outcome, + obs_cost=observation_spend(obs), + scope="user-key", + ) + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["chat_completions"]) + def test_tool_calls_logged_with_cost( + self, + client: LoggingClient, + resources: ResourceManager, + langfuse_creds: LangfuseCreds, + ) -> None: + _, key, key_alias = self._user_key( + client, resources, langfuse_creds, models=[DRIVER_MODEL] + ) + prompt_marker = unique_marker() + outcome = client.chat_raw( + key, + DRIVER_MODEL, + f"Use get_weather for Tokyo. marker={prompt_marker}", + tools=[WEATHER_TOOL], + tool_choice="required", + max_tokens=128, + ) + require_successful_call(outcome) + + obs = client.poll_langfuse_observation( + langfuse_creds, + key_alias=key_alias, + prompt_marker=prompt_marker, + require_positive_cost=True, + ) + assert obs is not None + assert observation_mentions_tool(obs, "get_weather"), ( + f"user/key tool path: tool missing from Langfuse; output={str(obs.output)[:200]}" + ) + _assert_logs_spend( + client, + key=key, + outcome=outcome, + obs_cost=observation_spend(obs), + scope="user-key-tools", + ) + + +class TestLangfuseOrgScopedLogging: + """Org-scoped run: organization + team under it + team Langfuse callback. + + There is no /organization/.../callback today; logging attaches at the team + (or key) under the org. This class proves org-linked team keys still deliver + accurate Langfuse spend and team attribution (StandardLogging metadata + user_api_key_team_id / user_api_key_org_id). + """ + + def _org_team_key( + self, + client: LoggingClient, + resources: ResourceManager, + creds: LangfuseCreds, + *, + models: list[str], + ) -> tuple[str, str, str, str]: + marker = unique_marker() + key_alias = f"e2e-lf-org-key-{marker}" + org_id = client.create_org(f"e2e-lf-org-{marker}", models=models) + resources.defer(lambda: client.delete_org(org_id)) + team_id = client.create_team( + f"e2e-lf-org-team-{marker}", + models=models, + organization_id=org_id, + ) + resources.defer(lambda: client.delete_team(team_id)) + client.add_team_langfuse_callback(team_id, creds) + key = client.key_with_alias( + key_alias, + models=models, + team_id=team_id, + organization_id=org_id, + ) + resources.defer(lambda: client.delete_key(key)) + return org_id, team_id, key, key_alias + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["chat_completions"]) + def test_success_logs_spend_with_team_attribution( + self, + client: LoggingClient, + resources: ResourceManager, + langfuse_creds: LangfuseCreds, + ) -> None: + org_id, team_id, key, key_alias = self._org_team_key( + client, resources, langfuse_creds, models=[DRIVER_MODEL] + ) + prompt_marker = unique_marker() + outcome = client.chat_raw( + key, DRIVER_MODEL, f"reply with one word only {prompt_marker}" + ) + require_successful_call(outcome) + + obs = client.poll_langfuse_observation( + langfuse_creds, + key_alias=key_alias, + prompt_marker=prompt_marker, + require_positive_cost=True, + ) + assert obs is not None, ( + f"org scope: Langfuse never received generation for key_alias={key_alias!r}" + ) + meta_blob = _json_blob(obs.metadata) + assert team_id in meta_blob, ( + f"org-scoped team key must stamp team_id on Langfuse metadata; " + f"team_id={team_id!r} metadata={meta_blob[:400]}" + ) + _ = org_id + _assert_logs_spend( + client, + key=key, + outcome=outcome, + obs_cost=observation_spend(obs), + scope="org-team", + ) diff --git a/tests/e2e/management/management_client.py b/tests/e2e/management/management_client.py index bc60ce87c98..144a3c0c435 100644 --- a/tests/e2e/management/management_client.py +++ b/tests/e2e/management/management_client.py @@ -75,7 +75,7 @@ class ManagementClient: case _: break assert last is not None - _ = unwrap(last) + raise AssertionError(last) def delete_key_strict(self, key: str) -> None: """Strict delete for the act phase of a test: a failed delete is a hard @@ -147,7 +147,7 @@ class ManagementClient: case _: time.sleep(_TEAM_READY_SLEEP_SECONDS) assert last is not None - _ = unwrap(last) + raise AssertionError(last) def add_team_member(self, team_id: str, user_id: str) -> None: last: Result[NoBody] | None = None @@ -169,7 +169,7 @@ class ManagementClient: case _: break assert last is not None - _ = unwrap(last) + raise AssertionError(last) def delete_team_member(self, team_id: str, user_id: str) -> None: _ = unwrap( diff --git a/tests/e2e/models.py b/tests/e2e/models.py index ab2835d87c4..e32f2709181 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -23,6 +23,22 @@ class BudgetWindow(BaseModel): max_budget: float +class KeyLoggingCallbackVars(BaseModel): + langfuse_public_key: str | None = None + langfuse_secret_key: str | None = None + langfuse_host: str | None = None + + +class KeyLoggingCallback(BaseModel): + callback_name: str + callback_type: str = "success_and_failure" + callback_vars: KeyLoggingCallbackVars + + +class KeyMetadata(BaseModel): + logging: list[KeyLoggingCallback] | None = None + + class KeyGenerateBody(BaseModel): models: list[str] = [] duration: str | None = None @@ -31,6 +47,7 @@ class KeyGenerateBody(BaseModel): budget_duration: str | None = None user_id: str | None = None team_id: str | None = None + organization_id: str | None = None budget_id: str | None = None key_alias: str | None = None model_max_budget: dict[str, ModelBudgetEntry] | None = None @@ -39,6 +56,7 @@ class KeyGenerateBody(BaseModel): tpm_limit: int | None = None rpm_limit: int | None = None allowed_routes: list[str] | None = None + metadata: KeyMetadata | None = None class KeyGenerateResponse(BaseModel): @@ -105,6 +123,17 @@ class ThinkingParam(BaseModel): budget_tokens: int | None = None +class ChatToolFunction(BaseModel): + name: str + description: str | None = None + parameters: dict[str, object] | None = None + + +class ChatTool(BaseModel): + type: str = "function" + function: ChatToolFunction + + class ChatBody(BaseModel): model: str messages: list[ChatMessage] @@ -115,6 +144,9 @@ class ChatBody(BaseModel): reasoning_effort: str | None = None thinking: ThinkingParam | None = None service_tier: str | None = None + tools: list[ChatTool] | None = None + tool_choice: str | None = None + guardrails: list[str] | None = None class AnthropicMessagesBody(BaseModel): @@ -123,6 +155,10 @@ class AnthropicMessagesBody(BaseModel): max_tokens: int +class AnthropicMessagesResponse(BaseModel): + model: str | None = None + + class OutMessage(BaseModel): content: str | None = None reasoning_content: str | None = None @@ -459,6 +495,7 @@ class TeamNewBody(BaseModel): team_alias: str models: list[str] = [] team_id: str | None = None + organization_id: str | None = None class TeamNewResponse(BaseModel): diff --git a/tests/local_testing/test_get_llm_provider.py b/tests/local_testing/test_get_llm_provider.py index 9ecb639fc3a..4c3e13da17a 100644 --- a/tests/local_testing/test_get_llm_provider.py +++ b/tests/local_testing/test_get_llm_provider.py @@ -509,10 +509,10 @@ def shipped_generalizations(): class TestClaudeModelPatternMatching: """ - The ``anthropic-claude`` fallback generalization rule routes future Claude - models to the Anthropic provider without requiring a + The ``anthropic-claude-ids`` fallback generalization routing rule routes future + Claude models to the Anthropic provider without requiring a model_prices_and_context_window.json entry. These tests exercise the rule - end-to-end through ``get_llm_provider`` and ``match_fallback_generalization``. + end-to-end through ``get_llm_provider`` and ``match_routing_generalization``. """ @pytest.mark.parametrize( @@ -556,10 +556,10 @@ class TestClaudeModelPatternMatching: self, model, shipped_generalizations ): from litellm.litellm_core_utils.fallback_generalizations import ( - match_fallback_generalization, + match_routing_generalization, ) - assert match_fallback_generalization(model) is None + assert match_routing_generalization(model) is None def test_routing_comes_from_the_rule_not_python(self, shipped_generalizations): """With the rule cleared, an unknown claude must no longer route to diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py index cf74bed9c15..0ea09463c16 100644 --- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py @@ -1,11 +1,13 @@ """ Tests for the declarative fallback-generalizations mechanism. -Covers both the pure module (litellm.litellm_core_utils.fallback_generalizations) -and its end-to-end wiring into provider routing (get_llm_provider) and model-info -resolution (get_model_info / supports_*). +Covers the pure module (litellm.litellm_core_utils.fallback_generalizations): the +routing/capability rule split, install-time validation, capability unioning; and +its end-to-end wiring into provider routing (get_llm_provider) and model-info +resolution (get_model_info) including the shipped rules in the bundled cost map. """ +import logging import os import sys @@ -14,10 +16,11 @@ import pytest sys.path.insert(0, os.path.abspath("../../..")) import litellm +from litellm._logging import verbose_logger from litellm.litellm_core_utils.fallback_generalizations import ( get_fallback_generalization_rules, - match_all_fallback_generalizations, - match_fallback_generalization, + match_capability_generalizations, + match_routing_generalization, set_fallback_generalizations, ) @@ -32,92 +35,205 @@ def restore_generalizations(): set_fallback_generalizations(previous) +class _RecordingHandler(logging.Handler): + def __init__(self): + super().__init__(level=logging.WARNING) + self.messages = [] + + def emit(self, record): + self.messages.append(record.getMessage()) + + +@pytest.fixture +def warning_messages(): + handler = _RecordingHandler() + previous_level = verbose_logger.level + verbose_logger.setLevel(logging.WARNING) + verbose_logger.addHandler(handler) + try: + yield handler.messages + finally: + verbose_logger.removeHandler(handler) + verbose_logger.setLevel(previous_level) + + # --------------------------------------------------------------------------- # -# Pure module behaviour +# Engine: routing rules # --------------------------------------------------------------------------- # -def test_match_returns_model_info_of_first_matching_rule(restore_generalizations): +def test_routing_inference_first_match_wins(restore_generalizations): + restore_generalizations( + [ + {"name": "first", "pattern": r"^acme-", "model_info": {"litellm_provider": "openai"}}, + {"name": "second", "pattern": r"^acme-pro-", "model_info": {"litellm_provider": "anthropic"}}, + ] + ) + assert match_routing_generalization("acme-pro-1") == "openai" + assert match_routing_generalization("gpt-4o") is None + assert match_routing_generalization("") is None + + +def test_capability_rules_do_not_route(restore_generalizations): + restore_generalizations([{"name": "caps", "pattern": r"^acme-", "model_info": {"supports_vision": True}}]) + assert match_routing_generalization("acme-pro-1") is None + + +def test_routing_match_is_case_insensitive(restore_generalizations): + restore_generalizations( + [{"name": "r", "pattern": r"^claude-opus", "model_info": {"litellm_provider": "anthropic"}}] + ) + assert match_routing_generalization("CLAUDE-OPUS-9-9") == "anthropic" + + +# --------------------------------------------------------------------------- # +# Engine: capability rules +# --------------------------------------------------------------------------- # + + +def test_capability_union_is_last_wins_in_file_order(restore_generalizations): restore_generalizations( [ { - "name": "first", + "name": "broad", "pattern": r"^acme-", - "model_info": {"litellm_provider": "openai", "tag": "first"}, + "model_info": {"mode": "chat", "supports_vision": True, "max_input_tokens": 1000}, }, { - "name": "second", + "name": "narrow", "pattern": r"^acme-pro-", - "model_info": {"litellm_provider": "anthropic", "tag": "second"}, + "model_info": {"supports_vision": False, "supports_reasoning": True}, }, ] ) - # Both rules match "acme-pro-1"; first-in-list wins (documented precedence). - matched = match_fallback_generalization("acme-pro-1") - assert matched is not None - assert matched["tag"] == "first" + assert match_capability_generalizations("acme-pro-1") == { + "mode": "chat", + "supports_vision": False, + "max_input_tokens": 1000, + "supports_reasoning": True, + } + assert match_capability_generalizations("acme-basic-1") == { + "mode": "chat", + "supports_vision": True, + "max_input_tokens": 1000, + } -def test_match_all_returns_every_matching_rule_in_order(restore_generalizations): +def test_routing_rules_are_excluded_from_capability_results(restore_generalizations): restore_generalizations( [ - { - "name": "first", - "pattern": r"^acme-", - "model_info": {"litellm_provider": "openai", "tag": "first"}, - }, - { - "name": "second", - "pattern": r"^acme-pro-", - "model_info": {"litellm_provider": "anthropic", "tag": "second"}, - }, + {"name": "route", "pattern": r"^acme-", "model_info": {"litellm_provider": "openai"}}, + {"name": "caps", "pattern": r"^acme-pro-", "model_info": {"supports_vision": True}}, ] ) - assert [m["tag"] for m in match_all_fallback_generalizations("acme-pro-1")] == ["first", "second"] - assert match_all_fallback_generalizations("gpt-4o") == [] + assert match_capability_generalizations("acme-pro-1") == {"supports_vision": True} + assert match_capability_generalizations("acme-basic-1") is None -def test_provider_scoped_rule_is_skipped_for_other_providers(restore_generalizations): - """Model-info resolution must fall through a provider-mismatched earlier rule to a - later applicable one, instead of discarding the model name at the first pattern hit.""" - restore_generalizations( - [ - { - "name": "bedrock-scoped", - "pattern": r"^acme-", - "model_info": {"litellm_provider": "bedrock", "supports_vision": False}, - }, - { - "name": "openai-scoped", - "pattern": r"^acme-", - "model_info": {"litellm_provider": "openai", "mode": "chat", "supports_vision": True}, - }, - ] - ) - litellm.get_model_info.cache_clear() - info = litellm.get_model_info("acme-fast-1", custom_llm_provider="openai") - assert info["litellm_provider"] == "openai" - assert info["supports_vision"] is True - - -def test_match_is_case_insensitive(restore_generalizations): - restore_generalizations( - [{"name": "r", "pattern": r"^claude-opus", "model_info": {"ok": True}}] - ) - assert match_fallback_generalization("CLAUDE-OPUS-9-9") == {"ok": True} - - -def test_no_match_returns_none(restore_generalizations): - restore_generalizations( - [{"name": "r", "pattern": r"^claude-", "model_info": {"ok": True}}] - ) - assert match_fallback_generalization("gpt-4o") is None - assert match_fallback_generalization("") is None - - -def test_empty_rules_match_nothing(restore_generalizations): +def test_no_capability_match_returns_none(restore_generalizations): + restore_generalizations([{"name": "r", "pattern": r"^claude-", "model_info": {"ok": True}}]) + assert match_capability_generalizations("gpt-4o") is None + assert match_capability_generalizations("") is None restore_generalizations([]) - assert match_fallback_generalization("claude-opus-9-9") is None + assert match_capability_generalizations("claude-opus-9-9") is None + + +def test_reinstalling_rules_replaces_compiled_rules(restore_generalizations): + restore_generalizations([{"name": "r", "pattern": r"^aaa", "model_info": {"v": 1}}]) + assert match_capability_generalizations("aaa-1") == {"v": 1} + set_fallback_generalizations([{"name": "r", "pattern": r"^bbb", "model_info": {"v": 2}}]) + assert match_capability_generalizations("aaa-1") is None + assert match_capability_generalizations("bbb-1") == {"v": 2} + + +# --------------------------------------------------------------------------- # +# Engine: install-time validation and legacy-schema shim +# --------------------------------------------------------------------------- # + + +def test_legacy_mixed_rule_acts_as_both_kinds(restore_generalizations): + """A legacy rule mixing ``litellm_provider`` with capability keys routes AND + contributes its full model_info (provider included) to the capability union.""" + restore_generalizations( + [ + { + "name": "legacy-mixed", + "pattern": r"^acme-", + "model_info": {"litellm_provider": "anthropic", "supports_vision": True}, + }, + {"name": "new-caps", "pattern": r"^acme-pro-", "model_info": {"supports_reasoning": True}}, + ] + ) + assert match_routing_generalization("acme-pro-1") == "anthropic" + assert match_capability_generalizations("acme-pro-1") == { + "litellm_provider": "anthropic", + "supports_vision": True, + "supports_reasoning": True, + } + + +LEGACY_MAIN_RULES = [ + { + "name": "anthropic-claude-adaptive-thinking", + "pattern": "(?:opus|sonnet|haiku)[-._](?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d{1,})[-._]\\d{1,2}(?!\\d))", + "description": "Claude opus/sonnet/haiku at version 4.6 or higher: 4.6 through 4.99, then any 5.x, 6.x or later major. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new families with no code change.", + "extends": "anthropic-claude", + "model_info": {"supports_adaptive_thinking": True}, + }, + { + "name": "anthropic-claude", + "pattern": "^claude-[a-z]+-\\d+[-.]\\d+(?:-\\d{8})?$", + "description": "Any Claude family-major-minor id, optionally with an 8-digit date suffix, anchored to the whole name. Version-neutral fallback that gives an unmapped Claude provider routing and baseline capabilities; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.", + "model_info": { + "litellm_provider": "anthropic", + "mode": "chat", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "supports_function_calling": True, + "supports_parallel_function_calling": True, + "supports_vision": True, + "supports_tool_choice": True, + "supports_assistant_prefill": True, + "supports_prompt_caching": True, + "supports_response_schema": True, + "supports_reasoning": True, + "supports_pdf_input": True, + "supports_system_messages": True, + }, + }, +] + + +def test_legacy_main_schema_keeps_unmapped_claude_working(restore_generalizations): + """Pins the remote-map transition window: a released proxy running this engine + against main's old-schema block (mixed provider+capability rule plus ``extends``, + copied verbatim above) must keep unmapped-Claude inference and info resolution + working until the new-schema JSON reaches main.""" + restore_generalizations([dict(rule) for rule in LEGACY_MAIN_RULES]) + litellm.get_model_info.cache_clear() + + _, provider, _, _ = litellm.get_llm_provider(model="claude-opus-9-9") + assert provider == "anthropic" + + info = litellm.get_model_info("claude-opus-9-9") + assert info["litellm_provider"] == "anthropic" + assert info["supports_adaptive_thinking"] is True + assert info["supports_function_calling"] is True + assert info["max_input_tokens"] == 200000 + assert not info.get("input_cost_per_token") + + low = litellm.get_model_info("claude-opus-4-0") + assert low["litellm_provider"] == "anthropic" + assert low["supports_function_calling"] is True + assert low.get("supports_adaptive_thinking") is None + + +def test_non_string_provider_rule_warns_and_is_skipped(restore_generalizations, warning_messages): + restore_generalizations([{"name": "bad-provider", "pattern": r"^acme-", "model_info": {"litellm_provider": 42}}]) + assert any("bad-provider" in message for message in warning_messages) + assert match_routing_generalization("acme-1") is None + assert match_capability_generalizations("acme-1") is None def test_malformed_rules_are_skipped_not_fatal(restore_generalizations): @@ -131,69 +247,7 @@ def test_malformed_rules_are_skipped_not_fatal(restore_generalizations): {"name": "good", "pattern": r"^claude-", "model_info": {"good": True}}, ] ) - # Non-dict entries and dicts with bad fields are all skipped; the one - # valid rule still matches. - assert match_fallback_generalization("claude-opus-9-9") == {"good": True} - - -def test_setting_rules_invalidates_compiled_cache(restore_generalizations): - restore_generalizations([{"name": "r", "pattern": r"^aaa", "model_info": {"v": 1}}]) - assert match_fallback_generalization("aaa-1") == {"v": 1} - # Re-install different rules; the compiled cache must be rebuilt. - set_fallback_generalizations( - [{"name": "r", "pattern": r"^bbb", "model_info": {"v": 2}}] - ) - assert match_fallback_generalization("aaa-1") is None - assert match_fallback_generalization("bbb-1") == {"v": 2} - - -def test_extends_inherits_parent_and_own_overrides(restore_generalizations): - """A rule's ``extends`` pulls in the parent's model_info; its own keys win on conflict, - so a narrow rule carries only its delta instead of duplicating the parent.""" - restore_generalizations( - [ - { - "name": "base", - "pattern": r"^base-only$", - "model_info": { - "litellm_provider": "anthropic", - "input_cost_per_token": 5e-06, - "supports_vision": True, - }, - }, - { - "name": "child", - "pattern": r"^kid-", - "extends": "base", - "model_info": { - "supports_adaptive_thinking": True, - "supports_vision": False, - }, - }, - ] - ) - matched = match_fallback_generalization("kid-1") - assert matched == { - "litellm_provider": "anthropic", - "input_cost_per_token": 5e-06, - "supports_vision": False, - "supports_adaptive_thinking": True, - } - - -def test_extends_with_unknown_parent_keeps_own_model_info(restore_generalizations): - """A dangling ``extends`` is non-fatal: the rule resolves to its own model_info.""" - restore_generalizations( - [ - { - "name": "orphan", - "pattern": r"^orphan-", - "extends": "does-not-exist", - "model_info": {"litellm_provider": "openai"}, - } - ] - ) - assert match_fallback_generalization("orphan-1") == {"litellm_provider": "openai"} + assert match_capability_generalizations("claude-opus-9-9") == {"good": True} # --------------------------------------------------------------------------- # @@ -201,91 +255,61 @@ def test_extends_with_unknown_parent_keeps_own_model_info(restore_generalization # --------------------------------------------------------------------------- # -@pytest.fixture -def myco_rule(restore_generalizations): - """A self-contained rule carrying provider, pricing, context and capabilities.""" +def test_unknown_model_routes_via_routing_rule(restore_generalizations): + restore_generalizations([{"name": "myco", "pattern": r"^myco-", "model_info": {"litellm_provider": "openai"}}]) + _, provider, _, _ = litellm.get_llm_provider(model="myco-fast-1") + assert provider == "openai" + + +def test_capability_info_backfills_requested_provider(restore_generalizations): restore_generalizations( [ { - "name": "myco", - "pattern": r"^myco-[a-z]+-\d+$", + "name": "beeco-caps", + "pattern": r"^beeco-[a-z]+-\d+$", "model_info": { - "litellm_provider": "openai", "mode": "chat", - "input_cost_per_token": 1e-06, - "output_cost_per_token": 2e-06, "max_input_tokens": 12345, - "max_output_tokens": 678, "supports_vision": True, "supports_function_calling": True, }, } ] ) - return "myco-fast-1" - - -def test_unknown_model_routes_via_rule(myco_rule): - _, provider, _, _ = litellm.get_llm_provider(model=myco_rule) - assert provider == "openai" - - -def test_unknown_model_gets_pricing_context_and_capabilities(myco_rule): - info = litellm.get_model_info(myco_rule) - assert info["litellm_provider"] == "openai" - assert info["input_cost_per_token"] == 1e-06 - assert info["output_cost_per_token"] == 2e-06 + litellm.get_model_info.cache_clear() + info = litellm.get_model_info("beeco-fast-1", custom_llm_provider="groq") + assert info["litellm_provider"] == "groq" assert info["max_input_tokens"] == 12345 assert info["supports_vision"] is True + other = litellm.get_model_info("beeco-fast-1", custom_llm_provider="openai") + assert other["litellm_provider"] == "openai" -def test_supports_helper_reads_through_generalization(myco_rule): - assert litellm.supports_vision(myco_rule) is True - assert litellm.supports_function_calling(myco_rule) is True +def test_routing_only_match_does_not_resolve_model_info(restore_generalizations): + restore_generalizations([{"name": "route", "pattern": r"^ceeco-", "model_info": {"litellm_provider": "openai"}}]) + litellm.get_model_info.cache_clear() + with pytest.raises(Exception): + litellm.get_model_info("ceeco-fast-1", custom_llm_provider="openai") def test_exact_entry_takes_precedence_over_rule(restore_generalizations): - """An exact cost-map entry must win over a rule that also matches it.""" restore_generalizations( - [ - { - "name": "shadow-gpt4o", - "pattern": r"^gpt-4o$", - "model_info": { - "litellm_provider": "anthropic", - "input_cost_per_token": 999.0, - }, - } - ] + [{"name": "shadow-gpt4o", "pattern": r"^gpt-4o$", "model_info": {"input_cost_per_token": 999.0}}] ) + litellm.get_model_info.cache_clear() info = litellm.get_model_info("gpt-4o") - # Resolved from the real exact entry, not the shadowing rule. assert info["litellm_provider"] == "openai" assert info["input_cost_per_token"] != 999.0 -def test_unknown_model_without_matching_rule_still_unmapped(restore_generalizations): - restore_generalizations( - [ - { - "name": "claude", - "pattern": r"^claude-", - "model_info": {"litellm_provider": "anthropic"}, - } - ] - ) - with pytest.raises(Exception): - litellm.get_model_info("totally-unknown-model-xyz") - - # --------------------------------------------------------------------------- # -# Shipped anthropic-claude rule +# Shipped rules (bundled cost map) # --------------------------------------------------------------------------- # @pytest.fixture def shipped_cost_map(monkeypatch): - """Activate the bundled cost map so the shipped anthropic-claude rule is installed.""" + """Activate the bundled cost map so the shipped rules are installed.""" original_cost = litellm.model_cost previous_rules = list(get_fallback_generalization_rules()) monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") @@ -299,40 +323,197 @@ def shipped_cost_map(monkeypatch): set_fallback_generalizations(previous_rules) -def test_shipped_rule_marks_unmapped_high_version_claude_adaptive_without_pricing( - shipped_cost_map, -): - """An unmapped Claude >= 4.6 resolves via the version-gated adaptive-thinking rule, which - inherits routing and capabilities from the base rule and adds ``supports_adaptive_thinking``. - The rule carries no pricing, so cost stays unpriced (zero, not a fabricated number) rather - than reporting a confidently-wrong price.""" - model = "claude-opus-9-9" +def test_shipped_bare_claude_id_routes_to_anthropic(shipped_cost_map): + _, provider, _, _ = litellm.get_llm_provider(model="claude-haiku-4-6") + assert provider == "anthropic" + + +def test_shipped_bedrock_syntax_claude_id_routes_to_bedrock(shipped_cost_map): + """Regression: a bedrock-syntax id must infer bedrock even when its version also + matches an unanchored Anthropic capability pattern. The old first-match-wins engine + routed global.anthropic.claude-haiku-4-6 to anthropic via the adaptive rule.""" + for model in [ + "global.anthropic.claude-haiku-4-6", + "us.anthropic.claude-haiku-4-6", + "anthropic.claude-haiku-4-6", + "eu.anthropic.claude-opus-5-0", + ]: + assert model not in litellm.model_cost + _, provider, _, _ = litellm.get_llm_provider(model=model) + assert provider == "bedrock", model + + +def test_shipped_rules_resolve_unmapped_bedrock_claude_with_bedrock_provider(shipped_cost_map): + model = "us.anthropic.claude-haiku-4-6" assert model not in litellm.model_cost - info = litellm.get_model_info(model) - assert info["litellm_provider"] == "anthropic" + info = litellm.get_model_info(model, custom_llm_provider="bedrock") + assert info["litellm_provider"] == "bedrock" assert info["supports_adaptive_thinking"] is True assert info["supports_function_calling"] is True + assert info["max_input_tokens"] == 200000 + assert info.get("supports_mid_conversation_system") is None assert not info.get("input_cost_per_token") assert not info.get("output_cost_per_token") -def test_shipped_rule_resolves_unmapped_low_version_claude_without_adaptive(shipped_cost_map): - """An unmapped Claude < 4.6 falls through to the version-neutral anthropic-claude rule: it - gets provider routing and baseline capabilities but no ``supports_adaptive_thinking`` flag, - so a sub-4.6 alias such as ``claude-opus-4-0`` resolves yet is never marked adaptive.""" - model = "claude-opus-4-0" +def test_shipped_rules_stack_adaptive_and_mid_conversation_flags(shipped_cost_map): + model = "claude-opus-4-9" + assert model not in litellm.model_cost + info = litellm.get_model_info(model, custom_llm_provider="anthropic") + assert info["litellm_provider"] == "anthropic" + assert info["supports_adaptive_thinking"] is True + assert info["supports_mid_conversation_system"] is True + assert info["supports_function_calling"] is True + + +@pytest.mark.parametrize( + "model,provider", + [ + ("claude-opus-4-9@20260101", "vertex_ai"), + ("databricks-claude-opus-5-1", "databricks"), + ], +) +def test_shipped_rules_are_provider_neutral_for_unmapped_ids(shipped_cost_map, model, provider): + assert model not in litellm.model_cost + info = litellm.get_model_info(model, custom_llm_provider=provider) + assert info["litellm_provider"] == provider + assert info["supports_adaptive_thinking"] is True + assert info["supports_mid_conversation_system"] is True + assert info["supports_function_calling"] is True + + +@pytest.mark.parametrize( + "model,provider,adaptive,mid_conversation", + [ + ("us.anthropic.claude-opus-4-5", "bedrock", None, None), + ("claude-haiku-4-6", "anthropic", True, None), + ("claude-haiku-4-7", "anthropic", True, None), + ("claude-haiku-4-8", "anthropic", True, True), + ("claude-haiku-4-9", "anthropic", True, True), + ("claude-haiku-4-10", "anthropic", True, True), + ("claude-haiku-5-0", "anthropic", True, True), + ("claude-sonnet-5-1", "anthropic", True, True), + ], +) +def test_shipped_version_boundaries(shipped_cost_map, model, provider, adaptive, mid_conversation): + assert model not in litellm.model_cost + info = litellm.get_model_info(model, custom_llm_provider=provider) + assert info["litellm_provider"] == provider + assert info["supports_function_calling"] is True + assert not info.get("input_cost_per_token") + assert info.get("supports_adaptive_thinking") is adaptive, model + assert info.get("supports_mid_conversation_system") is mid_conversation, model + + +def test_shipped_rules_cover_new_families_like_fable_at_5_plus(shipped_cost_map): + """Both version gates accept any claude-- id at major 5 or higher, bare + major or major-minor, so a new family shaped like claude-fable-5 gets adaptive + thinking and mid-conversation system support without a cost-map entry.""" + model = "claude-fable-5-1" + assert model not in litellm.model_cost + info = litellm.get_model_info(model, custom_llm_provider="anthropic") + assert info["supports_mid_conversation_system"] is True + assert info["supports_adaptive_thinking"] is True + assert info["supports_function_calling"] is True + + +def test_shipped_rules_flag_bare_5_plus_majors_of_any_family(shipped_cost_map): + """A bare 5+ major with no minor gets both flags at the rule level; the mapped + claude-fable-5 entry itself still resolves from the cost map, so this pins the + pattern via the capability union rather than get_model_info.""" + matched = match_capability_generalizations("claude-fable-5") + assert matched is not None + assert matched["supports_adaptive_thinking"] is True + assert matched["supports_mid_conversation_system"] is True + + +def test_shipped_version_gates_are_family_agnostic_at_4x(shipped_cost_map): + """Both version gates apply to any claude-- id, 4.x included: a non-core + family at 4.9 gets adaptive and mid-conversation, while the same family at 4.5 + gets baseline only. Only opus/sonnet/haiku ever shipped 4.x ids, so the + family-agnostic 4.6+ gate changes nothing for real models.""" + high = litellm.get_model_info("claude-newfam-4-9", custom_llm_provider="anthropic") + assert high["supports_adaptive_thinking"] is True + assert high["supports_mid_conversation_system"] is True + assert high["supports_function_calling"] is True + + low = litellm.get_model_info("claude-newfam-4-5", custom_llm_provider="anthropic") + assert low.get("supports_adaptive_thinking") is None + assert low.get("supports_mid_conversation_system") is None + assert low["supports_function_calling"] is True + + +def test_shipped_rules_give_bare_majors_the_full_baseline_union(shipped_cost_map): + """A bare-major unmapped id (no minor) resolves the same baseline union as its + major-minor sibling: the baseline pattern's minor is optional, so claude-newt-5 + is not left with version flags but no mode, token limits, or capability facts.""" + model = "anthropic/claude-newt-5" assert model not in litellm.model_cost info = litellm.get_model_info(model) assert info["litellm_provider"] == "anthropic" + assert info["mode"] == "chat" + assert info["max_tokens"] == 64000 assert info["supports_function_calling"] is True - assert info.get("supports_adaptive_thinking") is None - assert not info.get("input_cost_per_token") + assert info["supports_adaptive_thinking"] is True + assert info["supports_mid_conversation_system"] is True + + +def test_shipped_routing_rule_covers_bare_majors(shipped_cost_map): + _, provider, _, _ = litellm.get_llm_provider(model="claude-newt-5") + assert provider == "anthropic" + + +def test_shipped_adaptive_rule_requires_claude_prefix(shipped_cost_map): + """A non-Claude name embedding a core-family 4.6+/5.x version substring must not + resolve from the rules; serving it a zero-priced rule entry would silently + swallow cost tracking for arbitrary custom deployment names.""" + model = "openai/team-sonnet-5-1-alias" + assert model not in litellm.model_cost + assert match_capability_generalizations("team-sonnet-5-1-alias") is None + with pytest.raises(Exception): + litellm.get_model_info(model) + + +def test_shipped_exact_entry_beats_rules(shipped_cost_map): + model = "us.anthropic.claude-sonnet-4-6" + assert model in litellm.model_cost + info = litellm.get_model_info(model, custom_llm_provider="bedrock") + assert info["litellm_provider"] == "bedrock_converse" + assert info["input_cost_per_token"] == 3.3e-06 + assert info["max_input_tokens"] == 1000000 + assert info["supports_adaptive_thinking"] is True + assert info.get("supports_mid_conversation_system") is None + + +def test_shipped_rules_lose_to_exact_entries_across_cost_ladder_variants(shipped_cost_map): + """A route-mangled variant of an exactly-mapped model must never resolve from + rules. The cost calculator tries model-name variants in order; a rule-derived + unpriced entry served for an early variant (here bedrock/claude-haiku-4-5-20251001, + whose bare form is exactly mapped under anthropic) would zero out the bill even + though the exact priced bedrock entry is one variant later. An exactly-mapped id + under a mismatched provider raises instead of resolving from rules.""" + from litellm import completion_cost + from litellm.types.utils import ModelResponse, Usage + + assert "claude-haiku-4-5-20251001" in litellm.model_cost + with pytest.raises(Exception): + litellm.get_model_info("claude-haiku-4-5-20251001", custom_llm_provider="bedrock") + + entry = litellm.model_cost["us.anthropic.claude-haiku-4-5-20251001-v1:0"] + response = ModelResponse(model="claude-haiku-4-5-20251001", usage=Usage(prompt_tokens=100, completion_tokens=50)) + cost = completion_cost( + completion_response=response, + model="bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0", + custom_llm_provider="bedrock", + ) + assert cost == 100 * entry["input_cost_per_token"] + 50 * entry["output_cost_per_token"] + assert cost > 0 def test_shipped_adaptive_rule_gates_on_version_not_pricing(shipped_cost_map): - """The version-gated ``anthropic-claude-adaptive-thinking`` rule marks an unmapped - Claude adaptive only from >= 4.6, including provider-prefixed ids the anchored pricing - rule cannot match, while leaving < 4.6 (and the dated Opus 4.0 form) non-adaptive.""" + """The version-gated adaptive-thinking capability rule marks an unmapped Claude + adaptive only from >= 4.6, including provider-prefixed ids the anchored routing + rule cannot match, while leaving the dated Opus 4.0 form non-adaptive.""" from litellm.llms.anthropic.common_utils import AnthropicModelInfo adaptive = "us.anthropic.claude-opus-4-9" @@ -343,12 +524,10 @@ def test_shipped_adaptive_rule_gates_on_version_not_pricing(shipped_cost_map): assert AnthropicModelInfo._is_adaptive_thinking_model(non_adaptive) is False -def test_shipped_bedrock_rule_resolves_unmapped_future_claude_for_bedrock(shipped_cost_map): - """An unmapped Bedrock Claude >= 4.8 resolves via the bedrock-scoped - ``bedrock-anthropic-claude-mid-conversation-system`` rule even when the lookup - carries ``custom_llm_provider="bedrock"``, which the provider check uses to drop - the anthropic-scoped rules. It inherits base capabilities, gains both - version-gated flags, and stays unpriced.""" +def test_shipped_rules_resolve_unmapped_future_bedrock_claude_with_both_flags(shipped_cost_map): + """An unmapped Bedrock Claude >= 4.8 resolves for custom_llm_provider="bedrock" with + baseline capabilities, both version-gated flags, the bedrock provider backfilled, and + no fabricated pricing.""" model = "us.anthropic.claude-opus-4-9" assert model not in litellm.model_cost info = litellm.get_model_info(model, custom_llm_provider="bedrock") @@ -359,11 +538,11 @@ def test_shipped_bedrock_rule_resolves_unmapped_future_claude_for_bedrock(shippe assert not info.get("input_cost_per_token") -def test_shipped_bedrock_mid_conversation_rule_gates_on_version_and_naming(shipped_cost_map): - """The bedrock rule only claims Bedrock-style ids at 4.8+, bare 5+ majors and - new families included; pre-4.8 Bedrock ids and native ids never gain the flag, - and the rule outranks the anthropic-scoped ones for Bedrock ids because it is - listed first.""" +def test_shipped_mid_conversation_gate_on_bedrock_ids(shipped_cost_map): + """Bedrock-syntax ids gain ``supports_mid_conversation_system`` only from 4.8 upward, + bare 5+ majors and new families included; 4.7-and-below Bedrock ids never gain it. + The flag comes from the provider-neutral capability rule rather than a bedrock-scoped + one, so the same gate covers native and vertex-shaped ids too.""" for flagged in ( "us.anthropic.claude-opus-4-8", "jp.anthropic.claude-opus-4-8", @@ -371,17 +550,14 @@ def test_shipped_bedrock_mid_conversation_rule_gates_on_version_and_naming(shipp "us.anthropic.claude-fable-5", "anthropic.claude-sonnet-5-20260101-v1:0", ): - matched = match_fallback_generalization(flagged) + matched = match_capability_generalizations(flagged) assert matched is not None, flagged - assert matched["litellm_provider"] == "bedrock", flagged assert matched["supports_mid_conversation_system"] is True, flagged for unflagged in ( "us.anthropic.claude-opus-4-7", "us.anthropic.claude-sonnet-4-6", "us.anthropic.claude-haiku-4-5-20251001-v1:0", "anthropic.claude-3-5-sonnet-20240620-v1:0", - "claude-opus-4-9", - "claude-sonnet-5", ): - matched = match_fallback_generalization(unflagged) + matched = match_capability_generalizations(unflagged) assert matched is None or not matched.get("supports_mid_conversation_system"), unflagged diff --git a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py index 924aaa7775d..bdd71f28b1a 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py @@ -13,7 +13,8 @@ sys.path.insert(0, os.path.abspath("../../..")) from litellm.litellm_core_utils.fallback_generalizations import ( get_fallback_generalization_rules, - match_fallback_generalization, + match_capability_generalizations, + match_routing_generalization, set_fallback_generalizations, ) from litellm.litellm_core_utils.get_model_cost_map import ( @@ -102,9 +103,7 @@ def test_finalize_pops_key_and_installs_rules(): # The reserved key is removed from the returned model map ... assert FALLBACK_GENERALIZATIONS_KEY not in finalized # ... and its rules are installed into the generalizations module. - assert match_fallback_generalization("widget-9") == { - "litellm_provider": "openai" - } + assert match_routing_generalization("widget-9") == "openai" finally: set_fallback_generalizations(previous) @@ -116,27 +115,25 @@ def test_finalize_with_no_block_clears_rules(): [{"name": "stale", "pattern": r"^x", "model_info": {"a": 1}}] ) _finalize_model_cost_map(_make_models(2)) - assert match_fallback_generalization("x-1") is None + assert match_capability_generalizations("x-1") is None finally: set_fallback_generalizations(previous) -def test_shipped_backup_carries_the_anthropic_claude_rule(): - """The bundled backup must ship the anthropic-claude rule so a fresh install - (or an offline fallback) routes unknown Claude models without code changes.""" +def test_shipped_backup_carries_the_claude_routing_rules(): + """The bundled backup must ship the Claude routing rules so a fresh install + (or an offline fallback) routes unknown Claude models without code changes. + Bedrock-syntax ids must hit the bedrock rule before the bare-id Anthropic rule.""" backup = GetModelCostMap.load_local_model_cost_map() rules = backup.get(FALLBACK_GENERALIZATIONS_KEY, {}).get("rules", []) - names = {r.get("name") for r in rules} - assert "anthropic-claude" in names - - rule = next(r for r in rules if r.get("name") == "anthropic-claude") - assert rule["model_info"]["litellm_provider"] == "anthropic" + names = [r.get("name") for r in rules] + assert names.index("bedrock-claude-ids") < names.index("anthropic-claude-ids") previous = list(get_fallback_generalization_rules()) try: set_fallback_generalizations(rules) - matched = match_fallback_generalization("claude-opus-4-9") - assert matched is not None and matched["litellm_provider"] == "anthropic" + assert match_routing_generalization("claude-opus-4-9") == "anthropic" + assert match_routing_generalization("global.anthropic.claude-opus-4-9") == "bedrock" finally: set_fallback_generalizations(previous) @@ -147,23 +144,19 @@ def test_shipped_backup_marks_claude_4_6_plus_adaptive_not_4_0(): route) and on the version-gated anthropic-claude-adaptive-thinking rule for unmapped future Claudes, while leaving the dated Claude 4.0 names ("...-4-20250514") unflagged so a date can never be mistaken for a 4.6+ minor - version. The version-neutral anthropic-claude pricing rule must not flag it, so - an unmapped sub-4.6 name is priced but stays non-adaptive. The adaptive rule must - inherit pricing from the pricing rule via ``extends`` and carry only its delta, so - the Opus-tier price block is never duplicated across rules.""" + version. The version-neutral claude-family-baseline capability rule must not flag + it, so an unmapped sub-4.6 name resolves but stays non-adaptive. The adaptive rule + carries only its delta; capability unioning stacks it onto the baseline, so the + baseline block is never duplicated across rules and no rule needs ``extends``.""" backup = GetModelCostMap.load_local_model_cost_map() rules = backup[FALLBACK_GENERALIZATIONS_KEY]["rules"] - pricing_rule = next(r for r in rules if r.get("name") == "anthropic-claude") - adaptive_rule = next( - r for r in rules if r.get("name") == "anthropic-claude-adaptive-thinking" - ) - assert "supports_adaptive_thinking" not in pricing_rule["model_info"] - assert adaptive_rule["model_info"]["supports_adaptive_thinking"] is True - - assert "extends" not in pricing_rule - assert adaptive_rule.get("extends") == "anthropic-claude" + baseline_rule = next(r for r in rules if r.get("name") == "claude-family-baseline") + adaptive_rule = next(r for r in rules if r.get("name") == "claude-adaptive-thinking") + assert "supports_adaptive_thinking" not in baseline_rule["model_info"] + assert "litellm_provider" not in baseline_rule["model_info"] assert adaptive_rule["model_info"] == {"supports_adaptive_thinking": True} + assert all("extends" not in r for r in rules) for adaptive in [ "anthropic.claude-opus-4-8", diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py index f7a234c8ae8..4bd270bb7fb 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py @@ -1649,17 +1649,18 @@ class TestClaudeOpus48AdaptiveThinking: @pytest.mark.parametrize( "model", [ - "us.anthropic.claude-fable-5-preview", - "claude-fable-5-preview", + "us.anthropic.claude-fable-preview", + "claude-fable-preview", ], ) def test_unmapped_aliases_without_parseable_version_stay_non_adaptive( self, local_model_cost_map, model ): """An alias absent from the map, not matched by any ``fallback_generalizations`` - rule, and without a parseable opus/sonnet/haiku >= 4.6 family version stays - non-adaptive. ``fable`` is outside the version-rule family set, so neither the - cost map nor the declarative rule marks it adaptive.""" + rule, and without any parseable family version stays non-adaptive. ``fable`` + without a major version matches neither the core-family 4.6+ gate nor the + family-agnostic 5+ gate, so neither the cost map nor the declarative rule marks + it adaptive.""" import litellm from litellm.llms.anthropic.common_utils import AnthropicModelInfo @@ -1676,16 +1677,18 @@ class TestClaudeOpus48AdaptiveThinking: "claude-opus-5-0", "claude-opus-4-10", "claude-opus-4-8-some-future-suffix", + "claude-fable-5-preview", + "us.anthropic.claude-fable-5-preview", ], ) def test_adaptive_thinking_version_fallback_for_unmapped_high_versions( self, local_model_cost_map, model ): - """Provider-prefixed or suffixed Claude names that resolve to no mapped entry and - are not matched by the anchored ``anthropic-claude`` pricing rule still resolve to - adaptive when their opus/sonnet/haiku family version is >= 4.6. The version gate is - the declarative ``anthropic-claude-adaptive-thinking`` rule, so 5.x, 6.x and any - later family are covered with no code change.""" + """Provider-prefixed or suffixed Claude names that resolve to no mapped entry + still resolve to adaptive when the id carries claude-- at version 4.6 + or higher, bare 5+ majors included. The version gate is the declarative + ``claude-adaptive-thinking`` rule, so 5.x, 6.x and any later family are covered + with no code change.""" import litellm from litellm.llms.anthropic.common_utils import AnthropicModelInfo diff --git a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index 43ce4873023..52f136a1fd9 100644 --- a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -1974,13 +1974,22 @@ def test_bedrock_invoke_transform_merges_list_content_system_role_into_system(): ] -def test_bedrock_invoke_transform_keeps_mid_conversation_system_role_in_place(local_model_cost_map): +@pytest.mark.parametrize( + "model", + [ + "anthropic.claude-opus-4-8", + "jp.anthropic.claude-opus-4-8", + "us.anthropic.claude-sonnet-5", + "us.anthropic.claude-fable-5", + ], +) +def test_bedrock_invoke_transform_keeps_mid_conversation_system_role_in_place(local_model_cost_map, model): """Regression test for the Bedrock prompt-cache collapse: hoisting a mid-conversation ``role: "system"`` message (e.g. Claude Code's ``mid-conversation-system-2026-04-07`` reminders) into the top-level ``system`` field mutates the cache prefix and invalidates the cached message - history, so on models flagged ``supports_mid_conversation_system`` (the Opus - 4.8 family, which Invoke accepts the role on) such entries must be forwarded + history, so on models flagged ``supports_mid_conversation_system`` (Claude + 4.8+, which Invoke accepts the role on) such entries must be forwarded in place. Billing-header blocks must still be stripped from the top-level ``system`` field even when nothing is hoisted.""" from litellm.types.router import GenericLiteLLMParams @@ -1994,7 +2003,7 @@ def test_bedrock_invoke_transform_keeps_mid_conversation_system_role_in_place(lo ] result = cfg.transform_anthropic_messages_request( - model="anthropic.claude-opus-4-8", + model=model, messages=copy.deepcopy(messages), anthropic_messages_optional_request_params={ "max_tokens": 256, @@ -2121,9 +2130,9 @@ def test_bedrock_invoke_transform_hoists_all_system_for_unmapped_model(local_mod def test_bedrock_invoke_transform_keeps_system_in_place_for_unmapped_future_claude(local_model_cost_map): """An unmapped Bedrock Claude at 4.8 or higher resolves through the - ``bedrock-anthropic-claude-mid-conversation-system`` fallback rule, so a - future model that has not landed in the cost map yet keeps the - cache-preserving in-place behavior instead of falling back to hoist-all.""" + ``claude-mid-conversation-system`` capability rule, so a future model that + has not landed in the cost map yet keeps the cache-preserving in-place + behavior instead of falling back to hoist-all.""" from litellm.types.router import GenericLiteLLMParams cfg = AmazonAnthropicClaudeMessagesConfig() @@ -2146,6 +2155,36 @@ def test_bedrock_invoke_transform_keeps_system_in_place_for_unmapped_future_clau assert "system" not in result +def test_bedrock_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_flag(): + """Exact cost-map hits resolve before fallback-generalization rules, so a + mapped Bedrock Claude 4.8+ entry without ``supports_mid_conversation_system`` + silently loses the cache-preserving in-place handling that the + ``claude-mid-conversation-system`` capability rule grants unmapped ids. + Every mapped bedrock entry the rule's own pattern matches must carry the + flag explicitly.""" + import re + + import litellm + + cost_map_path = os.path.join(os.path.dirname(litellm.__file__), "model_prices_and_context_window_backup.json") + with open(cost_map_path) as f: + cost_map = json.load(f) + rules = cost_map["fallback_generalizations"]["rules"] + pattern = re.compile( + next(r["pattern"] for r in rules if r["name"] == "claude-mid-conversation-system"), + re.IGNORECASE, + ) + missing = [ + key + for key, info in cost_map.items() + if isinstance(info, dict) + and str(info.get("litellm_provider", "")).startswith("bedrock") + and pattern.search(key) + and info.get("supports_mid_conversation_system") is not True + ] + assert missing == [] + + def test_as_system_content_blocks_handles_each_shape(): """``_as_system_content_blocks`` normalizes every system shape: ``None`` -> empty, a string -> a single text block, a list -> a shallow copy, and any other value diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 11a6fea0cea..c55a631c7b3 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -660,6 +660,7 @@ async def test_register_client_persists_dcr_client_identity(): assert update_data.credentials["client_id"] == "generated-client" assert update_data.credentials["client_secret"] == "generated-secret" assert update_data.credentials["token_endpoint_auth_method"] == "client_secret_basic" + assert update_data.credentials["redirect_uris"] == ["https://proxy.litellm.example/callback"] assert update_data.oauth2_flow == "authorization_code" mock_update_server.assert_called_once() @@ -1141,6 +1142,296 @@ async def test_register_client_returns_reused_client_when_concurrent_persist_win mock_update_server.assert_called_once_with(persisted_server) +def _dcr_redirect_test_server(client_id): + from litellm.proxy._types import MCPTransport + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + return MCPServer( + server_id="remote_server", + name="remote_server", + server_name="remote_server", + alias="remote_server", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + client_id=client_id, + client_secret=None, + authorization_url="https://provider.example/oauth/authorize", + token_url="https://provider.example/oauth/token", + registration_url="https://provider.example/oauth/register", + ) + + +@pytest.mark.asyncio +async def test_register_client_re_registers_when_persisted_redirect_uri_no_longer_matches_origin(): + """A persisted DCR client is bound to the redirect_uri it was registered with. When the + proxy's resolved public origin changes, every authorize built for the reused client is + rejected IdP-side and the server is permanently stranded (GH #32473). A positive mismatch + between the recorded redirect_uris and the current callback must therefore re-register on + the admin path and persist the replacement client, with the new binding recorded and the + old client's secret/auth method cleared rather than merged into the new identity.""" + try: + from fastapi import Request + + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + register_client_with_server, + ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + except ImportError: + pytest.skip("MCP discoverable endpoints not available") + + oauth2_server = _dcr_redirect_test_server(client_id="stale-client") + + mock_request = MagicMock(spec=Request) + mock_request.base_url = "https://proxy.litellm.example/" + mock_request.headers = {} + + mock_response = MagicMock() + mock_response.json.return_value = { + "client_id": "fresh-client", + "redirect_uris": ["https://proxy.litellm.example/callback"], + } + mock_response.raise_for_status = MagicMock() + mock_async_client = MagicMock() + mock_async_client.post = AsyncMock(return_value=mock_response) + + persisted_server = MagicMock() + persisted_server.credentials = { + "client_id": "stale-client", + "client_secret": "stale-secret", + "token_endpoint_auth_method": "client_secret_basic", + "redirect_uris": ["https://old-origin.example/callback"], + } + mock_get_mcp_server = AsyncMock(return_value=persisted_server) + mock_update_mcp_server = AsyncMock(return_value=MagicMock()) + mock_update_server = AsyncMock() + + with ( + patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + return_value=mock_async_client, + ), + patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()), + patch( + "litellm.proxy._experimental.mcp_server.db.get_mcp_server", + new=mock_get_mcp_server, + ), + patch( + "litellm.proxy._experimental.mcp_server.db.update_mcp_server", + new=mock_update_mcp_server, + ), + patch.object(global_mcp_server_manager, "update_server", new=mock_update_server), + ): + response = await register_client_with_server( + request=mock_request, + mcp_server=oauth2_server, + client_name="Litellm Proxy", + grant_types=["authorization_code", "refresh_token"], + response_types=["code"], + token_endpoint_auth_method="none", + persist_credentials=True, + ) + + mock_async_client.post.assert_called_once() + register_payload = mock_async_client.post.call_args.kwargs["json"] + assert register_payload["redirect_uris"] == ["https://proxy.litellm.example/callback"] + + mock_update_mcp_server.assert_called_once() + update_data = mock_update_mcp_server.call_args.kwargs["data"] + assert update_data.credentials["client_id"] == "fresh-client" + assert update_data.credentials["redirect_uris"] == ["https://proxy.litellm.example/callback"] + assert update_data.credentials["client_secret"] is None + assert update_data.credentials["token_endpoint_auth_method"] is None + + assert response.status_code == 200 + assert json.loads(response.body.decode("utf-8"))["client_id"] == "fresh-client" + + +@pytest.mark.asyncio +async def test_register_client_grandfathers_persisted_client_without_recorded_redirect_uris(): + """Clients persisted before redirect_uris were recorded (and admin-configured clients, + which never get a recording) have nothing to compare against; treating that as a mismatch + would re-mint a client_id for every existing install on upgrade and orphan all users' + refresh tokens for those servers. A missing recording must read as a match: no DCR call, + no persistence write, existing client returned.""" + try: + from fastapi import Request + + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + register_client_with_server, + ) + except ImportError: + pytest.skip("MCP discoverable endpoints not available") + + oauth2_server = _dcr_redirect_test_server(client_id="legacy-client") + + mock_request = MagicMock(spec=Request) + mock_request.base_url = "https://proxy.litellm.example/" + mock_request.headers = {} + + mock_async_client = MagicMock() + mock_async_client.post = AsyncMock() + + persisted_server = MagicMock() + persisted_server.credentials = {"client_id": "legacy-client"} + mock_get_mcp_server = AsyncMock(return_value=persisted_server) + mock_update_mcp_server = AsyncMock() + + with ( + patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + return_value=mock_async_client, + ), + patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()), + patch( + "litellm.proxy._experimental.mcp_server.db.get_mcp_server", + new=mock_get_mcp_server, + ), + patch( + "litellm.proxy._experimental.mcp_server.db.update_mcp_server", + new=mock_update_mcp_server, + ), + ): + response = await register_client_with_server( + request=mock_request, + mcp_server=oauth2_server, + client_name="Litellm Proxy", + grant_types=["authorization_code", "refresh_token"], + response_types=["code"], + token_endpoint_auth_method="none", + persist_credentials=True, + ) + + mock_async_client.post.assert_not_called() + mock_update_mcp_server.assert_not_called() + assert response["client_secret"] == "dummy" + assert oauth2_server.client_id == "legacy-client" + + +@pytest.mark.asyncio +async def test_register_client_keeps_persisted_client_when_recorded_redirect_uri_matches_origin(): + """When the recorded redirect_uris still cover the current callback the persisted client + is valid; re-registering would orphan refresh tokens for no reason.""" + try: + from fastapi import Request + + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + register_client_with_server, + ) + except ImportError: + pytest.skip("MCP discoverable endpoints not available") + + oauth2_server = _dcr_redirect_test_server(client_id="kept-client") + + mock_request = MagicMock(spec=Request) + mock_request.base_url = "https://proxy.litellm.example/" + mock_request.headers = {} + + mock_async_client = MagicMock() + mock_async_client.post = AsyncMock() + + persisted_server = MagicMock() + persisted_server.credentials = { + "client_id": "kept-client", + "redirect_uris": ["https://proxy.litellm.example/callback"], + } + mock_get_mcp_server = AsyncMock(return_value=persisted_server) + mock_update_mcp_server = AsyncMock() + + with ( + patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + return_value=mock_async_client, + ), + patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()), + patch( + "litellm.proxy._experimental.mcp_server.db.get_mcp_server", + new=mock_get_mcp_server, + ), + patch( + "litellm.proxy._experimental.mcp_server.db.update_mcp_server", + new=mock_update_mcp_server, + ), + ): + response = await register_client_with_server( + request=mock_request, + mcp_server=oauth2_server, + client_name="Litellm Proxy", + grant_types=["authorization_code", "refresh_token"], + response_types=["code"], + token_endpoint_auth_method="none", + persist_credentials=True, + ) + + mock_async_client.post.assert_not_called() + mock_update_mcp_server.assert_not_called() + assert response["client_secret"] == "dummy" + + +@pytest.mark.asyncio +async def test_register_client_non_admin_reuses_persisted_client_despite_redirect_mismatch(): + """Non-persisting callers (the public register routes and non-admin users) must keep + today's reuse behavior even when the recorded redirect_uris mismatch: re-registering + without persistence would mint an orphan upstream client on every connect while the + stored client keeps being used at authorize time. Only the admin path re-registers.""" + try: + from fastapi import Request + + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + register_client_with_server, + ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + except ImportError: + pytest.skip("MCP discoverable endpoints not available") + + oauth2_server = _dcr_redirect_test_server(client_id=None) + + mock_request = MagicMock(spec=Request) + mock_request.base_url = "https://proxy.litellm.example/" + mock_request.headers = {} + + mock_async_client = MagicMock() + mock_async_client.post = AsyncMock() + + persisted_server = MagicMock() + persisted_server.credentials = { + "client_id": "persisted-client", + "redirect_uris": ["https://old-origin.example/callback"], + } + mock_get_mcp_server = AsyncMock(return_value=persisted_server) + mock_update_server = AsyncMock() + + with ( + patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + return_value=mock_async_client, + ), + patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()), + patch( + "litellm.proxy._experimental.mcp_server.db.get_mcp_server", + new=mock_get_mcp_server, + ), + patch.object(global_mcp_server_manager, "update_server", new=mock_update_server), + ): + response = await register_client_with_server( + request=mock_request, + mcp_server=oauth2_server, + client_name="Litellm Proxy", + grant_types=["authorization_code", "refresh_token"], + response_types=["code"], + token_endpoint_auth_method="none", + persist_credentials=False, + ) + + mock_async_client.post.assert_not_called() + assert oauth2_server.client_id == "persisted-client" + assert response["client_secret"] == "dummy" + + @pytest.mark.asyncio async def test_register_client_reuses_existing_client_id_without_re_dcr(): """A server that already has a client_id (admin-configured or previously DCR'd) must be @@ -1848,6 +2139,41 @@ async def test_token_endpoint_respects_x_forwarded_host(): None, "https://external.com", ), + ( + "http://localhost:4000/", + "https", + "proxy.example.com", + "443", + "https://proxy.example.com", + ), + ( + "http://localhost:4000/", + "http", + "proxy.example.com", + "80", + "http://proxy.example.com", + ), + ( + "http://internal.local/", + "https", + None, + "443", + "https://internal.local", + ), + ( + "http://localhost:4000/", + "https", + "proxy.example.com", + "8443", + "https://proxy.example.com:8443", + ), + ( + "http://localhost:4000/", + "https", + "proxy.example.com:443", + None, + "https://proxy.example.com", + ), ], ) def test_get_request_base_url_comprehensive( diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index d150591c8de..e7d2909263a 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -3706,3 +3706,341 @@ async def test_per_tag_untagged_request_governed_by_key_limit_v3(monkeypatch): await call({"tags": ["cell-99"]}) assert exc_info.value.status_code == 429 assert "tag_per_key" not in str(exc_info.value.detail) + + +# -------------------------------------------------------------------------- +# Streaming success logging mirrors x-ratelimit-* remaining values into +# standard_logging_object.hidden_params.additional_headers so Prometheus / +# logging callbacks see them for streams too (non-streaming already gets +# them via async_post_call_success_hook, which the streaming path skips). +# -------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_streaming_end_to_end_populates_slp_ratelimit_headers(monkeypatch): + """ + End-to-end regression: on a streaming request, the same pre-call + + success-callback pair the proxy uses must land ``x-ratelimit-*`` + remaining/limit values in + ``kwargs["standard_logging_object"]["hidden_params"]["additional_headers"]``. + """ + monkeypatch.setenv("LITELLM_RATE_LIMIT_WINDOW_SIZE", "60") + _api_key = hash_token("sk-stream-e2e") + user_api_key_dict = UserAPIKeyAuth( + api_key=_api_key, + rpm_limit=100, + tpm_limit=10000, + ) + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + + # Real pre-call: populates data and stashes the response into metadata + # so the success callback can find it via litellm_params.metadata. + data: Dict[str, Any] = { + "model": "gpt-4o-mini", + "metadata": {}, + "stream": True, + } + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data=data, + call_type="", + ) + + # Simulate the wrapper handing the pre-call metadata dict to the + # completion() call: it becomes kwargs["litellm_params"]["metadata"] by + # the time the success callback fires. + mock_response = ModelResponse( + id="mock-stream-e2e", + object="chat.completion", + created=int(datetime.now().timestamp()), + model="gpt-4o-mini", + usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150), + choices=[], + ) + mock_kwargs: Dict[str, Any] = { + "standard_logging_object": { + "metadata": { + "user_api_key_hash": _api_key, + "user_api_key_user_id": None, + "user_api_key_team_id": None, + "user_api_key_end_user_id": None, + } + }, + "litellm_params": {"metadata": data["metadata"]}, + "model": "gpt-4o-mini", + } + + async def _noop_increment(increment_list, **_): + return True + + monkeypatch.setattr( + handler.internal_usage_cache.dual_cache, + "async_increment_cache_pipeline", + _noop_increment, + ) + + # async_logging_hook runs before async_log_success_event, so any + # downstream callback that reads the SLP sees the mirrored values. + await handler.async_logging_hook( + kwargs=mock_kwargs, + result=mock_response, + call_type="acompletion", + ) + + additional_headers = ( + mock_kwargs["standard_logging_object"] + .get("hidden_params", {}) + .get("additional_headers", {}) + ) + + # api_key-scoped remaining/limit values are the baseline every request + # emits and must always reach the SLP. + remaining_keys = [ + k for k in additional_headers if "-remaining-" in k + ] + assert ( + remaining_keys + ), f"streaming success must populate remaining values, got {additional_headers!r}" + limit_keys = [k for k in additional_headers if "-limit-" in k] + assert limit_keys, "streaming success must also populate limit values" + assert ( + additional_headers.get("x-ratelimit-api_key-remaining-requests") == 99 + ), ( + "api_key remaining requests should reflect the just-consumed slot;" + f" got {additional_headers!r}" + ) + + +@pytest.mark.asyncio +async def test_streaming_populates_model_per_key_ratelimit_headers(monkeypatch): + """ + Streaming must land the per-(key, model) remaining/limit values in the + SLP under ``x-ratelimit-model_per_key-{remaining|limit}-{requests,tokens}``. + """ + monkeypatch.setenv("LITELLM_RATE_LIMIT_WINDOW_SIZE", "60") + _api_key = hash_token("sk-stream-mirror") + user_api_key_dict = UserAPIKeyAuth( + api_key=_api_key, + metadata={ + "model_rpm_limit": {"gpt-4o-mini": 100}, + "model_tpm_limit": {"gpt-4o-mini": 10000}, + }, + ) + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + + async def _noop_increment(increment_list, **_): + return True + + monkeypatch.setattr( + handler.internal_usage_cache.dual_cache, + "async_increment_cache_pipeline", + _noop_increment, + ) + + data: Dict[str, Any] = {"model": "gpt-4o-mini", "metadata": {}, "stream": True} + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data=data, + call_type="", + ) + + mock_response = ModelResponse( + id="mock-stream", + object="chat.completion", + created=int(datetime.now().timestamp()), + model="gpt-4o-mini", + usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150), + choices=[], + ) + + mock_kwargs: Dict[str, Any] = { + "standard_logging_object": { + "metadata": { + "user_api_key_hash": _api_key, + "user_api_key_user_id": None, + "user_api_key_team_id": None, + "user_api_key_end_user_id": None, + } + }, + "litellm_params": {"metadata": data["metadata"]}, + "model": "gpt-4o-mini", + } + + await handler.async_logging_hook( + kwargs=mock_kwargs, + result=mock_response, + call_type="acompletion", + ) + + hidden_params = mock_kwargs["standard_logging_object"].get("hidden_params") or {} + additional_headers = hidden_params.get("additional_headers") or {} + + assert ( + additional_headers.get("x-ratelimit-model_per_key-remaining-requests") == 99 + ), f"got {additional_headers!r}" + assert additional_headers.get("x-ratelimit-model_per_key-limit-requests") == 100 + + # response._hidden_params is also updated for late readers. + response_hidden = getattr(mock_response, "_hidden_params", None) or {} + response_headers = response_hidden.get("additional_headers") or {} + assert response_headers.get("x-ratelimit-model_per_key-remaining-requests") == 99 + + +@pytest.mark.asyncio +async def test_async_log_success_event_no_mirror_when_no_snapshot(monkeypatch): + """ + No pre-call snapshot (no descriptors matched) -> no fabricated + ``x-ratelimit-*`` headers. + """ + monkeypatch.setenv("LITELLM_RATE_LIMIT_WINDOW_SIZE", "60") + _api_key = hash_token("sk-stream-no-mirror") + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(DualCache()) + ) + + async def _noop_increment(increment_list, **_): + return True + + monkeypatch.setattr( + handler.internal_usage_cache.dual_cache, + "async_increment_cache_pipeline", + _noop_increment, + ) + + mock_response = ModelResponse( + id="mock-stream-none", + object="chat.completion", + created=int(datetime.now().timestamp()), + model="gpt-4o-mini", + usage=Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2), + choices=[], + ) + + mock_kwargs: Dict[str, Any] = { + "standard_logging_object": { + "metadata": { + "user_api_key_hash": _api_key, + "user_api_key_user_id": None, + "user_api_key_team_id": None, + "user_api_key_end_user_id": None, + } + }, + "litellm_params": {"metadata": {}}, + "model": "gpt-4o-mini", + } + + await handler.async_logging_hook( + kwargs=mock_kwargs, + result=mock_response, + call_type="acompletion", + ) + + hidden_params = mock_kwargs["standard_logging_object"].get("hidden_params") or {} + additional_headers = hidden_params.get("additional_headers") or {} + ratelimit_keys = [k for k in additional_headers if k.startswith("x-ratelimit-")] + assert ( + not ratelimit_keys + ), f"no snapshot must produce no rate-limit headers, got {ratelimit_keys}" + + +@pytest.mark.asyncio +async def test_streaming_mirror_matches_non_streaming_header_shape(monkeypatch): + """ + Given the same pre-call state, streaming and non-streaming must write + the identical ``x-ratelimit-*`` key/value shape to their respective + ``additional_headers`` slots. + """ + monkeypatch.setenv("LITELLM_RATE_LIMIT_WINDOW_SIZE", "60") + _api_key = hash_token("sk-shape") + user_api_key_dict = UserAPIKeyAuth( + api_key=_api_key, + metadata={ + "model_rpm_limit": {"gpt-4o-mini": 50}, + "model_tpm_limit": {"gpt-4o-mini": 5000}, + }, + ) + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + + async def _noop_increment(increment_list, **_): + return True + + monkeypatch.setattr( + handler.internal_usage_cache.dual_cache, + "async_increment_cache_pipeline", + _noop_increment, + ) + + # Drive pre-call once so both paths have the same authoritative snapshot. + data: Dict[str, Any] = {"model": "gpt-4o-mini", "metadata": {}} + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data=data, + call_type="", + ) + + # Non-streaming path: async_post_call_success_hook mutates response._hidden_params. + non_stream_response = ModelResponse( + id="mock-non-stream", + object="chat.completion", + created=int(datetime.now().timestamp()), + model="gpt-4o-mini", + usage=Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2), + choices=[], + ) + non_stream_response._hidden_params = {} + await handler.async_post_call_success_hook( + data=data, + user_api_key_dict=user_api_key_dict, + response=non_stream_response, + ) + non_stream_headers = non_stream_response._hidden_params.get( + "additional_headers", {} + ) + + # Streaming path: async_logging_hook mirrors into standard_logging_object. + stream_kwargs: Dict[str, Any] = { + "standard_logging_object": { + "metadata": {"user_api_key_hash": _api_key} + }, + "litellm_params": {"metadata": data["metadata"]}, + "model": "gpt-4o-mini", + } + stream_response = ModelResponse( + id="mock-stream", + object="chat.completion", + created=int(datetime.now().timestamp()), + model="gpt-4o-mini", + usage=Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2), + choices=[], + ) + await handler.async_logging_hook( + kwargs=stream_kwargs, + result=stream_response, + call_type="acompletion", + ) + stream_slp_headers = ( + stream_kwargs["standard_logging_object"] + .get("hidden_params", {}) + .get("additional_headers", {}) + ) + + def _rl_only(headers: Dict[str, Any]) -> Dict[str, Any]: + return {k: v for k, v in headers.items() if k.startswith("x-ratelimit-")} + + assert _rl_only(stream_slp_headers) == _rl_only(non_stream_headers), ( + f"streaming={_rl_only(stream_slp_headers)}" + f" non_streaming={_rl_only(non_stream_headers)}" + ) + assert "x-ratelimit-model_per_key-remaining-requests" in stream_slp_headers diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 2c44ab5a049..87b4c96e323 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,9 +1,9 @@ { "LIT001": { - "limit": 23452 + "limit": 23409 }, "LIT002": { - "limit": 27522 + "limit": 27511 }, "LIT003": { "limit": 292 diff --git a/ui/litellm-dashboard/eslint-metrics.json b/ui/litellm-dashboard/eslint-metrics.json index 7abbb186771..8ae7dc88c3d 100644 --- a/ui/litellm-dashboard/eslint-metrics.json +++ b/ui/litellm-dashboard/eslint-metrics.json @@ -1,7 +1,7 @@ { - "@typescript-eslint/no-explicit-any": 1978, - "complexity": 130, - "local/no-large-inline-object-arg": 509, + "@typescript-eslint/no-explicit-any": 1971, + "complexity": 129, + "local/no-large-inline-object-arg": 501, "local/no-long-condition-chain": 234, "max-depth": 59, "no-console": 16 diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 49fdb38c47b..648503abb27 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -28,6 +28,7 @@ "moment": "2.30.1", "next": "16.2.6", "openai": "4.104.0", + "openapi-fetch": "^0.17.0", "papaparse": "5.5.3", "react": "18.3.1", "react-copy-to-clipboard": "5.1.1", @@ -10534,6 +10535,15 @@ "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", "license": "MIT" }, + "node_modules/openapi-fetch": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/openapi-fetch/-/openapi-fetch-0.17.0.tgz", + "integrity": "sha512-PsbZR1wAPcG91eEthKhN+Zn92FMHxv+/faECIwjXdxfTODGSGegYv0sc1Olz+HYPvKOuoXfp+0pA2XVt2cI0Ig==", + "license": "MIT", + "dependencies": { + "openapi-typescript-helpers": "^0.1.0" + } + }, "node_modules/openapi-typescript": { "version": "7.13.0", "resolved": "https://registry.npmjs.org/openapi-typescript/-/openapi-typescript-7.13.0.tgz", @@ -10555,6 +10565,12 @@ "typescript": "^5.x" } }, + "node_modules/openapi-typescript-helpers": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/openapi-typescript-helpers/-/openapi-typescript-helpers-0.1.0.tgz", + "integrity": "sha512-OKTGPthhivLw/fHz6c3OPtg72vi86qaMlqbJuVJ23qOvQ+53uw1n7HdmkJFibloF7QEjDrDkzJiOJuockM/ljw==", + "license": "MIT" + }, "node_modules/openapi-typescript/node_modules/supports-color": { "version": "10.2.2", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-10.2.2.tgz", diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index ab01ec70cea..e35620c4996 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -44,6 +44,7 @@ "moment": "2.30.1", "next": "16.2.6", "openai": "4.104.0", + "openapi-fetch": "^0.17.0", "papaparse": "5.5.3", "react": "18.3.1", "react-copy-to-clipboard": "5.1.1", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/customers/useCustomers.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/customers/useCustomers.test.ts index 716d6f75399..1e614b709e2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/customers/useCustomers.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/customers/useCustomers.test.ts @@ -1,334 +1,104 @@ -import { allEndUsersCall } from "@/components/networking"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { renderHook, waitFor } from "@testing-library/react"; import React, { ReactNode } from "react"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import type { Customer, CustomersResponse } from "./useCustomers"; -import { useCustomers } from "./useCustomers"; +import { useCustomers, type EndUser } from "./useCustomers"; -// Mock the networking function -vi.mock("@/components/networking", () => ({ - allEndUsersCall: vi.fn(), +const mockGet = vi.fn(); +vi.mock("@/lib/http/api", () => ({ + fetchClient: { GET: (...args: unknown[]) => mockGet(...args) }, })); -// Mock useAuthorized hook - we can override this in individual tests const mockUseAuthorized = vi.fn(); vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: () => mockUseAuthorized(), })); -// Import actual roles instead of mocking them - -// Mock data -const mockCustomers: Customer[] = [ - { - user_id: "customer-1", - alias: "Test Customer 1", - spend: 150.5, - blocked: false, - allowed_model_region: "us-east-1", - default_model: "gpt-3.5-turbo", - budget_id: "budget-1", - litellm_budget_table: { - budget_id: "budget-1", - max_budget: 1000, - soft_budget: 800, - max_parallel_requests: 10, - tpm_limit: 1000, - rpm_limit: 100, - model_max_budget: { "gpt-4": 500 }, - budget_duration: "monthly", - budget_reset_at: "2024-02-01T00:00:00Z", - created_at: "2024-01-01T00:00:00Z", - created_by: "admin-1", - updated_at: "2024-01-01T00:00:00Z", - updated_by: "admin-1", - }, - }, - { - user_id: "customer-2", - alias: null, - spend: 0, - blocked: true, - allowed_model_region: null, - default_model: null, - budget_id: null, - litellm_budget_table: null, - }, +const mockCustomers: EndUser[] = [ + { user_id: "customer-1", alias: "Test Customer 1", spend: 150.5, blocked: false }, + { user_id: "customer-2", alias: null, spend: 0, blocked: true }, ]; -const mockCustomersResponse: CustomersResponse = mockCustomers; +const authorized = { + accessToken: "test-access-token", + userRole: "Admin", + userId: "test-user-id", + token: "test-token", + userEmail: "test@example.com", + premiumUser: false, + disabledPersonalKeyCreation: null, + showSSOBanner: false, +}; describe("useCustomers", () => { let queryClient: QueryClient; beforeEach(() => { - queryClient = new QueryClient({ - defaultOptions: { - queries: { - retry: false, - }, - }, - }); - - // Reset all mocks + queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); vi.clearAllMocks(); - - // Set default mock for useAuthorized (enabled state) - mockUseAuthorized.mockReturnValue({ - accessToken: "test-access-token", - userRole: "Admin", - userId: "test-user-id", - token: "test-token", - userEmail: "test@example.com", - premiumUser: false, - disabledPersonalKeyCreation: null, - showSSOBanner: false, - }); + mockUseAuthorized.mockReturnValue(authorized); }); const wrapper = ({ children }: { children: ReactNode }) => React.createElement(QueryClientProvider, { client: queryClient }, children); - it("should return customers data when query is successful", async () => { - // Mock successful API call - (allEndUsersCall as any).mockResolvedValue(mockCustomersResponse); + it("fetches /customer/list and returns the typed list on success", async () => { + mockGet.mockResolvedValue({ data: mockCustomers }); const { result } = renderHook(() => useCustomers(), { wrapper }); - // Initially loading expect(result.current.isLoading).toBe(true); - expect(result.current.data).toBeUndefined(); - // Wait for success await waitFor(() => { - expect(result.current.isLoading).toBe(false); expect(result.current.isSuccess).toBe(true); }); - expect(result.current.data).toEqual(mockCustomersResponse); - expect(result.current.error).toBeNull(); - expect(allEndUsersCall).toHaveBeenCalledWith("test-access-token"); - expect(allEndUsersCall).toHaveBeenCalledTimes(1); + expect(result.current.data).toEqual(mockCustomers); + expect(mockGet).toHaveBeenCalledWith("/customer/list"); + expect(mockGet).toHaveBeenCalledTimes(1); }); - it("should handle error when allEndUsersCall fails", async () => { - const errorMessage = "Failed to fetch customers"; - const testError = new Error(errorMessage); - - // Mock failed API call - (allEndUsersCall as any).mockRejectedValue(testError); + it("surfaces an error when the request rejects", async () => { + const testError = new Error("Failed to fetch customers"); + mockGet.mockRejectedValue(testError); const { result } = renderHook(() => useCustomers(), { wrapper }); - // Initially loading - expect(result.current.isLoading).toBe(true); - - // Wait for error await waitFor(() => { - expect(result.current.isLoading).toBe(false); expect(result.current.isError).toBe(true); }); expect(result.current.error).toEqual(testError); expect(result.current.data).toBeUndefined(); - expect(allEndUsersCall).toHaveBeenCalledWith("test-access-token"); - expect(allEndUsersCall).toHaveBeenCalledTimes(1); }); - it("should not execute query when accessToken is missing", async () => { - // Mock missing accessToken - mockUseAuthorized.mockReturnValue({ - accessToken: null, - userRole: "Admin", - userId: "test-user-id", - token: null, - userEmail: "test@example.com", - premiumUser: false, - disabledPersonalKeyCreation: null, - showSSOBanner: false, - }); + it("falls back to an empty list when the response has no body", async () => { + mockGet.mockResolvedValue({ data: undefined }); const { result } = renderHook(() => useCustomers(), { wrapper }); - // Query should not execute - expect(result.current.isLoading).toBe(false); - expect(result.current.data).toBeUndefined(); - expect(result.current.isFetched).toBe(false); - - // API should not be called - expect(allEndUsersCall).not.toHaveBeenCalled(); - }); - - it("should not execute query when userRole is not an admin role", async () => { - // Mock non-admin userRole - mockUseAuthorized.mockReturnValue({ - accessToken: "test-access-token", - userRole: "member", // Not in all_admin_roles - userId: "test-user-id", - token: "test-token", - userEmail: "test@example.com", - premiumUser: false, - disabledPersonalKeyCreation: null, - showSSOBanner: false, - }); - - const { result } = renderHook(() => useCustomers(), { wrapper }); - - // Query should not execute - expect(result.current.isLoading).toBe(false); - expect(result.current.data).toBeUndefined(); - expect(result.current.isFetched).toBe(false); - - // API should not be called - expect(allEndUsersCall).not.toHaveBeenCalled(); - }); - - it("should not execute query when userRole is null", async () => { - // Mock null userRole - mockUseAuthorized.mockReturnValue({ - accessToken: "test-access-token", - userRole: null, - userId: "test-user-id", - token: "test-token", - userEmail: "test@example.com", - premiumUser: false, - disabledPersonalKeyCreation: null, - showSSOBanner: false, - }); - - const { result } = renderHook(() => useCustomers(), { wrapper }); - - // Query should not execute - expect(result.current.isLoading).toBe(false); - expect(result.current.data).toBeUndefined(); - expect(result.current.isFetched).toBe(false); - - // API should not be called - expect(allEndUsersCall).not.toHaveBeenCalled(); - }); - - it("should not execute query when userRole is empty string", async () => { - // Mock empty string userRole - mockUseAuthorized.mockReturnValue({ - accessToken: "test-access-token", - userRole: "", - userId: "test-user-id", - token: "test-token", - userEmail: "test@example.com", - premiumUser: false, - disabledPersonalKeyCreation: null, - showSSOBanner: false, - }); - - const { result } = renderHook(() => useCustomers(), { wrapper }); - - // Query should not execute - expect(result.current.isLoading).toBe(false); - expect(result.current.data).toBeUndefined(); - expect(result.current.isFetched).toBe(false); - - // API should not be called - expect(allEndUsersCall).not.toHaveBeenCalled(); - }); - - it("should not execute query when both accessToken and userRole are missing", async () => { - // Mock both auth values missing - mockUseAuthorized.mockReturnValue({ - accessToken: null, - userRole: null, - userId: "test-user-id", - token: null, - userEmail: "test@example.com", - premiumUser: false, - disabledPersonalKeyCreation: null, - showSSOBanner: false, - }); - - const { result } = renderHook(() => useCustomers(), { wrapper }); - - // Query should not execute - expect(result.current.isLoading).toBe(false); - expect(result.current.data).toBeUndefined(); - expect(result.current.isFetched).toBe(false); - - // API should not be called - expect(allEndUsersCall).not.toHaveBeenCalled(); - }); - - it("should execute query when accessToken is present and userRole is Admin", async () => { - // Mock successful API call - (allEndUsersCall as any).mockResolvedValue(mockCustomersResponse); - - // Ensure auth values are set (already done in beforeEach) - const { result } = renderHook(() => useCustomers(), { wrapper }); - - // Wait for query to execute await waitFor(() => { - expect(result.current.isLoading).toBe(false); - }); - - expect(allEndUsersCall).toHaveBeenCalledWith("test-access-token"); - expect(allEndUsersCall).toHaveBeenCalledTimes(1); - }); - - it("should execute query when accessToken is present and userRole is proxy_admin", async () => { - // Mock successful API call - (allEndUsersCall as any).mockResolvedValue(mockCustomersResponse); - - // Mock proxy_admin role - mockUseAuthorized.mockReturnValue({ - accessToken: "test-access-token", - userRole: "proxy_admin", - userId: "test-user-id", - token: "test-token", - userEmail: "test@example.com", - premiumUser: false, - disabledPersonalKeyCreation: null, - showSSOBanner: false, - }); - - const { result } = renderHook(() => useCustomers(), { wrapper }); - - // Wait for query to execute - await waitFor(() => { - expect(result.current.isLoading).toBe(false); - }); - - expect(allEndUsersCall).toHaveBeenCalledWith("test-access-token"); - expect(allEndUsersCall).toHaveBeenCalledTimes(1); - }); - - it("should return empty customers array when API returns empty data", async () => { - // Mock API returning empty customers array - (allEndUsersCall as any).mockResolvedValue([]); - - const { result } = renderHook(() => useCustomers(), { wrapper }); - - // Wait for success - await waitFor(() => { - expect(result.current.isLoading).toBe(false); expect(result.current.isSuccess).toBe(true); }); expect(result.current.data).toEqual([]); - expect(allEndUsersCall).toHaveBeenCalledWith("test-access-token"); }); - it("should handle network timeout error", async () => { - const timeoutError = new Error("Network timeout"); - - // Mock network timeout - (allEndUsersCall as any).mockRejectedValue(timeoutError); + it("does not fetch when the access token is missing", () => { + mockUseAuthorized.mockReturnValue({ ...authorized, accessToken: null, token: null }); const { result } = renderHook(() => useCustomers(), { wrapper }); - // Wait for error - await waitFor(() => { - expect(result.current.isError).toBe(true); - }); + expect(result.current.isFetched).toBe(false); + expect(mockGet).not.toHaveBeenCalled(); + }); - expect(result.current.error).toEqual(timeoutError); - expect(result.current.data).toBeUndefined(); + it("does not fetch when the user is not an admin", () => { + mockUseAuthorized.mockReturnValue({ ...authorized, userRole: "member" }); + + const { result } = renderHook(() => useCustomers(), { wrapper }); + + expect(result.current.isFetched).toBe(false); + expect(mockGet).not.toHaveBeenCalled(); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/customers/useCustomers.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/customers/useCustomers.ts index d9f3e7cbb36..25e2e3f5e90 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/customers/useCustomers.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/customers/useCustomers.ts @@ -1,42 +1,19 @@ -import { allEndUsersCall } from "@/components/networking"; import { useQuery } from "@tanstack/react-query"; import { createQueryKeys } from "../common/queryKeysFactory"; +import { fetchClient } from "@/lib/http/api"; import { all_admin_roles } from "@/utils/roles"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import type { components } from "@/lib/http/schema"; + +export type EndUser = components["schemas"]["CustomerResponse"]; + const customersKeys = createQueryKeys("customers"); -export interface Customer { - user_id: string; - alias?: string | null; - spend: number; - blocked: boolean; - allowed_model_region?: string | null; - default_model?: string | null; - budget_id?: string | null; - litellm_budget_table?: { - budget_id: string; - max_budget?: number | null; - soft_budget?: number | null; - max_parallel_requests?: number | null; - tpm_limit?: number | null; - rpm_limit?: number | null; - model_max_budget?: Record | null; - budget_duration?: string | null; - budget_reset_at?: string | null; - created_at: string; - created_by: string; - updated_at: string; - updated_by: string; - } | null; -} - -export type CustomersResponse = Customer[]; - export const useCustomers = () => { const { accessToken, userRole } = useAuthorized(); - return useQuery({ + return useQuery({ queryKey: customersKeys.list({}), - queryFn: async () => await allEndUsersCall(accessToken!), + queryFn: async () => (await fetchClient.GET("/customer/list")).data ?? [], enabled: Boolean(accessToken) && all_admin_roles.includes(userRole!), }); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/workflows/WorkflowRuns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/workflows/WorkflowRuns.tsx index b668cdee0bd..9afa07251c2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/workflows/WorkflowRuns.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/workflows/WorkflowRuns.tsx @@ -1,9 +1,16 @@ import React, { useState, useEffect, useCallback, useMemo } from "react"; import { Button, Collapse, Drawer, Empty, Spin, Tooltip, Typography } from "antd"; import { ReloadOutlined } from "@ant-design/icons"; -import type { ColumnDef } from "@tanstack/react-table"; +import type { ColumnDef, ColumnFiltersState } from "@tanstack/react-table"; import { proxyBaseUrl } from "@/components/networking"; -import { DataTable } from "@/components/shared/DataTable"; +import { + DataTable, + DataTableFilterDrawer, + DataTableFilterField, + DataTableToolbar, +} from "@/components/shared/DataTable"; +import { Input } from "@/components/ui/input"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; const { Text } = Typography; @@ -59,6 +66,15 @@ const STATUS_DOT: Record = { failed: "#ef4444", }; +const RUN_STATUS_OPTIONS: RunStatus[] = ["pending", "running", "paused", "completed", "failed"]; +const STATUS_LABELS: Record = { + pending: "Pending", + running: "Running", + paused: "Paused", + completed: "Completed", + failed: "Failed", +}; + const EVENT_COLOR: Record = { "step.started": { bar: "#f0fdf4", border: "#86efac", text: "#16a34a" }, "step.failed": { bar: "#fef2f2", border: "#fca5a5", text: "#dc2626" }, @@ -482,6 +498,9 @@ const WorkflowRuns: React.FC = ({ accessToken }) => { const [messages, setMessages] = useState([]); const [loadingDetail, setLoadingDetail] = useState(false); const [drawerOpen, setDrawerOpen] = useState(false); + const [columnFilters, setColumnFilters] = useState([]); + const [globalFilter, setGlobalFilter] = useState(""); + const [filtersOpen, setFiltersOpen] = useState(false); const fetchRuns = useCallback(async () => { if (!accessToken) return; @@ -547,7 +566,9 @@ const WorkflowRuns: React.FC = ({ accessToken }) => { () => [ { id: "run", + accessorFn: (row) => `${runTitle(row)} ${row.run_id}`, header: "Run", + meta: { title: "Run", skeleton: "twoLine" }, cell: ({ row }) => { const run = row.original; return ( @@ -564,13 +585,18 @@ const WorkflowRuns: React.FC = ({ accessToken }) => { { accessorKey: "workflow_type", header: "Type", + meta: { title: "Type" }, + filterFn: "includesString", cell: ({ row }) => ( {row.original.workflow_type} ), }, { id: "status", + accessorKey: "status", header: "Status", + meta: { title: "Status" }, + filterFn: "equalsString", cell: ({ row }) => { const run = row.original; return ( @@ -586,6 +612,7 @@ const WorkflowRuns: React.FC = ({ accessToken }) => { { accessorKey: "created_at", header: "Created", + meta: { title: "Created" }, cell: ({ row }) => {timeAgo(row.original.created_at)}, }, ], @@ -603,28 +630,11 @@ const WorkflowRuns: React.FC = ({ accessToken }) => { }} > {/* page header */} -
-
-
Workflow Runs
-
- Durable state tracking for agents and automated workflows -
+
+
Workflow Runs
+
+ Durable state tracking for agents and automated workflows
-
= ({ accessToken }) => { } paginationMode="client" pageSizeOptions={[50, 100]} + filterMode="client" + columnFilters={columnFilters} + onColumnFiltersChange={setColumnFilters} + globalFilter={globalFilter} + onGlobalFilterChange={setGlobalFilter} onRowClick={fetchRunDetail} size="compact" + toolbar={(table) => ( + <> + setFiltersOpen(true)} + /> + + {({ get, set }) => ( + <> + + + + + set("workflow_type", event.target.value)} + placeholder="Filter by type…" + /> + + + )} + + + )} /> {/* detail drawer */} diff --git a/ui/litellm-dashboard/src/components/DashboardHeader.test.tsx b/ui/litellm-dashboard/src/components/DashboardHeader.test.tsx new file mode 100644 index 00000000000..aa83e70c56d --- /dev/null +++ b/ui/litellm-dashboard/src/components/DashboardHeader.test.tsx @@ -0,0 +1,57 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { DashboardHeader } from "./DashboardHeader"; + +const { mockUsePluginMode, mockUseUISettings, state } = vi.hoisted(() => { + const state = { + plugins: [] as { name: string; display_name: string; url: string }[], + enableChatUI: false, + }; + return { + state, + mockUsePluginMode: vi.fn(() => ({ mode: "ai-gateway", setMode: vi.fn(), plugins: state.plugins })), + mockUseUISettings: vi.fn(() => ({ data: { values: { enable_chat_ui: state.enableChatUI } } })), + }; +}); + +vi.mock("@/contexts/PluginModeContext", () => ({ usePluginMode: mockUsePluginMode })); +vi.mock("@/app/(dashboard)/hooks/uiSettings/useUISettings", () => ({ useUISettings: mockUseUISettings })); +vi.mock("next/navigation", () => ({ usePathname: () => "/ui/" })); +vi.mock("@/utils/migratedPages", () => ({ migratedHref: (seg: string) => `/ui/${seg}` })); +vi.mock("@/hooks/useWorker", () => ({ useWorker: () => ({ isControlPlane: false, selectedWorker: null }) })); +vi.mock("@/app/(dashboard)/hooks/useDisableShowPrompts", () => ({ useDisableShowPrompts: () => false })); +vi.mock("@/components/Navbar/BlogDropdown/BlogDropdown", () => ({ BlogDropdown: () => null })); +vi.mock("@/components/Navbar/CommunityEngagementButtons/CommunityEngagementButtons", () => ({ + CommunityEngagementButtons: () => null, +})); +vi.mock("@/components/Navbar/NotificationsBell/NotificationsBell", () => ({ NotificationsBell: () => null })); +vi.mock("@/components/Navbar/WorkerDropdown/WorkerDropdown", () => ({ default: () => null })); + +describe("DashboardHeader breadcrumb", () => { + afterEach(() => { + state.plugins = []; + state.enableChatUI = false; + }); + + it("roots the breadcrumb in the AI Gateway selector (with a Chat option) and drops the static section crumb when the selector is available", async () => { + state.enableChatUI = true; + render(); + + expect(screen.getByText("Logs")).toBeInTheDocument(); + expect(screen.queryByText("Observability")).not.toBeInTheDocument(); + + const selector = screen.getByRole("button", { name: /AI Gateway/i }); + act(() => { + fireEvent.click(selector); + }); + await waitFor(() => expect(screen.getByText("Chat")).toBeInTheDocument()); + }); + + it("keeps the AI Gateway selector at the root even when there is nothing to switch to (discovery)", () => { + render(); + + expect(screen.getByRole("button", { name: /AI Gateway/i })).toBeInTheDocument(); + expect(screen.getByText("Logs")).toBeInTheDocument(); + expect(screen.queryByText("Observability")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/DashboardHeader.tsx b/ui/litellm-dashboard/src/components/DashboardHeader.tsx index 4f285cefed9..8b928f8e6fe 100644 --- a/ui/litellm-dashboard/src/components/DashboardHeader.tsx +++ b/ui/litellm-dashboard/src/components/DashboardHeader.tsx @@ -27,7 +27,7 @@ interface DashboardHeaderProps { // Top bar for the dashboard shell. Sits only over the content column (the brand // lives in the sidebar header); mirrors the design's breadcrumb-left / tools-right layout. export function DashboardHeader({ page }: DashboardHeaderProps) { - const { section, title } = getBreadcrumb(page); + const { title } = getBreadcrumb(page); const { isControlPlane, selectedWorker } = useWorker(); const showWorkerSwitch = isControlPlane && selectedWorker !== null; const hideCommunityLinks = useDisableShowPrompts(); @@ -44,12 +44,10 @@ export function DashboardHeader({ page }: DashboardHeaderProps) {
- {section && ( - <> - {section} - - - )} + + + + {title} @@ -76,8 +74,6 @@ export function DashboardHeader({ page }: DashboardHeaderProps) { {!hideCommunityLinks && } - -
); diff --git a/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.test.tsx b/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.test.tsx index bdbf74cdf7d..d9a28b2ab8c 100644 --- a/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.test.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.test.tsx @@ -49,9 +49,23 @@ describe("ViewSwitcher", () => { state.setMode.mockClear(); }); - it("renders nothing with no plugins, chat disabled, and a non-admin user", () => { - const { container } = render(); - expect(container.firstChild).toBeNull(); + it("still renders the selector with a disabled Chat hint when there are no plugins and chat is off", async () => { + render(); + + const button = screen.getByRole("button"); + expect(button).toHaveTextContent("AI Gateway"); + + act(() => { + fireEvent.click(button); + }); + await waitFor(() => expect(screen.getByText("Chat")).toBeInTheDocument()); + expect(screen.getByText(/Admins can enable in Settings/i)).toBeInTheDocument(); + + act(() => { + fireEvent.click(screen.getByText("Chat")); + }); + expect(assignSpy).not.toHaveBeenCalled(); + expect(state.setMode).not.toHaveBeenCalled(); }); it("labels the button from the active plugin and lists AI Gateway + each plugin", async () => { @@ -95,7 +109,7 @@ describe("ViewSwitcher", () => { fireEvent.click(screen.getByRole("button")); }); await waitFor(() => expect(screen.getByText("Chat")).toBeInTheDocument()); - expect(screen.queryByText(/Enable in Admin Settings/i)).not.toBeInTheDocument(); + expect(screen.queryByText(/Admins can enable in Settings/i)).not.toBeInTheDocument(); act(() => { fireEvent.click(screen.getByText("Chat")); @@ -121,7 +135,7 @@ describe("ViewSwitcher", () => { expect(assignSpy).toHaveBeenCalledWith("/ui/"); }); - it("hides the Chat entry from everyone when disabled", async () => { + it("shows Chat as a disabled, non-navigating entry with an admin hint when disabled", async () => { state.enableChatUI = false; state.plugins = [{ name: "obs", display_name: "Observability", url: "http://localhost:9000" }]; render(); @@ -130,6 +144,12 @@ describe("ViewSwitcher", () => { fireEvent.click(screen.getByRole("button")); }); await waitFor(() => expect(screen.getByText("Observability")).toBeInTheDocument()); - expect(screen.queryByText("Chat")).not.toBeInTheDocument(); + expect(screen.getByText("Chat")).toBeInTheDocument(); + expect(screen.getByText(/Admins can enable in Settings/i)).toBeInTheDocument(); + + act(() => { + fireEvent.click(screen.getByText("Chat")); + }); + expect(assignSpy).not.toHaveBeenCalled(); }); }); diff --git a/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.tsx b/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.tsx index 834966aca35..09a4538ae18 100644 --- a/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.tsx @@ -1,7 +1,8 @@ import React from "react"; import { usePathname } from "next/navigation"; import { Dropdown } from "antd"; -import { AppstoreOutlined, CheckOutlined, DownOutlined } from "@ant-design/icons"; +import { AppstoreOutlined, CheckOutlined } from "@ant-design/icons"; +import { ChevronsUpDown } from "lucide-react"; import type { MenuProps } from "antd"; import { usePluginMode } from "@/contexts/PluginModeContext"; import { useUISettings } from "@/app/(dashboard)/hooks/uiSettings/useUISettings"; @@ -17,8 +18,6 @@ export default function ViewSwitcher() { const chatEnabled = Boolean(uiSettings?.values?.enable_chat_ui); - if (plugins.length === 0 && !chatEnabled) return null; - const chatHref = migratedHref(CHAT); const normalizedPathname = (pathname ?? "").replace(/\/+$/, ""); const isChatRoute = chatEnabled && (normalizedPathname === chatHref || normalizedPathname.startsWith(`${chatHref}/`)); @@ -30,6 +29,29 @@ export default function ViewSwitcher() { ...plugins.map((p) => ({ key: p.name, label: p.display_name })), ]; + const chatItem = chatEnabled + ? { + key: CHAT, + label: ( +
+ Chat + {isChatRoute && } +
+ ), + } + : { + key: CHAT, + disabled: true, + label: ( +
+ Chat + + Admins can enable in Settings + +
+ ), + }; + const items: MenuProps["items"] = [ ...modeEntries.map((e) => ({ key: e.key, @@ -40,19 +62,7 @@ export default function ViewSwitcher() {
), })), - ...(chatEnabled - ? [ - { - key: CHAT, - label: ( -
- Chat - {isChatRoute && } -
- ), - }, - ] - : []), + chatItem, ]; const onClick: MenuProps["onClick"] = ({ key }) => { @@ -72,11 +82,13 @@ export default function ViewSwitcher() { ); diff --git a/ui/litellm-dashboard/src/components/mcp_tools/DcrBridgeToggle.tsx b/ui/litellm-dashboard/src/components/mcp_tools/DcrBridgeToggle.tsx new file mode 100644 index 00000000000..f1b642293c0 --- /dev/null +++ b/ui/litellm-dashboard/src/components/mcp_tools/DcrBridgeToggle.tsx @@ -0,0 +1,40 @@ +import React from "react"; +import { Form, Switch, Tooltip } from "antd"; +import { InfoCircleOutlined } from "@ant-design/icons"; +import { isClientForwardedTokenMode } from "./types"; + +/** + * DCR-bridge toggle for the client-forwarded token modes (true_passthrough / + * oauth_delegate); self-gates to those two auth types and renders nothing + * otherwise. When on, OAuth-only clients like Claude Desktop can register and + * sign in through the gateway; when off, the gateway relays the upstream + * server's own OAuth metadata instead. `initialChecked` seeds the antd + * Form.Item `initialValue` (not the Switch's DOM defaultChecked): the create + * form defaults it on, the edit form seeds it from the stored value. + */ +export default function DcrBridgeToggle({ + authType, + initialChecked, +}: { + authType?: string | null; + initialChecked?: boolean; +}) { + if (!isClientForwardedTokenMode(authType)) return null; + return ( + + Gateway-hosted sign-in (DCR bridge) + + + + + } + name="dcr_bridge" + valuePropName="checked" + initialValue={initialChecked} + > + + + ); +} diff --git a/ui/litellm-dashboard/src/components/mcp_tools/PassthroughAuthorizeSection.test.tsx b/ui/litellm-dashboard/src/components/mcp_tools/PassthroughAuthorizeSection.test.tsx new file mode 100644 index 00000000000..0a09ef3f856 --- /dev/null +++ b/ui/litellm-dashboard/src/components/mcp_tools/PassthroughAuthorizeSection.test.tsx @@ -0,0 +1,54 @@ +import React from "react"; +import { describe, it, expect } from "vitest"; +import { render, screen } from "@testing-library/react"; +import { Form } from "antd"; +import PassthroughAuthorizeSection from "./PassthroughAuthorizeSection"; + +const WithForm: React.FC<{ children: React.ReactNode }> = ({ children }) => { + const [form] = Form.useForm(); + return
{children}
; +}; + +const noopFlow = { startOAuthFlow: () => {}, status: "idle", error: null, tokenResponse: null }; + +describe("PassthroughAuthorizeSection credential-class-aware copy", () => { + it("shows keep-existing copy when the credential class is unchanged (true_passthrough <-> oauth_delegate)", () => { + render( + + + , + ); + expect(screen.getByPlaceholderText("Leave blank to keep the currently saved app (if any)")).toBeInTheDocument(); + expect(screen.getByPlaceholderText("Leave blank to keep the currently saved secret (if any)")).toBeInTheDocument(); + }); + + it("shows the discard warning copy when switching from a different class (oauth2 -> true_passthrough)", () => { + render( + + + , + ); + expect(screen.getByPlaceholderText("Leave blank to use dynamic client registration")).toBeInTheDocument(); + expect(screen.getByPlaceholderText("Leave blank for public clients / PKCE")).toBeInTheDocument(); + expect(screen.getByText(/Switching the auth type discards the previously saved app/)).toBeInTheDocument(); + }); + + it("shows the keep+warn banner when the upstream may no longer match", () => { + render( + + + , + ); + expect(screen.getByText(/registered for the previous upstream/)).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/mcp_tools/PassthroughAuthorizeSection.tsx b/ui/litellm-dashboard/src/components/mcp_tools/PassthroughAuthorizeSection.tsx index af81f2713ae..0ed4ee555d1 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/PassthroughAuthorizeSection.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/PassthroughAuthorizeSection.tsx @@ -1,6 +1,7 @@ import React from "react"; -import { Button, Form, Input } from "antd"; -import { isClientForwardedTokenMode } from "./types"; +import { Button, Checkbox, Form, Input } from "antd"; +import DcrBridgeToggle from "./DcrBridgeToggle"; +import { credentialAuthClass, isClientForwardedTokenMode } from "./types"; interface PassthroughOAuthFlow { startOAuthFlow: () => void | Promise; @@ -11,20 +12,42 @@ interface PassthroughOAuthFlow { /** * Browser-only Authorize & Fetch for the client-forwarded token modes - * (true_passthrough / oauth_delegate). LiteLLM never stores upstream - * credentials for these modes, so the token obtained here lives in this - * browser session only: it is forwarded per-server for the tools preview and - * allowlist configuration, and is never written to the server row or the - * per-user credential store. The optional client credentials cover IdPs - * without dynamic client registration (e.g. a pre-registered Slack app) and - * ride the temporary authorize session only. + * (true_passthrough / oauth_delegate). Tokens are never stored: the token + * obtained here lives in this browser session only, forwarded per-server for + * the tools preview and allowlist configuration, and is never written to the + * server row or the per-user credential store. The optional OAuth client + * credentials cover IdPs without dynamic client registration (e.g. a + * pre-registered Slack app); unlike the token they ARE saved onto the server + * as declared config, so internal users' Authorize relays through the org's + * app instead of dead-ending on upstreams that cannot mint clients. + * + * Blank fields follow the same convention as the M2M credential fields. On + * create they mean "no app configured" (dynamic client registration). On edit + * they mean "keep existing" ONLY when the credential class is unchanged: the + * backend merges a partial update within the client-forwarded class, so a + * true_passthrough <-> oauth_delegate switch keeps the stored app, but a switch + * from a different class (e.g. oauth2) replaces it, so blanks then mean "no + * app". Removing a stored app is an explicit checkbox (edit only) that writes + * an explicit-null credential. */ export default function PassthroughAuthorizeSection({ authType, oauthFlow, + dcrBridgeInitialChecked, + isEditing = false, + savedAuthType, + removeStoredApp = false, + onRemoveStoredAppChange, + appMayNotMatchUpstream = false, }: { authType?: string | null; oauthFlow: PassthroughOAuthFlow; + dcrBridgeInitialChecked?: boolean; + isEditing?: boolean; + savedAuthType?: string | null; + removeStoredApp?: boolean; + onRemoveStoredAppChange?: (remove: boolean) => void; + appMayNotMatchUpstream?: boolean; }) { if (!isClientForwardedTokenMode(authType)) return null; const authorizeButtonLabels: Record = { @@ -32,32 +55,61 @@ export default function PassthroughAuthorizeSection({ exchanging: "Exchanging authorization code...", }; const authorizeButtonLabel = authorizeButtonLabels[oauthFlow.status] ?? "Authorize & Fetch Tools (browser-only)"; + // On edit, "keep existing" only holds when the stored credential class is unchanged; a cross-class + // switch (e.g. oauth2 -> true_passthrough) replaces credentials, so blanks then mean "no app". + const classUnchanged = isEditing && credentialAuthClass(savedAuthType) === credentialAuthClass(authType); + const clientIdPlaceholder = classUnchanged + ? "Leave blank to keep the currently saved app (if any)" + : "Leave blank to use dynamic client registration"; + const clientSecretPlaceholder = classUnchanged + ? "Leave blank to keep the currently saved secret (if any)" + : "Leave blank for public clients / PKCE"; + const clientIdExtra = classUnchanged + ? "Set this to make everyone authorize through a specific app; required for upstreams without dynamic client registration (e.g. a pre-registered Slack app)." + : "Switching the auth type discards the previously saved app; enter a client ID here or leave blank to use dynamic client registration."; return (

- Callers bring their own upstream token for this auth type, so LiteLLM stores no upstream credentials. To preview - tools and configure the tool allowlist, authorize against the upstream here: the token stays in this browser - session only and is never saved to LiteLLM. + Callers bring their own upstream token for this auth type, so LiteLLM never stores tokens. To preview tools and + configure the tool allowlist, authorize against the upstream here: the token stays in this browser session only + and is never saved to LiteLLM. An OAuth app configured below IS saved with the server, so internal users who + authorize from the Tools page go through it.

+ {appMayNotMatchUpstream && ( +

+ You changed the upstream URL or endpoints; the OAuth app entered here was registered for the previous upstream + and may not be valid. Update the client ID, or clear it to use dynamic client registration. +

+ )} OAuth Client ID (optional, not saved)} + label={OAuth Client ID (optional)} name={["credentials", "client_id"]} - extra="Only needed when the upstream does not support dynamic client registration (e.g. a pre-registered Slack app). Used for this browser authorization only." + extra={clientIdExtra} > OAuth Client Secret (optional, not saved)} + label={OAuth Client Secret (optional)} name={["credentials", "client_secret"]} > + + {isEditing && onRemoveStoredAppChange && ( + onRemoveStoredAppChange(e.target.checked)}> + + Remove the saved OAuth app on save (the server goes back to dynamic client registration) + + + )}
diff --git a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx index 02374a3ffa4..32fe439a316 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx @@ -31,6 +31,7 @@ const oauthHook = vi.hoisted(() => ({ | ((token: Record | null, registeredClient?: { clientId?: string; clientSecret?: string }) => void) | null, getCredentials: null as (() => Record | undefined) | null, + getTemporaryPayload: null as (() => Record | null) | null, })); vi.mock("@/hooks/useMcpOAuthFlow", () => ({ useMcpOAuthFlow: (opts: { @@ -39,9 +40,11 @@ vi.mock("@/hooks/useMcpOAuthFlow", () => ({ registeredClient?: { clientId?: string; clientSecret?: string }, ) => void; getCredentials?: () => Record | undefined; + getTemporaryPayload?: () => Record | null; }) => { oauthHook.onTokenReceived = opts.onTokenReceived; oauthHook.getCredentials = opts.getCredentials ?? null; + oauthHook.getTemporaryPayload = opts.getTemporaryPayload ?? null; return { startOAuthFlow: vi.fn(), status: "idle", @@ -202,8 +205,8 @@ describe("CreateMCPServer", () => { await waitFor(() => { expect(screen.getByRole("button", { name: "Authorize & Fetch Tools (browser-only)" })).toBeInTheDocument(); }); - expect(screen.getByText("OAuth Client ID (optional, not saved)")).toBeInTheDocument(); - expect(screen.getByText("OAuth Client Secret (optional, not saved)")).toBeInTheDocument(); + expect(screen.getByText("OAuth Client ID (optional)")).toBeInTheDocument(); + expect(screen.getByText("OAuth Client Secret (optional)")).toBeInTheDocument(); }, ); @@ -436,6 +439,434 @@ describe("CreateMCPServer", () => { ); }); + it.each([ + ["true_passthrough", "True Passthrough (no LiteLLM auth)"], + ["oauth_delegate", "OAuth Delegate (client-supplied upstream token)"], + ])( + "persists admin-entered OAuth app credentials on create for %s while the token stays browser-held", + async (_authType, optionLabel) => { + oauthHook.tokenResponse = { access_token: "upstream-tok", token_type: "Bearer" }; + await selectHttpTransport(); + + const user = userEvent.setup({ delay: null }); + await user.type(getServerNameInput(), "CF_App_Server"); + await user.type(screen.getByPlaceholderText("https://your-mcp-server.com"), "https://example.com/mcp"); + + await selectAntOption("Authentication", optionLabel); + + // Admin declares the org's pre-registered upstream app; unlike the browser-authorized + // token, this is config and must survive onto the server row so internal users' + // Tools-page Authorize relays through it (required for non-DCR upstreams like Slack). + await user.type( + screen.getByPlaceholderText("Leave blank to use dynamic client registration"), + "org-app-client-id", + ); + await user.type(screen.getByPlaceholderText("Leave blank for public clients / PKCE"), "org-app-secret"); + + await waitFor(() => expect(oauthHook.onTokenReceived).toBeTruthy()); + await act(async () => { + oauthHook.onTokenReceived!({ access_token: "upstream-tok", token_type: "Bearer" }, undefined); + }); + + const createdServer = { + server_id: "new-cf-app-server", + server_name: "CF_App_Server", + alias: "CF_App_Server", + url: "https://example.com/mcp", + transport: "http", + auth_type: _authType, + created_at: "2024-01-01T00:00:00Z", + created_by: "user-1", + updated_at: "2024-01-01T00:00:00Z", + updated_by: "user-1", + }; + vi.mocked(networking.createMCPServer).mockResolvedValue(createdServer); + + const submitButton = screen.getByRole("button", { name: "Add MCP Server" }); + await act(async () => { + fireEvent.click(submitButton); + }); + + await waitFor(() => expect(networking.createMCPServer).toHaveBeenCalledTimes(1)); + const [, payload] = vi.mocked(networking.createMCPServer).mock.calls[0]; + + // The declared app persists; the browser-authorized token still appears nowhere in the + // payload and no per-user DB credential is written. + expect(payload.credentials).toEqual({ + client_id: "org-app-client-id", + client_secret: "org-app-secret", + }); + expect(JSON.stringify(payload)).not.toContain("upstream-tok"); + expect(networking.storeMCPOAuthUserCredential).not.toHaveBeenCalled(); + expect(setToken).toHaveBeenCalledWith( + "new-cf-app-server", + expect.objectContaining({ access_token: "upstream-tok" }), + undefined, + ); + }, + ); + + it("preserves admin-entered app credentials when the URL changes after authorize for true_passthrough", async () => { + await selectHttpTransport(); + + const user = userEvent.setup({ delay: null }); + await user.type(getServerNameInput(), "CF_Keep_Server"); + await user.type(screen.getByPlaceholderText("https://your-mcp-server.com"), "https://example.com/mcp"); + + await selectAntOption("Authentication", "True Passthrough (no LiteLLM auth)"); + + await user.type( + screen.getByPlaceholderText("Leave blank to use dynamic client registration"), + "org-app-client-id", + ); + await user.type(screen.getByPlaceholderText("Leave blank for public clients / PKCE"), "org-app-secret"); + + await waitFor(() => expect(oauthHook.onTokenReceived).toBeTruthy()); + await act(async () => { + oauthHook.onTokenReceived!({ access_token: "upstream-tok", token_type: "Bearer" }, undefined); + }); + + // Editing the URL after authorize invalidates the held token (identity change), but the + // declared app is config, not minted material: it must survive the invalidation instead of + // being silently reset, or the server would persist without the configured app. + await act(async () => { + fireEvent.change(screen.getByPlaceholderText("https://your-mcp-server.com"), { + target: { value: "https://other.example.com/mcp" }, + }); + }); + + const keptAppServer = { + server_id: "kept-app-server", + server_name: "CF_Keep_Server", + alias: "CF_Keep_Server", + url: "https://other.example.com/mcp", + transport: "http", + auth_type: "true_passthrough", + created_at: "2024-01-01T00:00:00Z", + created_by: "user-1", + updated_at: "2024-01-01T00:00:00Z", + updated_by: "user-1", + }; + vi.mocked(networking.createMCPServer).mockResolvedValue(keptAppServer); + + await act(async () => { + fireEvent.click(screen.getByRole("button", { name: "Add MCP Server" })); + }); + + await waitFor(() => expect(networking.createMCPServer).toHaveBeenCalledTimes(1)); + const [, payload] = vi.mocked(networking.createMCPServer).mock.calls[0]; + expect(payload.url).toBe("https://other.example.com/mcp"); + expect(payload.credentials).toEqual({ + client_id: "org-app-client-id", + client_secret: "org-app-secret", + }); + expect(JSON.stringify(payload)).not.toContain("upstream-tok"); + }); + + it("wipes oauth2-minted credentials when the auth type switches to a client-forwarded mode", async () => { + await selectHttpTransport(); + + const user = userEvent.setup({ delay: null }); + await user.type(getServerNameInput(), "Switch_Server"); + await user.type(screen.getByPlaceholderText("https://your-mcp-server.com"), "https://example.com/mcp"); + + await selectAntOption("Authentication", "OAuth"); + + // The oauth2 onTokenReceived branch writes the fetched token AND the DCR client into + // form.credentials; both are minted for the oauth2 identity. + await waitFor(() => expect(oauthHook.onTokenReceived).toBeTruthy()); + await act(async () => { + oauthHook.onTokenReceived!( + { access_token: "oauth2-minted-tok", refresh_token: "oauth2-minted-refresh", token_type: "Bearer" }, + { clientId: "dcr-minted-client", clientSecret: "dcr-minted-secret" }, + ); + }); + + // Switching into a client-forwarded mode changes the identity with auth_type in the changed + // values, so the preserve carve-out must NOT apply: the minted material would otherwise ride + // into a mode that now persists credentials onto the server row. + await selectAntOption("Authentication", "True Passthrough (no LiteLLM auth)"); + + const switchedServer = { + server_id: "switched-server", + server_name: "Switch_Server", + alias: "Switch_Server", + url: "https://example.com/mcp", + transport: "http", + auth_type: "true_passthrough", + created_at: "2024-01-01T00:00:00Z", + created_by: "user-1", + updated_at: "2024-01-01T00:00:00Z", + updated_by: "user-1", + }; + vi.mocked(networking.createMCPServer).mockResolvedValue(switchedServer); + + await act(async () => { + fireEvent.click(screen.getByRole("button", { name: "Add MCP Server" })); + }); + + await waitFor(() => expect(networking.createMCPServer).toHaveBeenCalledTimes(1)); + const [, payload] = vi.mocked(networking.createMCPServer).mock.calls[0]; + expect(payload.credentials).toBeUndefined(); + expect(JSON.stringify(payload)).not.toContain("dcr-minted-client"); + expect(JSON.stringify(payload)).not.toContain("oauth2-minted-tok"); + }); + + it("keeps the DCR-minted client out of form.credentials but reuses it via getCredentials", async () => { + await selectHttpTransport(); + const user = userEvent.setup({ delay: null }); + await user.type(getServerNameInput(), "DCR_Server"); + await user.type(screen.getByPlaceholderText("https://your-mcp-server.com"), "https://example.com/mcp"); + await selectAntOption("Authentication", "OAuth"); + + await waitFor(() => expect(oauthHook.onTokenReceived).toBeTruthy()); + await act(async () => { + oauthHook.onTokenReceived!( + { access_token: "oauth2-tok", token_type: "Bearer" }, + { clientId: "dcr-client", clientSecret: "dcr-secret" }, + ); + }); + + // The DCR client must NOT be in the form store (or it could be collected as a CF server's app), + // but getCredentials merges it so a re-authorize reuses the registered client instead of re-DCRing. + expect(oauthHook.getCredentials?.()?.client_id).toBe("dcr-client"); + // getTemporaryPayload must mirror getCredentials for oauth2, or a re-authorize's temp session omits + // the registered client and useMcpOAuthFlow re-registers instead of reusing it. + expect(oauthHook.getTemporaryPayload?.()?.credentials).toMatchObject({ client_id: "dcr-client" }); + }); + + it("clears the DCR ref and the upstream warning when the modal closes so nothing leaks to the next session", async () => { + const { rerender } = render(); + await selectAntOption("Transport Type", "Streamable HTTP"); + await waitFor(() => expect(screen.getByPlaceholderText("https://your-mcp-server.com")).toBeInTheDocument()); + const user = userEvent.setup({ delay: null }); + await user.type(getServerNameInput(), "Leak_Server"); + await user.type(screen.getByPlaceholderText("https://your-mcp-server.com"), "https://example.com/mcp"); + await selectAntOption("Authentication", "OAuth"); + + await waitFor(() => expect(oauthHook.onTokenReceived).toBeTruthy()); + await act(async () => { + oauthHook.onTokenReceived!( + { access_token: "oauth2-tok", token_type: "Bearer" }, + { clientId: "leak-client", clientSecret: "leak-secret" }, + ); + }); + // Ref is held while the modal is open. + expect(oauthHook.getCredentials?.()?.client_id).toBe("leak-client"); + + // A parent dismiss (isModalVisible -> false) that does not route through Cancel/Create must still + // clear the DCR ref, or the next server's oauth2 submit would carry this server's registered client. + await act(async () => { + rerender(); + }); + + expect(oauthHook.getCredentials?.()?.client_id).toBeUndefined(); + expect(oauthHook.getTemporaryPayload?.()?.credentials ?? {}).not.toMatchObject({ client_id: "leak-client" }); + }); + + it("persists the DCR client on an oauth2 submit via the ref", async () => { + await selectHttpTransport(); + const user = userEvent.setup({ delay: null }); + await user.type(getServerNameInput(), "DCR_Submit_Server"); + await user.type(screen.getByPlaceholderText("https://your-mcp-server.com"), "https://example.com/mcp"); + await selectAntOption("Authentication", "OAuth"); + + await waitFor(() => expect(oauthHook.onTokenReceived).toBeTruthy()); + await act(async () => { + oauthHook.onTokenReceived!( + { access_token: "oauth2-tok", token_type: "Bearer" }, + { clientId: "dcr-client", clientSecret: "dcr-secret" }, + ); + }); + + const dcrSubmitServer = { + server_id: "dcr-submit", + server_name: "DCR_Submit_Server", + alias: "DCR_Submit_Server", + url: "https://example.com/mcp", + transport: "http", + auth_type: "oauth2", + created_at: "2024-01-01T00:00:00Z", + created_by: "user-1", + updated_at: "2024-01-01T00:00:00Z", + updated_by: "user-1", + }; + vi.mocked(networking.createMCPServer).mockResolvedValue(dcrSubmitServer); + await act(async () => { + fireEvent.click(screen.getByRole("button", { name: "Add MCP Server" })); + }); + + await waitFor(() => expect(networking.createMCPServer).toHaveBeenCalledTimes(1)); + const [, payload] = vi.mocked(networking.createMCPServer).mock.calls[0]; + expect(payload.credentials.client_id).toBe("dcr-client"); + expect(payload.credentials.client_secret).toBe("dcr-secret"); + }); + + // These two tests drive multiple antd auth-type switches; use single-shot fireEvent.change for the + // text fields (not per-keystroke userEvent.type) and a wider timeout so they do not flake under CI + // resource contention. The behavior under test is the credential preserve across the switches. + const fillText = (el: HTMLElement, value: string) => fireEvent.change(el, { target: { value } }); + + it("preserves the typed app across a switch between the two client-forwarded modes", async () => { + await selectHttpTransport(); + fillText(getServerNameInput(), "CF_Switch_Keep"); + fillText(screen.getByPlaceholderText("https://your-mcp-server.com"), "https://example.com/mcp"); + await selectAntOption("Authentication", "True Passthrough (no LiteLLM auth)"); + fillText(screen.getByPlaceholderText("Leave blank to use dynamic client registration"), "app-id"); + fillText(screen.getByPlaceholderText("Leave blank for public clients / PKCE"), "app-secret"); + + await waitFor(() => expect(oauthHook.onTokenReceived).toBeTruthy()); + await act(async () => { + oauthHook.onTokenReceived!({ access_token: "cf-tok", token_type: "Bearer" }, undefined); + }); + + await selectAntOption("Authentication", "OAuth Delegate (client-supplied upstream token)"); + + const switched = { + server_id: "cf-switch-keep", + server_name: "CF_Switch_Keep", + alias: "CF_Switch_Keep", + url: "https://example.com/mcp", + transport: "http", + auth_type: "oauth_delegate", + created_at: "2024-01-01T00:00:00Z", + created_by: "user-1", + updated_at: "2024-01-01T00:00:00Z", + updated_by: "user-1", + }; + vi.mocked(networking.createMCPServer).mockResolvedValue(switched); + await act(async () => { + fireEvent.click(screen.getByRole("button", { name: "Add MCP Server" })); + }); + + await waitFor(() => expect(networking.createMCPServer).toHaveBeenCalledTimes(1)); + const [, payload] = vi.mocked(networking.createMCPServer).mock.calls[0]; + expect(payload.credentials).toEqual({ client_id: "app-id", client_secret: "app-secret" }); + }, 60_000); + + it("preserves the typed app across a client-forwarded -> oauth2 -> client-forwarded round trip", async () => { + await selectHttpTransport(); + fillText(getServerNameInput(), "CF_Round"); + fillText(screen.getByPlaceholderText("https://your-mcp-server.com"), "https://example.com/mcp"); + await selectAntOption("Authentication", "True Passthrough (no LiteLLM auth)"); + fillText(screen.getByPlaceholderText("Leave blank to use dynamic client registration"), "app-id"); + fillText(screen.getByPlaceholderText("Leave blank for public clients / PKCE"), "app-secret"); + + await waitFor(() => expect(oauthHook.onTokenReceived).toBeTruthy()); + await act(async () => { + oauthHook.onTokenReceived!({ access_token: "cf-tok", token_type: "Bearer" }, undefined); + }); + + await selectAntOption("Authentication", "OAuth"); + await selectAntOption("Authentication", "True Passthrough (no LiteLLM auth)"); + + const cfRoundServer = { + server_id: "cf-round", + server_name: "CF_Round", + alias: "CF_Round", + url: "https://example.com/mcp", + transport: "http", + auth_type: "true_passthrough", + created_at: "2024-01-01T00:00:00Z", + created_by: "user-1", + updated_at: "2024-01-01T00:00:00Z", + updated_by: "user-1", + }; + vi.mocked(networking.createMCPServer).mockResolvedValue(cfRoundServer); + await act(async () => { + fireEvent.click(screen.getByRole("button", { name: "Add MCP Server" })); + }); + + await waitFor(() => expect(networking.createMCPServer).toHaveBeenCalledTimes(1)); + const [, payload] = vi.mocked(networking.createMCPServer).mock.calls[0]; + expect(payload.credentials).toEqual({ client_id: "app-id", client_secret: "app-secret" }); + }, 60_000); + + it("keeps the typed app but warns when the URL changes after a client-forwarded authorize", async () => { + await selectHttpTransport(); + const user = userEvent.setup({ delay: null }); + await user.type(getServerNameInput(), "CF_Warn"); + await user.type(screen.getByPlaceholderText("https://your-mcp-server.com"), "https://example.com/mcp"); + await selectAntOption("Authentication", "True Passthrough (no LiteLLM auth)"); + await user.type(screen.getByPlaceholderText("Leave blank to use dynamic client registration"), "app-id"); + + await waitFor(() => expect(oauthHook.onTokenReceived).toBeTruthy()); + await act(async () => { + oauthHook.onTokenReceived!({ access_token: "cf-tok", token_type: "Bearer" }, undefined); + }); + + await act(async () => { + fireEvent.change(screen.getByPlaceholderText("https://your-mcp-server.com"), { + target: { value: "https://other.example.com/mcp" }, + }); + }); + + // Keep + warn: the app stays in the field, and a non-blocking warning appears. + expect(screen.getByText(/OAuth app entered here was registered for the previous upstream/)).toBeInTheDocument(); + }); + + it("keeps client_secret when only client_id is edited after a client-forwarded authorize", async () => { + await selectHttpTransport(); + const user = userEvent.setup({ delay: null }); + await user.type(getServerNameInput(), "CF_Keystroke"); + await user.type(screen.getByPlaceholderText("https://your-mcp-server.com"), "https://example.com/mcp"); + await selectAntOption("Authentication", "True Passthrough (no LiteLLM auth)"); + await user.type(screen.getByPlaceholderText("Leave blank to use dynamic client registration"), "app-id"); + await user.type(screen.getByPlaceholderText("Leave blank for public clients / PKCE"), "app-secret"); + + await waitFor(() => expect(oauthHook.onTokenReceived).toBeTruthy()); + await act(async () => { + oauthHook.onTokenReceived!({ access_token: "cf-tok", token_type: "Bearer" }, undefined); + }); + + // Editing only client_id fires an invalidation whose changedValues carries only the client_id + // sub-field; the preserve + deep-merge re-apply must keep client_secret from being dropped. + await user.type(screen.getByPlaceholderText("Leave blank to use dynamic client registration"), "2"); + + const cfKeystrokeServer = { + server_id: "cf-keystroke", + server_name: "CF_Keystroke", + alias: "CF_Keystroke", + url: "https://example.com/mcp", + transport: "http", + auth_type: "true_passthrough", + created_at: "2024-01-01T00:00:00Z", + created_by: "user-1", + updated_at: "2024-01-01T00:00:00Z", + updated_by: "user-1", + }; + vi.mocked(networking.createMCPServer).mockResolvedValue(cfKeystrokeServer); + await act(async () => { + fireEvent.click(screen.getByRole("button", { name: "Add MCP Server" })); + }); + + await waitFor(() => expect(networking.createMCPServer).toHaveBeenCalledTimes(1)); + const [, payload] = vi.mocked(networking.createMCPServer).mock.calls[0]; + expect(payload.credentials).toEqual({ client_id: "app-id2", client_secret: "app-secret" }); + }); + + it("replaces the token set on re-authorize instead of leaving stale siblings", async () => { + await selectHttpTransport(); + const user = userEvent.setup({ delay: null }); + await user.type(getServerNameInput(), "Reauth_Server"); + await user.type(screen.getByPlaceholderText("https://your-mcp-server.com"), "https://example.com/mcp"); + await selectAntOption("Authentication", "OAuth"); + + await waitFor(() => expect(oauthHook.onTokenReceived).toBeTruthy()); + const firstToken = { access_token: "T1", refresh_token: "R1", scope: "read", token_type: "Bearer" }; + await act(async () => { + oauthHook.onTokenReceived!(firstToken, undefined); + }); + await act(async () => { + oauthHook.onTokenReceived!({ access_token: "T2", token_type: "Bearer" }, undefined); + }); + + const creds = oauthHook.getCredentials?.() ?? {}; + expect(creds.access_token).toBe("T2"); + expect(creds.refresh_token).toBeUndefined(); + expect(creds.scope).toBeUndefined(); + }); + it("should not show auth value field when None auth type is selected", async () => { await selectHttpTransport(); @@ -1509,3 +1940,181 @@ describe("CreateMCPServer oauth2_flow persistence", () => { expect(payload.oauth2_flow).toBeUndefined(); }); }); + +describe("CreateMCPServer dcr_bridge toggle", () => { + beforeEach(() => { + vi.clearAllMocks(); + oauthHook.tokenResponse = null; + oauthHook.onTokenReceived = null; + }); + + const createdServer = { + server_id: "new-cf-server", + server_name: "CF_Server", + alias: "CF_Server", + url: "https://example.com/mcp", + transport: "http", + auth_type: "true_passthrough", + created_at: "2024-01-01T00:00:00Z", + created_by: "user-1", + updated_at: "2024-01-01T00:00:00Z", + updated_by: "user-1", + }; + + const getDcrToggle = () => document.getElementById("dcr_bridge"); + + async function setupHttpServerForm() { + render(); + await selectAntOption("Transport Type", "Streamable HTTP"); + await waitFor(() => { + expect(screen.getByPlaceholderText("https://your-mcp-server.com")).toBeInTheDocument(); + }); + await act(async () => { + fireEvent.change(getServerNameInput(), { target: { value: "CF_Server" } }); + }); + await act(async () => { + fireEvent.change(screen.getByPlaceholderText("https://your-mcp-server.com"), { + target: { value: "https://example.com/mcp" }, + }); + }); + } + + async function submitCreate() { + const submitButton = screen.getByRole("button", { name: "Add MCP Server" }); + await act(async () => { + fireEvent.click(submitButton); + }); + await waitFor(() => { + expect(networking.createMCPServer).toHaveBeenCalledTimes(1); + }); + const [, payload] = vi.mocked(networking.createMCPServer).mock.calls[0]; + return payload; + } + + it.each([["True Passthrough (no LiteLLM auth)"], ["OAuth Delegate (client-supplied upstream token)"]])( + "renders the toggle default-checked when %s is selected", + async (optionLabel) => { + await setupHttpServerForm(); + + await selectAntOption("Authentication", optionLabel); + + await waitFor(() => { + expect(getDcrToggle()).toBeInTheDocument(); + }); + expect(screen.getByText("Gateway-hosted sign-in (DCR bridge)")).toBeInTheDocument(); + expect(getDcrToggle()).toHaveAttribute("aria-checked", "true"); + }, + ); + + it.each([["None"], ["API Key"], ["OAuth"]])("does not render the toggle for %s", async (optionLabel) => { + await setupHttpServerForm(); + + await selectAntOption("Authentication", optionLabel); + + await waitFor(() => { + expect(screen.queryByText("Gateway-hosted sign-in (DCR bridge)")).not.toBeInTheDocument(); + }); + expect(getDcrToggle()).not.toBeInTheDocument(); + }); + + it("renders the toggle between the OAuth client fields and the Authorize button", async () => { + await setupHttpServerForm(); + + await selectAntOption("Authentication", "True Passthrough (no LiteLLM auth)"); + + await waitFor(() => { + expect(getDcrToggle()).toBeInTheDocument(); + }); + const toggle = getDcrToggle() as HTMLElement; + const secretInput = screen.getByPlaceholderText("Leave blank for public clients / PKCE"); + const authorizeButton = screen.getByRole("button", { name: "Authorize & Fetch Tools (browser-only)" }); + expect(secretInput.compareDocumentPosition(toggle) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); + expect(toggle.compareDocumentPosition(authorizeButton) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); + }); + + it.each([ + ["true_passthrough", "True Passthrough (no LiteLLM auth)"], + ["oauth_delegate", "OAuth Delegate (client-supplied upstream token)"], + ])("sends dcr_bridge: true by default on create for %s", async (authType, optionLabel) => { + vi.mocked(networking.createMCPServer).mockResolvedValue({ ...createdServer, auth_type: authType }); + await setupHttpServerForm(); + await selectAntOption("Authentication", optionLabel); + await waitFor(() => { + expect(getDcrToggle()).toBeInTheDocument(); + }); + + const payload = await submitCreate(); + expect(payload.dcr_bridge).toBe(true); + }); + + it("sends an explicit dcr_bridge: false when the toggle is unchecked", async () => { + vi.mocked(networking.createMCPServer).mockResolvedValue({ ...createdServer, auth_type: "oauth_delegate" }); + await setupHttpServerForm(); + await selectAntOption("Authentication", "OAuth Delegate (client-supplied upstream token)"); + await waitFor(() => { + expect(getDcrToggle()).toBeInTheDocument(); + }); + + await act(async () => { + fireEvent.click(getDcrToggle()!); + }); + expect(getDcrToggle()).toHaveAttribute("aria-checked", "false"); + + const payload = await submitCreate(); + expect(payload.dcr_bridge).toBe(false); + }); + + it.each([ + ["none", "None"], + ["api_key", "API Key"], + ["oauth2", "OAuth"], + ])("forces an explicit dcr_bridge: false for %s", async (authType, optionLabel) => { + vi.mocked(networking.createMCPServer).mockResolvedValue({ ...createdServer, auth_type: authType }); + await setupHttpServerForm(); + await selectAntOption("Authentication", optionLabel); + + const payload = await submitCreate(); + expect(payload.dcr_bridge).toBe(false); + }); + + it("forces dcr_bridge: false when the auth type is switched away after toggling", async () => { + vi.mocked(networking.createMCPServer).mockResolvedValue({ ...createdServer, auth_type: "none" }); + await setupHttpServerForm(); + await selectAntOption("Authentication", "True Passthrough (no LiteLLM auth)"); + await waitFor(() => { + expect(getDcrToggle()).toBeInTheDocument(); + }); + await act(async () => { + fireEvent.click(getDcrToggle()!); + }); + + await selectAntOption("Authentication", "None"); + await waitFor(() => { + expect(getDcrToggle()).not.toBeInTheDocument(); + }); + + const payload = await submitCreate(); + expect(payload.dcr_bridge).toBe(false); + }); + + it("preserves the toggle value when switching between the two client-forwarded modes", async () => { + vi.mocked(networking.createMCPServer).mockResolvedValue({ ...createdServer, auth_type: "oauth_delegate" }); + await setupHttpServerForm(); + await selectAntOption("Authentication", "True Passthrough (no LiteLLM auth)"); + await waitFor(() => { + expect(getDcrToggle()).toBeInTheDocument(); + }); + expect(getDcrToggle()).toHaveAttribute("aria-checked", "true"); + + // The Form.Item is mounted in both client-forwarded modes, so switching between them keeps the + // live toggle value rather than forcing it back to the default or to false. + await selectAntOption("Authentication", "OAuth Delegate (client-supplied upstream token)"); + await waitFor(() => { + expect(getDcrToggle()).toBeInTheDocument(); + }); + expect(getDcrToggle()).toHaveAttribute("aria-checked", "true"); + + const payload = await submitCreate(); + expect(payload.dcr_bridge).toBe(true); + }); +}); diff --git a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx index eb48fd02474..b9a6f5d229d 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx @@ -18,6 +18,8 @@ import { getOAuthAuthorizationIdentity, CLEARED_ON_INVALIDATION, isHeldOAuthTokenStale, + preservedDeclaredAppCredentials, + withoutMintedTokenCredentials, } from "./types"; import OAuthFormFields from "./OAuthFormFields"; import TruePassthroughWarning from "./TruePassthroughWarning"; @@ -60,6 +62,8 @@ const AUTH_TYPES_REQUIRING_CREDENTIALS = [ AUTH_TYPE.OAUTH2, AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE, AUTH_TYPE.AWS_SIGV4, + AUTH_TYPE.TRUE_PASSTHROUGH, + AUTH_TYPE.OAUTH_DELEGATE, ]; const CREATE_OAUTH_UI_STATE_KEY = "litellm-mcp-oauth-create-state"; @@ -106,6 +110,14 @@ const CreateMCPServer: React.FC = ({ // was fetched; undefined when no valid token is held. If any mint-relevant field diverges from this, // the held token is stale and is discarded so the admin must re-authorize. const [authorizedIdentity, setAuthorizedIdentity] = useState(undefined); + // The DCR-minted OAuth client from an interactive (oauth2) Authorize. Held OUT of form.credentials so + // it can never be collected as a client-forwarded server's declared app; injected into the payload + // only on an oauth2 submit (where persisting the registered client is correct), and cleared on any + // invalidation or modal close. An abandoned authorize leaves it null, which is the desired asymmetry. + const dcrClientRef = React.useRef<{ client_id: string; client_secret?: string } | null>(null); + // Set when the upstream identity (url/endpoints) changed while a declared app is present, so the + // section can warn that the saved app may not match the new upstream (the app is kept, not wiped). + const [appMayNotMatchUpstream, setAppMayNotMatchUpstream] = useState(false); // Single hook call shared by MCPConnectionStatus and MCPToolConfiguration to avoid duplicate requests. const { @@ -147,6 +159,9 @@ const CreateMCPServer: React.FC = ({ searchValue, aliasManuallyEdited, logoUrl, + // Persist the identity so invalidation stays armed across the OAuth redirect round trip: a + // post-restore url/mode edit must still discard the held token instead of silently keeping it. + authorizedIdentity, }; setSecureItem(CREATE_OAUTH_UI_STATE_KEY, JSON.stringify(uiState)); } catch (err) { @@ -162,7 +177,12 @@ const CreateMCPServer: React.FC = ({ reset: resetOAuthFlow, } = useMcpOAuthFlow({ accessToken, - getCredentials: () => form.getFieldValue("credentials"), + // Merge the ref-held DCR client so a re-authorize reuses the registered client instead of + // re-registering; the form store itself never holds the DCR client (see onTokenReceived). + getCredentials: () => ({ + ...((form.getFieldValue("credentials") as Record | undefined) ?? {}), + ...(dcrClientRef.current ?? {}), + }), getTemporaryPayload: () => { const values = form.getFieldsValue(true); const transport = values.transport || transportType; @@ -184,7 +204,12 @@ const CreateMCPServer: React.FC = ({ url, transport: transport === TRANSPORT.OPENAPI ? "http" : transport, auth_type: isClientForwardedTokenMode(values.auth_type) ? values.auth_type : AUTH_TYPE.OAUTH2, - credentials: values.credentials, + // Mirror getCredentials: merge the ref-held DCR client for oauth2 so a re-authorize reuses the + // registered client (useMcpOAuthFlow keys reuse off credentials.client_id) instead of re-DCRing; + // the client-forwarded modes carry only the declared app. + credentials: isClientForwardedTokenMode(values.auth_type) + ? preservedDeclaredAppCredentials(values.credentials) + : { ...((values.credentials as Record | undefined) ?? {}), ...(dcrClientRef.current ?? {}) }, authorization_url: values.authorization_url, token_url: values.token_url, registration_url: values.registration_url, @@ -209,23 +234,36 @@ const CreateMCPServer: React.FC = ({ // edit form's onTokenReceived early return. setAuthorizedIdentity(getOAuthAuthorizationIdentity(form.getFieldsValue(true))); NotificationsManager.success( - "Token held for this browser session. Tools can now be previewed and configured; nothing will be saved to LiteLLM.", + "Token held for this browser session. Tools can now be previewed and configured; the token is not saved to LiteLLM.", ); return; } - const credentials = { + // The DCR-minted client is held in a ref, NOT written into form.credentials, so it can never be + // collected as a client-forwarded server's declared app; it is injected into the payload only on + // an oauth2 submit. An admin-typed client already lives in form.credentials and is left untouched. + dcrClientRef.current = registeredClient?.clientId + ? { + client_id: registeredClient.clientId, + ...(registeredClient.clientSecret && { client_secret: registeredClient.clientSecret }), + } + : null; + + const current = (form.getFieldValue("credentials") as Record | undefined) ?? {}; + const nextCredentials = { + ...(preservedDeclaredAppCredentials(current) ?? {}), + ...(current.scopes !== undefined && { scopes: current.scopes }), access_token: token.access_token, ...(token.refresh_token && { refresh_token: token.refresh_token }), ...(token.expires_in && { expires_in: token.expires_in }), ...(token.scope && { scope: token.scope }), - ...(registeredClient?.clientId && { client_id: registeredClient.clientId }), - ...(registeredClient?.clientSecret && { client_secret: registeredClient.clientSecret }), }; - - form.setFieldsValue({ credentials }); - // Capture the identity AFTER writing the DCR'd credentials so the held token is not spuriously - // invalidated by its own credential write. + // Path-replace (not deep-merge) so a re-authorize with fewer token fields does not leave stale + // siblings from the previous token behind; the admin-typed client keys and scopes are carried + // explicitly above. + form.setFieldValue("credentials", nextCredentials); + // Capture the identity AFTER writing the token so the held token is not spuriously invalidated by + // its own credential write. setAuthorizedIdentity(getOAuthAuthorizationIdentity(form.getFieldsValue(true))); NotificationsManager.success( @@ -246,7 +284,17 @@ const CreateMCPServer: React.FC = ({ clearTools(); resetOAuthFlow(); setAuthorizedIdentity(undefined); + dcrClientRef.current = null; + // Capture the admin-typed app before resetFields destroys it, then re-apply it: the app is + // upstream-scoped config, not minted material, so it survives every invalidation (the token is + // what gets discarded). Token-shaped keys are excluded by the helper's key filter. + const keptAppCredentials = preservedDeclaredAppCredentials(form.getFieldValue("credentials")); form.resetFields([...CLEARED_ON_INVALIDATION]); + if (keptAppCredentials) { + form.setFieldsValue({ credentials: keptAppCredentials }); + } + // Re-apply the in-flight edit last; rc-field-form deep-merges nested objects, so a changed + // credentials sub-field composes with the preserved sibling instead of replacing the object. const preserved = Object.fromEntries( CLEARED_ON_INVALIDATION.filter((key) => key in changedValues).map((key) => [key, changedValues[key]]), ); @@ -274,7 +322,18 @@ const CreateMCPServer: React.FC = ({ setTransportType(restoredTransport); } if (parsed.formValues) { - setPendingRestoredValues({ values: parsed.formValues, transport: restoredTransport }); + // Assign the cleaned credentials (strip minted token material so a stale token never rehydrates); + // the declared app the admin typed is kept. Create has no server-side stored app to merge. + const restoredValues = { + ...parsed.formValues, + credentials: withoutMintedTokenCredentials(parsed.formValues.credentials), + }; + setPendingRestoredValues({ values: restoredValues, transport: restoredTransport }); + } + if (typeof parsed.authorizedIdentity === "string") { + // Re-arm invalidation: without this the remounted form has authorizedIdentity=undefined, so a + // post-restore mode/url edit would never fire the stale-token discard. + setAuthorizedIdentity(parsed.authorizedIdentity); } if (parsed.costConfig) { setCostConfig(parsed.costConfig); @@ -380,6 +439,7 @@ const CreateMCPServer: React.FC = ({ available_on_public_internet: availableOnPublicInternetRaw, delegate_auth_to_upstream: delegateAuthToUpstreamRaw, oauth_passthrough: oauthPassthroughRaw, + dcr_bridge: dcrBridgeRaw, token_validation_json: rawTokenValidationJson, ...restValues } = values; @@ -486,6 +546,11 @@ const CreateMCPServer: React.FC = ({ available_on_public_internet: Boolean(availableOnPublicInternetRaw), delegate_auth_to_upstream: Boolean(delegateAuthToUpstreamRaw), oauth_passthrough: Boolean(oauthPassthroughRaw), + // ``dcr_bridge`` is only meaningful for the client-forwarded token + // modes (true_passthrough / oauth_delegate) and defaults on when the + // toggle is shown; force false for any other auth type so a stale + // ``true`` is never persisted. Mirrors the sibling flags above. + dcr_bridge: isClientForwardedTokenMode(restValues.auth_type) ? Boolean(dcrBridgeRaw ?? true) : false, ...(restValues.auth_type === AUTH_TYPE.OAUTH2 ? { oauth2_flow: @@ -500,8 +565,20 @@ const CreateMCPServer: React.FC = ({ const includeCredentials = restValues.auth_type && AUTH_TYPES_REQUIRING_CREDENTIALS.includes(restValues.auth_type); - if (includeCredentials && credentialsPayload && Object.keys(credentialsPayload).length > 0) { - payload.credentials = credentialsPayload; + // Client-forwarded rows persist ONLY the declared app; strip any token material that lingered in + // the form (e.g. from a prior oauth2 authorize on the same session) so it can never reach the row. + const submitCredentials = isClientForwardedTokenMode(restValues.auth_type) + ? preservedDeclaredAppCredentials(credentialsPayload) + : credentialsPayload; + + if (includeCredentials && submitCredentials && Object.keys(submitCredentials).length > 0) { + payload.credentials = submitCredentials; + } + + // An interactive (oauth2) create persists its DCR-minted client from the ref (kept out of the + // form store); reuse a re-authorize's registered client instead of re-registering. + if (restValues.auth_type === AUTH_TYPE.OAUTH2 && dcrClientRef.current) { + payload.credentials = { ...(payload.credentials ?? {}), ...dcrClientRef.current }; } if (accessToken != null) { @@ -576,6 +653,9 @@ const CreateMCPServer: React.FC = ({ setHasToolAllowlistInteraction(false); setAliasManuallyEdited(false); setLogoUrl(undefined); + setAuthorizedIdentity(undefined); + dcrClientRef.current = null; + setAppMayNotMatchUpstream(false); setModalVisible(false); }; @@ -655,6 +735,8 @@ const CreateMCPServer: React.FC = ({ clearTools(); resetOAuthFlow(); setAuthorizedIdentity(undefined); + dcrClientRef.current = null; + setAppMayNotMatchUpstream(false); } }, [isModalVisible, form, clearTools, resetOAuthFlow]); @@ -663,12 +745,31 @@ const CreateMCPServer: React.FC = ({ const handleFormValuesChange = (changedValues: Record, allValues: Record) => { // Any change to a mint-relevant field (url, auth_type, oauth_flow_type, client creds/scopes, or the // authorization/token/registration endpoints — see getOAuthAuthorizationIdentity) makes a held token - // stale, so discard it and force a fresh authorize. When that happens, formValues must be rebuilt - // from the form's post-reset state, not the pre-reset allValues snapshot: the snapshot still holds - // the discarded token in credentials, and useTestMCPConnection reads formValues for tool preview. - if (isHeldOAuthTokenStale(allValues, authorizedIdentity)) { + // stale, so discard it and force a fresh authorize. The stale check reads getFieldsValue(true): the + // onValuesChange allValues argument holds only MOUNTED paths, so an unmounted identity field (e.g. + // an oauth_flow_type initialValue while in a client-forwarded mode) would compare as changed on + // every keystroke and churn the held token. When a clear happens, formValues is rebuilt from the + // form's post-reset state (not the pre-reset snapshot, which still holds the discarded token). + // Editing the client fields is the admin managing/acknowledging the app, so it always dismisses + // the "may not match upstream" warning regardless of the stale-token branch below. + // Editing the client fields is the admin managing/acknowledging the app, so it dismisses the "may + // not match upstream" warning. Otherwise a url/endpoint change while a declared app is present keeps + // the app but flags that it may not match the new upstream (the "keep + warn" behavior). This is + // independent of the held-token stale check below so it fires even without an authorize this session. + if ("credentials" in changedValues) { + setAppMayNotMatchUpstream(false); + } else { + const upstreamChanged = ["url", "spec_path", "authorization_url", "token_url", "registration_url"].some( + (key) => key in changedValues, + ); + const hasDeclaredApp = preservedDeclaredAppCredentials(form.getFieldValue("credentials")) !== undefined; + if (upstreamChanged && hasDeclaredApp) { + setAppMayNotMatchUpstream(true); + } + } + if (isHeldOAuthTokenStale(form.getFieldsValue(true), authorizedIdentity)) { clearHeldOAuthToken(changedValues); - setFormValues({ ...form.getFieldsValue(true), ...changedValues }); + setFormValues(form.getFieldsValue(true)); return; } setFormValues(allValues); @@ -989,12 +1090,14 @@ const CreateMCPServer: React.FC = ({ {shouldShowAuthValueField && ( diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx index adb3e161da5..5b8d0aac8c0 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx @@ -1,7 +1,9 @@ import React from "react"; import { describe, it, expect, vi, beforeEach } from "vitest"; import { render, screen, waitFor, fireEvent, act } from "@testing-library/react"; -import MCPServerEdit from "./mcp_server_edit"; +import userEvent from "@testing-library/user-event"; +import MCPServerEdit, { EDIT_OAUTH_UI_STATE_KEY } from "./mcp_server_edit"; +import { setSecureItem } from "@/utils/secureStorage"; import * as networking from "../networking"; import NotificationsManager from "../molecules/notifications_manager"; import { selectAntOption } from "./testUtils"; @@ -1377,6 +1379,258 @@ describe("MCPServerEdit (OAuth token persistence on save)", () => { }, ); + it.each([["true_passthrough"], ["oauth_delegate"]])( + "persists admin-entered OAuth app credentials in the update payload for the %s mode", + async (authType) => { + mockOauth.tokenResponse = { access_token: "cf-tok", expires_in: 1800, token_type: "bearer" }; + vi.mocked(networking.updateMCPServer).mockResolvedValue({ + ...interactiveOAuthServer, + auth_type: authType, + }); + + render( + , + ); + + const user = userEvent.setup({ delay: null }); + await user.type( + screen.getByPlaceholderText("Leave blank to keep the currently saved app (if any)"), + "org-app-client-id", + ); + await user.type( + screen.getByPlaceholderText("Leave blank to keep the currently saved secret (if any)"), + "org-app-secret", + ); + + await act(async () => { + fireEvent.click(screen.getAllByRole("button", { name: "Save Changes" })[0]); + }); + + await waitFor(() => expect(networking.updateMCPServer).toHaveBeenCalledTimes(1)); + const [, payload] = vi.mocked(networking.updateMCPServer).mock.calls[0]; + + // The declared app is config and persists onto the row; the browser-held token still never + // reaches the payload or the per-user credential store. + expect(payload.credentials).toMatchObject({ + client_id: "org-app-client-id", + client_secret: "org-app-secret", + }); + expect(JSON.stringify(payload)).not.toContain("cf-tok"); + expect(networking.storeMCPOAuthUserCredential).not.toHaveBeenCalled(); + }, + ); + + it.each([["true_passthrough"], ["oauth_delegate"]])( + "preserves admin-entered app credentials when the URL changes after authorize for the %s mode", + async (authType) => { + vi.mocked(networking.updateMCPServer).mockResolvedValue({ + ...interactiveOAuthServer, + auth_type: authType, + }); + + render( + , + ); + + const user = userEvent.setup({ delay: null }); + await user.type( + screen.getByPlaceholderText("Leave blank to keep the currently saved app (if any)"), + "org-app-client-id", + ); + await user.type( + screen.getByPlaceholderText("Leave blank to keep the currently saved secret (if any)"), + "org-app-secret", + ); + + act(() => { + mockOauth.onTokenReceived?.({ access_token: "cf-tok", token_type: "bearer" }); + }); + + // The URL edit invalidates the held browser token (removeToken fires), but the declared app + // is config and must survive the invalidation into the update payload. + await act(async () => { + fireEvent.change(screen.getByPlaceholderText("https://your-mcp-server.com"), { + target: { value: "https://other.example.com/mcp" }, + }); + }); + expect(mockRemoveToken).toHaveBeenCalledWith("oauth_server_1", "user-1"); + + await act(async () => { + fireEvent.click(screen.getAllByRole("button", { name: "Save Changes" })[0]); + }); + + await waitFor(() => expect(networking.updateMCPServer).toHaveBeenCalledTimes(1)); + const [, payload] = vi.mocked(networking.updateMCPServer).mock.calls[0]; + expect(payload.url).toBe("https://other.example.com/mcp"); + expect(payload.credentials).toMatchObject({ + client_id: "org-app-client-id", + client_secret: "org-app-secret", + }); + expect(JSON.stringify(payload)).not.toContain("cf-tok"); + }, + ); + + it("sends an explicit-null credential write when removing the saved app for true_passthrough", async () => { + vi.mocked(networking.updateMCPServer).mockResolvedValue({ + ...interactiveOAuthServer, + auth_type: "true_passthrough", + }); + + render( + , + ); + + // Blank fields keep the stored app (the backend merges partial credential updates), so the + // edit form states that convention and removal is an explicit checkbox that saves nulls. + expect(screen.getByPlaceholderText("Leave blank to keep the currently saved app (if any)")).toBeInTheDocument(); + + fireEvent.click( + screen.getByRole("checkbox", { + name: /Remove the saved OAuth app on save/, + }), + ); + + await act(async () => { + fireEvent.click(screen.getAllByRole("button", { name: "Save Changes" })[0]); + }); + + await waitFor(() => expect(networking.updateMCPServer).toHaveBeenCalledTimes(1)); + const [, payload] = vi.mocked(networking.updateMCPServer).mock.calls[0]; + expect(payload.credentials).toEqual({ client_id: null, client_secret: null }); + }); + + it("warns that the saved app may not match after a URL change on a client-forwarded server", async () => { + render( + , + ); + + // No warning until the upstream changes. + expect(screen.queryByText(/registered for the previous upstream/)).not.toBeInTheDocument(); + + await act(async () => { + fireEvent.change(screen.getByPlaceholderText("https://your-mcp-server.com"), { + target: { value: "https://different.example.com/mcp" }, + }); + }); + + // Keep + warn parity with the create form: the stored app is kept, and the banner appears. + expect(screen.getByText(/registered for the previous upstream/)).toBeInTheDocument(); + }); + + it("preserves a stored client_id on OAuth-resume restore even when the saved snapshot is token-only", async () => { + // Post-redirect restore: the sessionStorage snapshot carries only a minted token (no client keys), + // while the loaded server has a stored client_id. The restore must merge the server's declared app + // under the snapshot before stripping tokens, so the stored client_id is never cleared to blank. + setSecureItem( + EDIT_OAUTH_UI_STATE_KEY, + JSON.stringify({ + serverId: "oauth_server_1", + formValues: { auth_type: "true_passthrough", credentials: { access_token: "leftover-token" } }, + }), + ); + + render( + , + ); + + const clientIdField = await screen.findByPlaceholderText("Leave blank to keep the currently saved app (if any)"); + await waitFor(() => expect((clientIdField as HTMLInputElement).value).toBe("stored-client")); + // The leftover minted token must not have rehydrated anywhere. + expect(document.body.innerHTML).not.toContain("leftover-token"); + }); + + it("resets the remove-app checkbox on a server switch so it never deletes the next server's stored app", async () => { + vi.mocked(networking.updateMCPServer).mockResolvedValue({ + ...interactiveOAuthServer, + auth_type: "true_passthrough", + }); + + const { rerender } = render( + , + ); + + // Check "remove saved app" on server A. + fireEvent.click(screen.getByRole("checkbox", { name: /Remove the saved OAuth app on save/ })); + expect( + (screen.getByRole("checkbox", { name: /Remove the saved OAuth app on save/ }) as HTMLInputElement).checked, + ).toBe(true); + + // Switch the panel to server B without unmounting. + rerender( + , + ); + + // The checkbox must have reset, so saving server B does not send the explicit-null delete write. + expect( + (screen.getByRole("checkbox", { name: /Remove the saved OAuth app on save/ }) as HTMLInputElement).checked, + ).toBe(false); + + await act(async () => { + fireEvent.click(screen.getAllByRole("button", { name: "Save Changes" })[0]); + }); + + await waitFor(() => expect(networking.updateMCPServer).toHaveBeenCalledTimes(1)); + const [, payload] = vi.mocked(networking.updateMCPServer).mock.calls[0]; + expect(payload.credentials).not.toEqual({ client_id: null, client_secret: null }); + }); + it("forwards a newly authorized browser-held token for tool loading before the form is saved", async () => { // Regression: fetchTools keyed the browser-held decision off the saved mcpServer.auth_type, so // after switching the form to true_passthrough and authorizing, the fresh token was not sent as @@ -1737,3 +1991,168 @@ describe("MCPServerEdit (max concurrent requests)", () => { expect(payload.max_concurrent_requests).toBeNull(); }); }); + +describe("MCPServerEdit (dcr_bridge toggle)", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockOauth.tokenResponse = null; + }); + + const getDcrToggle = () => document.getElementById("dcr_bridge"); + + function renderEdit(server: Record) { + render( + , + ); + } + + async function saveAndGetPayload() { + const saveButtons = screen.getAllByRole("button", { name: "Save Changes" }); + await act(async () => { + fireEvent.click(saveButtons[0]); + }); + await waitFor(() => { + expect(networking.updateMCPServer).toHaveBeenCalledTimes(1); + }); + const [, payload] = vi.mocked(networking.updateMCPServer).mock.calls[0]; + return payload; + } + + it.each([["true_passthrough"], ["oauth_delegate"]])("renders the toggle for a %s server", async (authType) => { + renderEdit({ auth_type: authType }); + + await waitFor(() => { + expect(getDcrToggle()).toBeInTheDocument(); + }); + expect(screen.getByText("Gateway-hosted sign-in (DCR bridge)")).toBeInTheDocument(); + }); + + it.each([["oauth2"], ["api_key"], ["none"]])("does not render the toggle for an %s server", async (authType) => { + renderEdit({ auth_type: authType }); + + await waitFor(() => { + expect(screen.getAllByRole("button", { name: "Save Changes" }).length).toBeGreaterThan(0); + }); + expect(screen.queryByText("Gateway-hosted sign-in (DCR bridge)")).not.toBeInTheDocument(); + expect(getDcrToggle()).not.toBeInTheDocument(); + }); + + it("renders the toggle between the OAuth client fields and the Authorize button", async () => { + renderEdit({ auth_type: "true_passthrough" }); + + await waitFor(() => { + expect(getDcrToggle()).toBeInTheDocument(); + }); + const toggle = getDcrToggle() as HTMLElement; + const secretInput = screen.getByPlaceholderText("Leave blank to keep the currently saved secret (if any)"); + const authorizeButton = screen.getByRole("button", { name: "Authorize & Fetch Tools (browser-only)" }); + expect(secretInput.compareDocumentPosition(toggle) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); + expect(toggle.compareDocumentPosition(authorizeButton) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); + }); + + it("initializes unchecked from a null stored value and saves an explicit false", async () => { + vi.mocked(networking.updateMCPServer).mockResolvedValue({ + ...interactiveOAuthServer, + auth_type: "true_passthrough", + }); + renderEdit({ auth_type: "true_passthrough", dcr_bridge: null }); + + await waitFor(() => { + expect(getDcrToggle()).toBeInTheDocument(); + }); + expect(getDcrToggle()).toHaveAttribute("aria-checked", "false"); + + const payload = await saveAndGetPayload(); + expect(payload.dcr_bridge).toBe(false); + }); + + it("initializes checked from a stored true and saves an explicit true", async () => { + vi.mocked(networking.updateMCPServer).mockResolvedValue({ + ...interactiveOAuthServer, + auth_type: "oauth_delegate", + dcr_bridge: true, + }); + renderEdit({ auth_type: "oauth_delegate", dcr_bridge: true }); + + await waitFor(() => { + expect(getDcrToggle()).toBeInTheDocument(); + }); + expect(getDcrToggle()).toHaveAttribute("aria-checked", "true"); + + const payload = await saveAndGetPayload(); + expect(payload.dcr_bridge).toBe(true); + }); + + it("saves an explicit false after the admin unchecks a stored true", async () => { + vi.mocked(networking.updateMCPServer).mockResolvedValue({ + ...interactiveOAuthServer, + auth_type: "true_passthrough", + dcr_bridge: false, + }); + renderEdit({ auth_type: "true_passthrough", dcr_bridge: true }); + + await waitFor(() => { + expect(getDcrToggle()).toBeInTheDocument(); + }); + await act(async () => { + fireEvent.click(getDcrToggle()!); + }); + expect(getDcrToggle()).toHaveAttribute("aria-checked", "false"); + + const payload = await saveAndGetPayload(); + expect(payload.dcr_bridge).toBe(false); + }); + + it("forces dcr_bridge: false when the auth type is switched away", async () => { + vi.mocked(networking.updateMCPServer).mockResolvedValue({ + ...interactiveOAuthServer, + auth_type: "api_key", + }); + renderEdit({ auth_type: "true_passthrough", dcr_bridge: true }); + + await waitFor(() => { + expect(getDcrToggle()).toBeInTheDocument(); + }); + + await selectAntOption("Authentication", "API Key"); + await waitFor(() => { + expect(getDcrToggle()).not.toBeInTheDocument(); + }); + + // Mirrors the sibling delegate_auth_to_upstream / oauth_passthrough force-false: a stale true is + // never left behind to silently re-activate if the mode is switched back. + const payload = await saveAndGetPayload(); + expect(payload.dcr_bridge).toBe(false); + }); + + it("preserves the toggle value when switching between the two client-forwarded modes", async () => { + vi.mocked(networking.updateMCPServer).mockResolvedValue({ + ...interactiveOAuthServer, + auth_type: "oauth_delegate", + dcr_bridge: true, + }); + renderEdit({ auth_type: "true_passthrough", dcr_bridge: true }); + + await waitFor(() => { + expect(getDcrToggle()).toBeInTheDocument(); + }); + expect(getDcrToggle()).toHaveAttribute("aria-checked", "true"); + + // The Form.Item stays mounted across the two client-forwarded modes, so the live toggle value is + // preserved rather than forced false by the switch. + await selectAntOption("Authentication", "OAuth Delegate (client-supplied upstream token)"); + await waitFor(() => { + expect(getDcrToggle()).toBeInTheDocument(); + }); + expect(getDcrToggle()).toHaveAttribute("aria-checked", "true"); + + const payload = await saveAndGetPayload(); + expect(payload.dcr_bridge).toBe(true); + }); +}); diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx index 7446c96c40e..6709eb02c65 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx @@ -8,6 +8,8 @@ import { getOAuthAuthorizationIdentity, CLEARED_ON_INVALIDATION, isHeldOAuthTokenStale, + preservedDeclaredAppCredentials, + withoutMintedTokenCredentials, OAUTH_FLOW, MCP_OAUTH2_FLOW_M2M, MCP_OAUTH2_FLOW_INTERACTIVE, @@ -56,6 +58,8 @@ const AUTH_TYPES_REQUIRING_CREDENTIALS = [ AUTH_TYPE.OAUTH2, AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE, AUTH_TYPE.AWS_SIGV4, + AUTH_TYPE.TRUE_PASSTHROUGH, + AUTH_TYPE.OAUTH_DELEGATE, ]; export const EDIT_OAUTH_UI_STATE_KEY = "litellm-mcp-oauth-edit-state"; @@ -74,6 +78,10 @@ const MCPServerEdit: React.FC = ({ const [toolsError, setToolsError] = useState(null); const [searchValue, setSearchValue] = useState(""); const [aliasManuallyEdited, setAliasManuallyEdited] = useState(false); + const [removeStoredApp, setRemoveStoredApp] = useState(false); + // Set when the upstream identity (url/endpoints) changed while a declared app is present, so the + // section warns that the saved app may not match the new upstream (the app is kept, not wiped). + const [appMayNotMatchUpstream, setAppMayNotMatchUpstream] = useState(false); const [allowedTools, setAllowedTools] = useState([]); const [hasToolAllowlistInteraction, setHasToolAllowlistInteraction] = useState(false); const [toolNameToDisplayName, setToolNameToDisplayName] = useState>({}); @@ -179,7 +187,9 @@ const MCPServerEdit: React.FC = ({ url, transport, auth_type: isClientForwardedTokenMode(values.auth_type) ? values.auth_type : AUTH_TYPE.OAUTH2, - credentials: values.credentials, + credentials: isClientForwardedTokenMode(values.auth_type) + ? preservedDeclaredAppCredentials(values.credentials) + : values.credentials, mcp_access_groups: values.mcp_access_groups || mcpServer.mcp_access_groups, static_headers: staticHeaders, command: values.command, @@ -202,19 +212,23 @@ const MCPServerEdit: React.FC = ({ }; setToken(mcpServer.server_id, browserHeldToken, userID); NotificationsManager.success( - "Token held for this browser session. Tools can now be loaded and configured; nothing was saved to LiteLLM.", + "Token held for this browser session. Tools can now be loaded and configured; the token is not saved to LiteLLM.", ); return; } - const credentials = { + const current = (form.getFieldValue("credentials") as Record | undefined) ?? {}; + const nextCredentials = { + ...(preservedDeclaredAppCredentials(current) ?? {}), + ...(current.scopes !== undefined && { scopes: current.scopes }), access_token: token.access_token, ...(token.refresh_token && { refresh_token: token.refresh_token }), ...(token.expires_in && { expires_in: token.expires_in }), ...(token.scope && { scope: token.scope }), }; - - form.setFieldsValue({ credentials }); + // Path-replace (not deep-merge) so a re-authorize with fewer token fields does not leave stale + // siblings behind; the admin-typed client keys and scopes are carried explicitly above. + form.setFieldValue("credentials", nextCredentials); // Re-capture after writing credentials so the token is not invalidated by its own credential write. authorizedIdentityRef.current = getOAuthAuthorizationIdentity(form.getFieldsValue(true)); @@ -276,6 +290,7 @@ const MCPServerEdit: React.FC = ({ env_vars: initialEnvVars, extra_headers: mcpServer.extra_headers || [], oauth_flow_type: oauth2FlowToFormValue(mcpServer.oauth2_flow), + dcr_bridge: Boolean(mcpServer.dcr_bridge), token_validation_json: mcpServer.token_validation ? JSON.stringify(mcpServer.token_validation, null, 2) : undefined, @@ -295,6 +310,11 @@ const MCPServerEdit: React.FC = ({ } syncedServerIdRef.current = mcpServer.server_id; form.setFieldsValue(initialValues); + // Reset per-server OAuth UI state so it never carries across a server switch without an unmount: a + // stale removeStoredApp would send an explicit-null credential write that deletes the new server's + // stored app, and a stale warning would show on a server whose upstream did not change. + setAppMayNotMatchUpstream(false); + setRemoveStoredApp(false); }, [mcpServer.server_id, initialValues, form]); // Initialize cost config from existing server data @@ -332,8 +352,24 @@ const MCPServerEdit: React.FC = ({ return; } if (parsed.formValues) { - setPendingRestoredValues({ ...mcpServer, ...parsed.formValues }); + // Rebuild credentials from the declared app in EITHER the loaded server or the saved snapshot, + // then strip minted token material. Merging the two (server under snapshot) before stripping is + // what guarantees a token-only snapshot never clears a stored client_id/client_secret: the + // server's declared app survives and only the token keys drop. Assigning the cleaned result (not + // spreading the raw snapshot) also ensures a stale token can never rehydrate into the form. + const restoredCredentials = withoutMintedTokenCredentials({ + ...(mcpServer.credentials ?? {}), + ...((parsed.formValues.credentials as Record | undefined) ?? {}), + }); + const restoredValues = { + ...mcpServer, + ...parsed.formValues, + credentials: restoredCredentials, + }; + setPendingRestoredValues(restoredValues); } + // The ref is re-armed by onTokenReceived when the redirect completes the code exchange, so there + // is no separate restore-side re-arm here (writing a ref inside an effect is disallowed). if (parsed.costConfig) { setCostConfig(parsed.costConfig); } @@ -407,7 +443,13 @@ const MCPServerEdit: React.FC = ({ } setTools([]); resetOAuthFlow(); + // The admin-typed app is upstream-scoped config, not minted material, so it survives every + // invalidation; only the held token is discarded. Token-shaped keys are excluded by the filter. + const keptAppCredentials = preservedDeclaredAppCredentials(form.getFieldValue("credentials")); form.resetFields([...CLEARED_ON_INVALIDATION]); + if (keptAppCredentials) { + form.setFieldsValue({ credentials: keptAppCredentials }); + } const preserved = Object.fromEntries( CLEARED_ON_INVALIDATION.filter((key) => key in changedValues).map((key) => [key, changedValues[key]]), ); @@ -417,6 +459,21 @@ const MCPServerEdit: React.FC = ({ }; const handleFormValuesChange = (changedValues: Record) => { + // Editing the client fields dismisses the "may not match upstream" warning; otherwise a url/endpoint + // change while a declared app is present keeps the app but flags that it may not match the new + // upstream (the "keep + warn" behavior). Mirrors the create form; independent of the held-token + // stale check so it fires even without an authorize this session (the stored app is for the old url). + if ("credentials" in changedValues) { + setAppMayNotMatchUpstream(false); + } else { + const upstreamChanged = ["url", "spec_path", "authorization_url", "token_url", "registration_url"].some( + (key) => key in changedValues, + ); + const hasDeclaredApp = preservedDeclaredAppCredentials(form.getFieldValue("credentials")) !== undefined; + if (upstreamChanged && hasDeclaredApp) { + setAppMayNotMatchUpstream(true); + } + } if (isHeldOAuthTokenStale(form.getFieldsValue(true), authorizedIdentityRef.current)) { clearHeldOAuthToken(changedValues); } @@ -627,6 +684,7 @@ const MCPServerEdit: React.FC = ({ available_on_public_internet: availableOnPublicInternetRaw, delegate_auth_to_upstream: delegateAuthToUpstreamRaw, oauth_passthrough: oauthPassthroughRaw, + dcr_bridge: dcrBridgeRaw, token_validation_json: rawTokenValidationJson, ...restValues } = values; @@ -837,6 +895,15 @@ const MCPServerEdit: React.FC = ({ ? Boolean(oauthPassthroughRaw ?? mcpServer.oauth_passthrough) : false; })(), + // ``dcr_bridge`` is only meaningful for the client-forwarded token + // modes (true_passthrough / oauth_delegate). The Form.Item is + // conditionally rendered so the value drops out of the form on + // auth_type change; force false for any other configuration to avoid + // persisting a stale ``true`` that would silently re-activate if the + // mode is later switched back. + dcr_bridge: isClientForwardedTokenMode(restValues.auth_type) + ? Boolean(dcrBridgeRaw ?? mcpServer.dcr_bridge) + : false, ...(restValues.auth_type === AUTH_TYPE.OAUTH2 && restValues.oauth_flow_type ? { oauth2_flow: @@ -850,8 +917,22 @@ const MCPServerEdit: React.FC = ({ const includeCredentials = restValues.auth_type && AUTH_TYPES_REQUIRING_CREDENTIALS.includes(restValues.auth_type); - if (includeCredentials && credentialsPayload && Object.keys(credentialsPayload).length > 0) { - payload.credentials = credentialsPayload; + // Client-forwarded rows persist ONLY the declared app; strip any token material lingering in the + // form (e.g. from a prior oauth2 authorize this session) so it can never reach the row. + const submitCredentials = isClientForwardedTokenMode(restValues.auth_type) + ? preservedDeclaredAppCredentials(credentialsPayload) + : credentialsPayload; + + if (includeCredentials && submitCredentials && Object.keys(submitCredentials).length > 0) { + payload.credentials = submitCredentials; + } + + // Explicit removal of a saved app for the client-forwarded modes, applied AFTER the filter so it + // always wins. Blank fields are the keep-existing convention (the backend merges partial + // credential updates), so removal must be an explicit-null write: encrypt skips nulls and the + // merge overrides the stored keys, returning the server to dynamic client registration. + if (removeStoredApp && isClientForwardedTokenMode(restValues.auth_type)) { + payload.credentials = { client_id: null, client_secret: null }; } const updated = await updateMCPServer(accessToken, payload); @@ -895,6 +976,7 @@ const MCPServerEdit: React.FC = ({ } NotificationsManager.success("MCP Server updated successfully"); + setAppMayNotMatchUpstream(false); onSuccess(updated); } catch (error: any) { NotificationsManager.fromBackend("Failed to update MCP Server" + (error?.message ? `: ${error.message}` : "")); @@ -1040,6 +1122,11 @@ const MCPServerEdit: React.FC = ({ error: oauthError, tokenResponse: oauthTokenResponse, }} + isEditing + savedAuthType={mcpServer.auth_type} + removeStoredApp={removeStoredApp} + onRemoveStoredAppChange={setRemoveStoredApp} + appMayNotMatchUpstream={appMayNotMatchUpstream} /> )} diff --git a/ui/litellm-dashboard/src/components/mcp_tools/types.test.tsx b/ui/litellm-dashboard/src/components/mcp_tools/types.test.tsx index c6faca1fb51..7ce803d583c 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/types.test.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/types.test.tsx @@ -10,6 +10,9 @@ import { getOAuthAuthorizationIdentity, isHeldOAuthTokenStale, oauth2FlowToFormValue, + preservedDeclaredAppCredentials, + withoutMintedTokenCredentials, + credentialAuthClass, } from "./types"; describe("getOAuthAuthorizationIdentity", () => { @@ -180,3 +183,51 @@ describe("oauth2FlowToFormValue", () => { expect(oauth2FlowToFormValue(undefined)).toBeUndefined(); }); }); + +describe("preservedDeclaredAppCredentials", () => { + it("keeps only non-empty string declared-app keys and never token-shaped keys", () => { + expect(preservedDeclaredAppCredentials(undefined)).toBeUndefined(); + expect(preservedDeclaredAppCredentials({})).toBeUndefined(); + expect(preservedDeclaredAppCredentials({ client_id: 123 })).toBeUndefined(); + expect(preservedDeclaredAppCredentials({ client_id: "" })).toBeUndefined(); + expect(preservedDeclaredAppCredentials({ client_id: "a", access_token: "t", scopes: ["s"] })).toEqual({ + client_id: "a", + }); + expect(preservedDeclaredAppCredentials({ client_secret: "s" })).toEqual({ client_secret: "s" }); + expect(preservedDeclaredAppCredentials({ client_id: "a", client_secret: "b", refresh_token: "r" })).toEqual({ + client_id: "a", + client_secret: "b", + }); + }); +}); + +describe("withoutMintedTokenCredentials", () => { + it("drops token keys and keeps the declared app and other config", () => { + expect(withoutMintedTokenCredentials(undefined)).toBeUndefined(); + const mixed = { + client_id: "a", + client_secret: "b", + access_token: "t", + refresh_token: "r", + expires_in: 3600, + scope: "read", + scopes: ["read"], + }; + expect(withoutMintedTokenCredentials(mixed)).toEqual({ client_id: "a", client_secret: "b", scopes: ["read"] }); + }); + + it("returns undefined (not {}) when only minted keys are present, so a restore never blanks the fields", () => { + expect(withoutMintedTokenCredentials({ access_token: "t", refresh_token: "r", expires_in: 3600 })).toBeUndefined(); + // A declared client is always kept, so a stored client_id can never be overwritten with empty. + expect(withoutMintedTokenCredentials({ client_id: "x", access_token: "t" })).toEqual({ client_id: "x" }); + }); +}); + +describe("credentialAuthClass", () => { + it("collapses the client-forwarded modes to one class and leaves others distinct", () => { + expect(credentialAuthClass(AUTH_TYPE.TRUE_PASSTHROUGH)).toBe("client_forwarded"); + expect(credentialAuthClass(AUTH_TYPE.OAUTH_DELEGATE)).toBe("client_forwarded"); + expect(credentialAuthClass(AUTH_TYPE.OAUTH2)).toBe(AUTH_TYPE.OAUTH2); + expect(credentialAuthClass(null)).toBeNull(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx index 3eba8b30968..04766aad7b4 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx @@ -96,6 +96,54 @@ export const getOAuthAuthorizationIdentity = (values: Record): // edit forms so what gets wiped cannot drift. export const CLEARED_ON_INVALIDATION = ["credentials"] as const; +// The declared-app filter over form.credentials. It is a pure key filter with no mode/transition +// guard because the surrounding code establishes that a client_id/client_secret in form.credentials +// is ALWAYS admin-typed in every reachable state: the create form holds the DCR-minted client in a +// ref and never writes it into the form store, the edit form's onTokenReceived never writes client +// keys, and the invalidation reset clears the whole object atomically. So preserving the string +// client keys across any invalidation (URL/endpoint edit, true_passthrough<->oauth_delegate switch, +// or a round trip through another mode) is always legitimate, while the output key filter excludes +// token-shaped keys so a preserve can never carry minted material through. Shared by both forms. +const DECLARED_APP_CREDENTIAL_KEYS = ["client_id", "client_secret"] as const; + +// Minted token material the oauth2 authorize path writes beside the app keys; stripped from restored +// snapshots and from any credentials that transit to the temp-session preview so a stale token never +// reaches the backend or a client-forwarded server row. +export const MINTED_TOKEN_CREDENTIAL_KEYS = ["access_token", "refresh_token", "expires_in", "scope"] as const; + +export const preservedDeclaredAppCredentials = ( + credentials: Record | null | undefined, +): Record | undefined => { + if (!credentials) return undefined; + const kept = Object.fromEntries( + DECLARED_APP_CREDENTIAL_KEYS.filter((key) => typeof credentials[key] === "string" && credentials[key] !== "").map( + (key) => [key, credentials[key] as string], + ), + ); + return Object.keys(kept).length > 0 ? kept : undefined; +}; + +// Drop minted token keys, keeping everything else (the declared app plus any non-token config). +export const withoutMintedTokenCredentials = ( + credentials: Record | null | undefined, +): Record | undefined => { + if (!credentials) return undefined; + const kept = Object.fromEntries( + Object.entries(credentials).filter(([key]) => !(MINTED_TOKEN_CREDENTIAL_KEYS as readonly string[]).includes(key)), + ); + // Return undefined (not {}) when only minted keys were present, so a restore spreads `credentials: + // undefined` (the fields keep their placeholder / keep-existing state) rather than blanking them. + return Object.keys(kept).length > 0 ? kept : undefined; +}; + +// The client-forwarded modes share one credential class (same declared app, same authorize relay), so +// a switch between them must NOT be treated as an app change. Mirrors the backend _credential_auth_class +// in db.py; kept in sync so the UI's keep-existing copy and the backend's merge cannot disagree. +export const credentialAuthClass = (authType: string | null | undefined): string | null => { + if (authType === AUTH_TYPE.TRUE_PASSTHROUGH || authType === AUTH_TYPE.OAUTH_DELEGATE) return "client_forwarded"; + return authType ?? null; +}; + // True when a token was authorized in this session (authorizedIdentity recorded at mint time) and the // form's current identity no longer matches it. Every invalidation decision in both forms goes through // this single check: onValuesChange for user edits, and an explicit recheck after any programmatic @@ -319,7 +367,10 @@ export interface MCPServer { available_on_public_internet?: boolean; delegate_auth_to_upstream?: boolean; oauth_passthrough?: boolean; + dcr_bridge?: boolean | null; max_concurrent_requests?: number | null; + /** Redacted to null in server responses; present when constructing a server locally. */ + credentials?: Record | null; /** Stdio-only fields (present when transport === 'stdio') */ command?: string | null; diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 781799286e9..da6bb079876 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -22,7 +22,8 @@ export const getCallbackConfigsCall = async (accessToken: string) => { * Helper file for calls being made to proxy */ import MessageManager from "@/components/molecules/message_manager"; -import { clearTokenCookies, storeLoginToken } from "@/utils/cookieUtils"; +import { clearTokenCookies, getCookie, storeLoginToken } from "@/utils/cookieUtils"; +import { decodeToken } from "@/utils/jwtUtils"; import { TagNewRequest, TagUpdateRequest, TagListResponse, TagInfoResponse } from "./tag_management/types"; import { Team } from "./key_team_helpers/key_list"; import { EmailEventSettingsResponse, EmailEventSettingsUpdateRequest } from "./email_events/types"; @@ -38,6 +39,12 @@ import type { import { MCP_TOOLS_PREVIEW_FORBIDDEN_MESSAGE } from "./mcp_tools/constants"; import { createApiClient, deriveErrorMessage } from "@/lib/http/client"; import { resolveApiBase } from "@/lib/http/resolveApiBase"; +import { + registerAuthHeaderNameGetter, + registerAuthTokenGetter, + registerBaseUrlGetter, + registerErrorHandler, +} from "@/lib/http/runtime"; import { serverRootPath, setServerRootPath } from "@/lib/serverRootPath"; export { serverRootPath }; @@ -371,6 +378,11 @@ const apiClient = createApiClient({ onError: handleError, }); +registerBaseUrlGetter(getProxyBaseUrl); +registerAuthHeaderNameGetter(getGlobalLitellmHeaderName); +registerAuthTokenGetter(() => decodeToken(getCookie("token"))?.key ?? null); +registerErrorHandler(handleError); + export const makeModelGroupPublic = async (accessToken: string, modelGroups: string[]) => { const url = proxyBaseUrl ? `${proxyBaseUrl}/model_group/make_public` : `/model_group/make_public`; const response = await fetch(url, { diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx index df42c156975..ef0d842ad3e 100644 --- a/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx @@ -29,6 +29,16 @@ const nameCellColumns: ColumnDef[] = [ }, ]; +const filterableColumns: ColumnDef[] = [ + { + accessorKey: "name", + header: "Name", + meta: { title: "Name" }, + filterFn: (row, columnId, value) => row.getValue(columnId) === value, + cell: ({ row }) => {row.original.name}, + }, +]; + const headerCycleColumns: ColumnDef[] = [ { accessorKey: "name", @@ -195,10 +205,110 @@ describe("DataTable pagination", () => { }); }); +describe("DataTable filtering", () => { + it("client mode filters rows by columnFilters", () => { + const { rerender } = render( + , + ); + expect(names()).toEqual(["Charlie", "Alice", "Bob"]); + + rerender( + , + ); + expect(names()).toEqual(["Alice"]); + }); + + it("client global filter matches substrings across columns", () => { + const { rerender } = render( + , + ); + expect(names()).toEqual(["Charlie", "Alice", "Bob"]); + + rerender( + , + ); + expect(names()).toEqual(["Alice"]); + }); + + it("server mode never filters locally even when columnFilters is set", () => { + render( + , + ); + expect(names()).toEqual(["Charlie", "Alice", "Bob"]); + }); + + it("throws when server filtering is missing required props", () => { + const spy = vi.spyOn(console, "error").mockImplementation(() => {}); + expect(() => render()).toThrow( + /filterMode='server'/, + ); + spy.mockRestore(); + }); +}); + +describe("DataTable loading", () => { + it("renders skeleton rows while loading and real rows once loaded", () => { + const { rerender } = render(); + expect(screen.getAllByTestId("skeleton-row").length).toBeGreaterThan(0); + expect(screen.queryByTestId("name-cell")).toBeNull(); + + rerender(); + expect(screen.queryAllByTestId("skeleton-row")).toHaveLength(0); + expect(names()).toEqual(["Charlie", "Alice", "Bob"]); + }); + + it("varies skeleton shape and width per column instead of one fixed bar", () => { + const columns: ColumnDef[] = [ + { accessorKey: "name", header: "Name", meta: { skeleton: "twoLine" }, cell: () => null }, + { accessorKey: "email", header: "Email", cell: () => null }, + ]; + render(); + + const firstRow = screen.getAllByTestId("skeleton-row").at(0); + expect(firstRow).toBeDefined(); + const bars = Array.from(firstRow?.querySelectorAll('[data-slot="skeleton"]') ?? []); + + // twoLine column contributes a main + sub bar (2); the text column contributes 1 + expect(bars).toHaveLength(3); + // per-column widths differ instead of every cell sharing one fixed width + expect(new Set(bars.map((bar) => bar.className)).size).toBeGreaterThan(1); + }); +}); + describe("DataTable column visibility", () => { it("hides a column when toggled off in the view-options menu", async () => { const user = userEvent.setup(); - render( + const { container } = render( { />, ); - expect(screen.getByText("Email")).toBeInTheDocument(); + expect(container.querySelector('th[data-header-id="email"]')).not.toBeNull(); await user.click(screen.getByTestId("view-options-trigger")); await user.click(await screen.findByTestId("view-option-email")); - await waitFor(() => expect(screen.queryByText("Email")).not.toBeInTheDocument()); + await waitFor(() => expect(container.querySelector('th[data-header-id="email"]')).toBeNull()); await user.click(screen.getByTestId("view-option-email")); - await waitFor(() => expect(screen.getByText("Email")).toBeInTheDocument()); + await waitFor(() => expect(container.querySelector('th[data-header-id="email"]')).not.toBeNull()); }); it("omits columns that opt out of hiding from the menu", async () => { diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx index 2e95ee170fa..758ca5a597b 100644 --- a/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx @@ -4,12 +4,14 @@ import { type Cell, type Column, type ColumnDef, + type ColumnFiltersState, type ColumnPinningState, type ColumnSizingState, type ExpandedState, flexRender, getCoreRowModel, getExpandedRowModel, + getFilteredRowModel, getPaginationRowModel, getSortedRowModel, type Header, @@ -21,9 +23,11 @@ import { useReactTable, type VisibilityState, } from "@tanstack/react-table"; +import { SearchX } from "lucide-react"; import * as React from "react"; import { Fragment, useState } from "react"; +import { Skeleton } from "@/components/ui/skeleton"; import { Table as TableRoot, TableBody, @@ -37,7 +41,7 @@ import { cn } from "@/lib/cva.config"; import "./columnMeta"; import { DataTablePagination, DEFAULT_PAGE_SIZE_OPTIONS } from "./DataTablePagination"; -import type { ColumnPinnedSide, DataTableProps, DataTableSize, PaginationMode, SortingMode } from "./types"; +import type { ColumnPinnedSide, DataTableProps, DataTableSize, FilterMode, PaginationMode, SortingMode } from "./types"; const INTERACTIVE_SELECTOR = "button, a, input, select, textarea, [role=checkbox], [data-row-click-exempt]"; @@ -60,14 +64,22 @@ export function validateDataTableConfig( props.pagination === undefined || props.onPaginationChange === undefined || props.rowCount === undefined; const serverPaginationIncomplete = props.paginationMode === "server" && serverPaginationPropsMissing; + const serverFilteringIncomplete = + props.filterMode === "server" && (props.columnFilters === undefined || props.onColumnFiltersChange === undefined); + const bothSortingSources = props.defaultSorting !== undefined && props.sorting !== undefined; + const bothFilterSources = props.defaultColumnFilters !== undefined && props.columnFilters !== undefined; return [ serverSortingIncomplete ? "sortingMode='server' requires both `sorting` and `onSortingChange`." : null, serverPaginationIncomplete ? "paginationMode='server' requires `pagination`, `onPaginationChange`, and `rowCount`." : null, + serverFilteringIncomplete ? "filterMode='server' requires both `columnFilters` and `onColumnFiltersChange`." : null, bothSortingSources ? "Provide either `defaultSorting` (uncontrolled) or `sorting` (controlled), not both." : null, + bothFilterSources + ? "Provide either `defaultColumnFilters` (uncontrolled) or `columnFilters` (controlled), not both." + : null, ].filter((message): message is string => message !== null); } @@ -93,9 +105,11 @@ function derivePinning(columns: ColumnDef[]): Colu function buildRowModels( sortingMode: SortingMode, paginationMode: PaginationMode, + filterMode: FilterMode, getRowCanExpand: ((row: Row) => boolean) | undefined, ): Partial> { return { + ...(filterMode === "client" ? { getFilteredRowModel: getFilteredRowModel() } : {}), ...(sortingMode === "client" ? { getSortedRowModel: getSortedRowModel() } : {}), ...(paginationMode === "client" ? { getPaginationRowModel: getPaginationRowModel() } : {}), ...(getRowCanExpand !== undefined ? { getRowCanExpand, getExpandedRowModel: getExpandedRowModel() } : {}), @@ -307,6 +321,65 @@ function MessageRow({ colSpan, children }: { colSpan: number; children: React.Re ); } +function DefaultEmptyState() { + return ( +
+
+ +
+
No results
+
No rows match your search or filters.
+
+ ); +} + +const SKELETON_WIDTHS = ["w-[58%]", "w-[44%]", "w-[70%]", "w-[50%]", "w-[64%]", "w-[48%]"] as const; + +function SkeletonCell({ column, index }: { column: Column | undefined; index: number }) { + const meta = column?.columnDef.meta; + const width = SKELETON_WIDTHS[index % SKELETON_WIDTHS.length]; + if (meta?.skeleton === "twoLine") { + return ( +
+ + +
+ ); + } + return ; +} + +function SkeletonRows({ + rowCount, + columns, + size, + message, +}: { + rowCount: number; + columns: readonly Column[]; + size: DataTableSize; + message?: string; +}) { + const rowKeys = Array.from({ length: Math.max(rowCount, 1) }, (_, index) => index); + const cells = columns.length > 0 ? columns : [undefined]; + return ( + + {rowKeys.map((rowKey) => ( + + {cells.map((column, columnKey) => ( + + + {rowKey === 0 && columnKey === 0 && message !== undefined ? ( + {message} + ) : null} + + ))} + + ))} + + ); +} + function useControllable( controlled: T | undefined, controlledOnChange: OnChangeFn | undefined, @@ -334,6 +407,12 @@ function useDataTableInstance(props: DataTablePro onPaginationChange, rowCount, pageSizeOptions = DEFAULT_PAGE_SIZE_OPTIONS, + filterMode = "none", + columnFilters, + onColumnFiltersChange, + defaultColumnFilters, + globalFilter, + onGlobalFilterChange, enableColumnResizing = false, columnResizeMode = "onEnd", defaultColumnVisibility, @@ -348,6 +427,12 @@ function useDataTableInstance(props: DataTablePro pageIndex: 0, pageSize: pageSizeOptions[0] ?? 25, }); + const filterState = useControllable( + columnFilters, + onColumnFiltersChange, + defaultColumnFilters ?? [], + ); + const globalFilterState = useControllable(globalFilter, onGlobalFilterChange, ""); const expandedState = useControllable(expanded, onExpandedChange, {}); const [columnVisibility, setColumnVisibility] = useState(defaultColumnVisibility ?? {}); const [columnSizing, setColumnSizing] = useState({}); @@ -360,6 +445,8 @@ function useDataTableInstance(props: DataTablePro state: { sorting: sortingState.value, pagination: paginationState.value, + columnFilters: filterState.value, + globalFilter: globalFilterState.value, expanded: expandedState.value, columnVisibility, columnSizing, @@ -367,16 +454,19 @@ function useDataTableInstance(props: DataTablePro initialState: { columnPinning }, manualSorting: sortingMode === "server", manualPagination: paginationMode === "server", + manualFiltering: filterMode === "server", enableSortingRemoval, enableColumnResizing, columnResizeMode, onSortingChange: sortingState.onChange, onPaginationChange: paginationState.onChange, + onColumnFiltersChange: filterState.onChange, + onGlobalFilterChange: globalFilterState.onChange, onExpandedChange: expandedState.onChange, onColumnVisibilityChange: setColumnVisibility, onColumnSizingChange: setColumnSizing, getCoreRowModel: getCoreRowModel(), - ...buildRowModels(sortingMode, paginationMode, expansionGuard), + ...buildRowModels(sortingMode, paginationMode, filterMode, expansionGuard), ...(getRowId !== undefined ? { getRowId } : {}), ...(paginationMode === "server" && rowCount !== undefined ? { rowCount } : {}), }; @@ -397,7 +487,8 @@ export function DataTable(props: DataTableProps(props: DataTableProps { if (isLoading) { - return {loadingMessage}; + return ( + + ); } if (rows.length === 0) { - return {noDataMessage}; + return {noDataMessage ?? }; } return rows.map((row) => ( (props: DataTableProps - {toolbar !== undefined &&
{toolbar(table)}
} -
- - - {table.getHeaderGroups().map((headerGroup) => ( - - {headerGroup.headers.map((header) => ( - - ))} - - ))} - - {renderBody()} - {footer !== undefined && {footer(table)}} - +
+ {toolbar !== undefined &&
{toolbar(table)}
} +
+ + + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => ( + + ))} + + ))} + + {renderBody()} + {footer !== undefined && {footer(table)}} + +
+ {paginationNode !== null &&
{paginationNode}
}
- {renderPagination()}
); } diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTableFilterDrawer.test.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTableFilterDrawer.test.tsx new file mode 100644 index 00000000000..0770c17cba6 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTableFilterDrawer.test.tsx @@ -0,0 +1,98 @@ +import type { ColumnDef, ColumnFiltersState } from "@tanstack/react-table"; +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { useState } from "react"; +import { describe, expect, it } from "vitest"; + +import { DataTable } from "./DataTable"; +import { DataTableFilterDrawer } from "./DataTableFilterDrawer"; +import { DataTableToolbar } from "./DataTableToolbar"; + +interface Person { + id: string; + name: string; +} + +const DATA: Person[] = [ + { id: "a", name: "Alice" }, + { id: "b", name: "Bob" }, + { id: "c", name: "Carol" }, +]; + +const columns: ColumnDef[] = [ + { + accessorKey: "name", + header: "Name", + meta: { title: "Name" }, + filterFn: (row, columnId, value) => row.getValue(columnId) === value, + cell: ({ row }) => {row.original.name}, + }, +]; + +const names = (): (string | null)[] => screen.getAllByTestId("name-cell").map((el) => el.textContent); + +function Harness({ initialFilters }: { initialFilters?: ColumnFiltersState }) { + const [open, setOpen] = useState(false); + return ( + ( + <> + setOpen(true)} /> + + {({ get, set }) => ( + set("name", event.target.value)} + /> + )} + + + )} + /> + ); +} + +describe("DataTableFilterDrawer", () => { + it("stages edits and only commits them to the table on Apply", async () => { + const user = userEvent.setup(); + render(); + expect(names()).toEqual(["Alice", "Bob", "Carol"]); + + await user.click(screen.getByTestId("datatable-filters-trigger")); + await user.type(await screen.findByTestId("draft-name"), "Bob"); + + expect(names()).toEqual(["Alice", "Bob", "Carol"]); + expect(screen.queryByTestId("filter-chip-name")).toBeNull(); + + await user.click(screen.getByTestId("filter-drawer-apply")); + expect(names()).toEqual(["Bob"]); + expect(screen.getByTestId("filter-chip-name")).toHaveTextContent("Bob"); + }); + + it("seeds the draft from committed filters when opened", async () => { + const user = userEvent.setup(); + render(); + expect(names()).toEqual(["Bob"]); + + await user.click(screen.getByTestId("datatable-filters-trigger")); + expect(await screen.findByTestId("draft-name")).toHaveValue("Bob"); + }); + + it("reset clears the committed filters and the draft", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByTestId("datatable-filters-trigger")); + await user.click(await screen.findByTestId("filter-drawer-reset")); + + expect(names()).toEqual(["Alice", "Bob", "Carol"]); + expect(screen.queryByTestId("filter-chip-name")).toBeNull(); + expect(screen.getByTestId("draft-name")).toHaveValue(""); + }); +}); diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTableFilterDrawer.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTableFilterDrawer.tsx new file mode 100644 index 00000000000..8aaec3f13a0 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTableFilterDrawer.tsx @@ -0,0 +1,108 @@ +"use client"; + +import type { ColumnFiltersState, Table } from "@tanstack/react-table"; +import * as React from "react"; + +import { Button } from "@/components/ui/button"; +import { Label } from "@/components/ui/label"; +import { Sheet, SheetContent, SheetDescription, SheetFooter, SheetHeader, SheetTitle } from "@/components/ui/sheet"; + +export interface FilterDraft { + get: (columnId: string) => unknown; + set: (columnId: string, value: unknown) => void; +} + +interface DataTableFilterDrawerProps { + table: Table; + open: boolean; + onOpenChange: (open: boolean) => void; + title?: string; + description?: React.ReactNode; + applyLabel?: string; + resetLabel?: string; + children: (draft: FilterDraft) => React.ReactNode; +} + +function isEmpty(value: unknown): boolean { + if (Array.isArray(value)) { + return value.length === 0; + } + return value === undefined || value === null || value === ""; +} + +function toDraft(filters: ColumnFiltersState): Record { + return Object.fromEntries(filters.map((filter) => [filter.id, filter.value])); +} + +function toFilters(draft: Record): ColumnFiltersState { + return Object.entries(draft) + .filter(([, value]) => !isEmpty(value)) + .map(([id, value]) => ({ id, value })); +} + +export function DataTableFilterDrawer({ + table, + open, + onOpenChange, + title = "Filters", + description, + applyLabel = "Apply Filters", + resetLabel = "Reset", + children, +}: DataTableFilterDrawerProps) { + const [draft, setDraft] = React.useState>(() => toDraft(table.getState().columnFilters)); + const [wasOpen, setWasOpen] = React.useState(open); + + if (open !== wasOpen) { + setWasOpen(open); + if (open) { + setDraft(toDraft(table.getState().columnFilters)); + } + } + + const helpers: FilterDraft = { + get: (columnId) => draft[columnId], + set: (columnId, value) => setDraft((previous) => ({ ...previous, [columnId]: value })), + }; + + const apply = () => { + table.setColumnFilters(toFilters(draft)); + onOpenChange(false); + }; + + const reset = () => { + setDraft({}); + table.setColumnFilters([]); + }; + + return ( + + + + {title} + {description !== undefined && {description}} + +
+ {children(helpers)} +
+ + + + + + + ); +} + +export function DataTableFilterField({ label, children }: { label: string; children: React.ReactNode }) { + return ( +
+ + {children} +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTablePagination.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTablePagination.tsx index 5a30b12f27f..7802465ef74 100644 --- a/ui/litellm-dashboard/src/components/shared/DataTable/DataTablePagination.tsx +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTablePagination.tsx @@ -37,7 +37,7 @@ export function DataTablePagination({ const lastPage = Math.max(pageCount - 1, 0); return ( -
+
Rows per page onSearchChange(event.target.value)} + placeholder={searchPlaceholder} + className="h-8 w-56 pl-8" + data-testid="datatable-search" + /> +
)} - {onToggleFilters !== undefined && ( - + {filters.map((filter) => ( + + {labelFor(filter.id)}: + {valueFor(filter.id, filter.value)} + + + ))} + {filters.length > 0 && ( + + )} +
+
+ {children} + {onRefresh !== undefined && ( + + )} + {showViewOptions && } + {onOpenFilters !== undefined && ( + )} - {showReset && }
- {children !== undefined &&
{children}
}
); } diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTableViewOptions.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTableViewOptions.tsx index ab56aafe7b5..f462481ef9c 100644 --- a/ui/litellm-dashboard/src/components/shared/DataTable/DataTableViewOptions.tsx +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTableViewOptions.tsx @@ -2,7 +2,7 @@ import { Menu } from "@base-ui/react/menu"; import type { Table } from "@tanstack/react-table"; -import { Check, SlidersHorizontal } from "lucide-react"; +import { Check, Columns3 } from "lucide-react"; import { Button } from "@/components/ui/button"; @@ -24,7 +24,7 @@ export function DataTableViewOptions({ table, label = "View", className } - + {label} } @@ -44,7 +44,8 @@ export function DataTableViewOptions({ table, label = "View", className } - {column.columnDef.meta?.title ?? column.id} + {column.columnDef.meta?.title ?? + (typeof column.columnDef.header === "string" ? column.columnDef.header : column.id)} ))} diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/columnMeta.ts b/ui/litellm-dashboard/src/components/shared/DataTable/columnMeta.ts index 46e72226038..0f14c277c6f 100644 --- a/ui/litellm-dashboard/src/components/shared/DataTable/columnMeta.ts +++ b/ui/litellm-dashboard/src/components/shared/DataTable/columnMeta.ts @@ -1,6 +1,6 @@ import type { RowData } from "@tanstack/react-table"; -import type { ColumnPinnedSide } from "./types"; +import type { ColumnPinnedSide, DataTableSkeletonShape } from "./types"; declare module "@tanstack/react-table" { interface ColumnMeta { @@ -9,5 +9,6 @@ declare module "@tanstack/react-table" { headerClassName?: string; title?: string; pinned?: ColumnPinnedSide; + skeleton?: DataTableSkeletonShape; } } diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/index.ts b/ui/litellm-dashboard/src/components/shared/DataTable/index.ts index 49a4430bbee..c4218f6051a 100644 --- a/ui/litellm-dashboard/src/components/shared/DataTable/index.ts +++ b/ui/litellm-dashboard/src/components/shared/DataTable/index.ts @@ -1,6 +1,7 @@ import "./columnMeta"; export { DataTable, DataTableConfigError, validateDataTableConfig } from "./DataTable"; +export { DataTableFilterDrawer, DataTableFilterField, type FilterDraft } from "./DataTableFilterDrawer"; export { DataTablePagination, DEFAULT_PAGE_SIZE_OPTIONS } from "./DataTablePagination"; export { DataTableToolbar } from "./DataTableToolbar"; export { DataTableViewOptions } from "./DataTableViewOptions"; @@ -11,6 +12,7 @@ export type { ColumnResizeMode, DataTableProps, DataTableSize, + FilterMode, PaginationMode, SortingMode, } from "./types"; diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/types.ts b/ui/litellm-dashboard/src/components/shared/DataTable/types.ts index 8fa6f21c4d3..f5130b4c823 100644 --- a/ui/litellm-dashboard/src/components/shared/DataTable/types.ts +++ b/ui/litellm-dashboard/src/components/shared/DataTable/types.ts @@ -1,5 +1,6 @@ import type { ColumnDef, + ColumnFiltersState, ExpandedState, OnChangeFn, PaginationState, @@ -13,9 +14,11 @@ import type * as React from "react"; export type SortingMode = "none" | "client" | "server"; export type PaginationMode = "none" | "client" | "server"; +export type FilterMode = "none" | "client" | "server"; export type ColumnResizeMode = "onEnd" | "onChange"; export type DataTableSize = "compact" | "default"; export type ColumnPinnedSide = "left" | "right"; +export type DataTableSkeletonShape = "text" | "twoLine"; export interface DataTableProps { data: TData[]; @@ -24,6 +27,7 @@ export interface DataTableProps { isLoading?: boolean; loadingMessage?: string; + skeletonRowCount?: number; noDataMessage?: React.ReactNode; sortingMode?: SortingMode; @@ -38,6 +42,14 @@ export interface DataTableProps { rowCount?: number; pageSizeOptions?: number[]; + filterMode?: FilterMode; + columnFilters?: ColumnFiltersState; + onColumnFiltersChange?: OnChangeFn; + defaultColumnFilters?: ColumnFiltersState; + + globalFilter?: string; + onGlobalFilterChange?: OnChangeFn; + enableColumnResizing?: boolean; columnResizeMode?: ColumnResizeMode; defaultColumnVisibility?: VisibilityState; diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx index 4ccfb891417..f0e27607ffb 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx @@ -583,7 +583,7 @@ describe("TeamInfoView", () => { await waitFor(() => { expect(screen.getByRole("button", { name: "Filters" })).toBeInTheDocument(); }); - expect(screen.getByRole("button", { name: "Reset Filters" })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Columns" })).toBeInTheDocument(); expect(screen.getByTestId("pagination-range")).toHaveTextContent("Showing 1-1 of 1"); expect(screen.getByTestId("pagination-prev")).toBeInTheDocument(); expect(screen.getByTestId("pagination-next")).toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx index fd81aaaad99..1c9b14d0c9d 100644 --- a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx @@ -4,8 +4,6 @@ import { beforeEach, describe, expect, it, vi, MockedFunction } from "vitest"; import { renderWithProviders } from "../../../tests/test-utils"; import { TeamVirtualKeysTable } from "./TeamVirtualKeysTable"; import { KeysResponse, useKeys } from "@/app/(dashboard)/hooks/keys/useKeys"; -import { fetchTeamFilterOptions } from "../key_team_helpers/filter_helpers"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { KeyResponse } from "../key_team_helpers/key_list"; import { Organization } from "../networking"; @@ -13,18 +11,6 @@ vi.mock("@/app/(dashboard)/hooks/keys/useKeys", () => ({ useKeys: vi.fn(), })); -vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ - default: vi.fn(), -})); - -vi.mock("../key_team_helpers/filter_helpers", () => ({ - fetchTeamFilterOptions: vi.fn().mockResolvedValue({ - keyAliases: [], - organizationIds: [], - userIds: [], - }), -})); - vi.mock("../key_team_helpers/fetch_available_models_team_key", () => ({ getModelDisplayName: vi.fn((model: string) => model), })); @@ -38,8 +24,12 @@ vi.mock("../templates/key_info_view", () => ({ )), })); +// Resolve the debounced search synchronously so typed input lands in the useKeys query within the test tick. +vi.mock("@tanstack/react-pacer/debouncer", () => ({ + useDebouncedValue: (value: unknown) => [value, { cancel: vi.fn(), flush: vi.fn() }], +})); + const mockUseKeys = useKeys as MockedFunction; -const mockUseAuthorized = useAuthorized as MockedFunction; const createMockKey = (overrides: Partial = {}): KeyResponse => ({ @@ -85,7 +75,6 @@ describe("TeamVirtualKeysTable", () => { beforeEach(() => { vi.clearAllMocks(); - mockUseAuthorized.mockReturnValue({ accessToken: "test-token" } as any); mockUseKeys.mockReturnValue({ data: { keys: [], total_count: 0, current_page: 1, total_pages: 1 } as KeysResponse, isPending: false, @@ -262,30 +251,48 @@ describe("TeamVirtualKeysTable", () => { await waitFor(() => expect(mockUseKeys).toHaveBeenLastCalledWith(1, 50, expect.anything())); }); - it("resets the sort order to the default when filters are reset", async () => { + it("maps the User ID drawer filter to a server-side useKeys query and clears it", async () => { const user = userEvent.setup(); - const result = { + mockUseKeys.mockReturnValue({ data: { keys: [createMockKey()], total_count: 1, current_page: 1, total_pages: 1 }, isPending: false, isFetching: false, refetch: vi.fn(), - } as unknown as ReturnType; - mockUseKeys.mockReturnValue(result); + } as unknown as ReturnType); renderWithProviders(); - await user.click(await screen.findByTestId("sort-header-created_at")); + await user.click(await screen.findByTestId("datatable-filters-trigger")); + const drawerBody = await screen.findByTestId("filter-drawer-body"); + const userInput = drawerBody.querySelector("input") as HTMLElement; + await user.type(userInput, "user-42"); + await user.click(screen.getByTestId("filter-drawer-apply")); + await waitFor(() => - expect(mockUseKeys).toHaveBeenLastCalledWith(1, 50, expect.objectContaining({ sortOrder: "asc" })), + expect(mockUseKeys).toHaveBeenLastCalledWith(1, 50, expect.objectContaining({ userID: "user-42" })), ); - await user.click(screen.getByRole("button", { name: "Reset Filters" })); + await user.click(screen.getByTestId("datatable-clear-filters")); await waitFor(() => - expect(mockUseKeys).toHaveBeenLastCalledWith( - 1, - 50, - expect.objectContaining({ sortBy: "created_at", sortOrder: "desc" }), - ), + expect(mockUseKeys).toHaveBeenLastCalledWith(1, 50, expect.objectContaining({ userID: undefined })), + ); + }); + + it("maps the search box to a server-side key-alias query", async () => { + const user = userEvent.setup(); + mockUseKeys.mockReturnValue({ + data: { keys: [createMockKey()], total_count: 1, current_page: 1, total_pages: 1 }, + isPending: false, + isFetching: false, + refetch: vi.fn(), + } as unknown as ReturnType); + + renderWithProviders(); + + await user.type(await screen.findByTestId("datatable-search"), "check-002"); + + await waitFor(() => + expect(mockUseKeys).toHaveBeenLastCalledWith(1, 50, expect.objectContaining({ selectedKeyAlias: "check-002" })), ); }); @@ -304,7 +311,7 @@ describe("TeamVirtualKeysTable", () => { }); }); - it("should show No keys found when keys array is empty", async () => { + it("should show the empty state when keys array is empty", async () => { mockUseKeys.mockReturnValue({ data: { keys: [], total_count: 0, current_page: 1, total_pages: 1 } as KeysResponse, isPending: false, @@ -315,26 +322,7 @@ describe("TeamVirtualKeysTable", () => { renderWithProviders(); await waitFor(() => { - expect(screen.getByText("No keys found")).toBeInTheDocument(); - }); - }); - - it("should fetch team-scoped filter options for Key Alias, Organization ID, and User ID", async () => { - const mockFetchTeamFilterOptions = vi.mocked(fetchTeamFilterOptions); - mockFetchTeamFilterOptions.mockResolvedValue({ - keyAliases: ["alice_key_team1", "charlie_key_team1"], - organizationIds: ["org-123"], - userIds: [ - { id: "user-1", email: "alice@example.com" }, - { id: "user-2", email: "charlie@example.com" }, - ], - }); - - // Use unique teamId to avoid cache hit from previous tests (refetchOnMount: false) - renderWithProviders(); - - await waitFor(() => { - expect(mockFetchTeamFilterOptions).toHaveBeenCalledWith("test-token", "team-filter-options-test"); + expect(screen.getByText("No rows match your search or filters.")).toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx index 73f524e13f8..207e6f2ccfe 100644 --- a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx @@ -1,21 +1,25 @@ "use client"; import { useKeys } from "@/app/(dashboard)/hooks/keys/useKeys"; import { DateCell, IdCell, MoneyCell } from "@/components/shared/table_cells"; -import { DataTable, DataTablePagination, DataTableSortHeader } from "@/components/shared/DataTable"; +import { + DataTable, + DataTableFilterDrawer, + DataTableFilterField, + DataTableSortHeader, + DataTableToolbar, +} from "@/components/shared/DataTable"; +import { Input } from "@/components/ui/input"; import { ChevronDownIcon, ChevronRightIcon } from "@heroicons/react/outline"; -import { ColumnDef, PaginationState, SortingState } from "@tanstack/react-table"; +import { useDebouncedValue } from "@tanstack/react-pacer/debouncer"; +import { ColumnDef, ColumnFiltersState, OnChangeFn, PaginationState, SortingState } from "@tanstack/react-table"; import { Badge, Icon, Text } from "@tremor/react"; import { Popover, Tooltip, Typography } from "antd"; import DefaultProxyAdminTag from "../common_components/DefaultProxyAdminTag"; import React, { useCallback, useEffect, useMemo, useState } from "react"; import { getModelDisplayName } from "../key_team_helpers/fetch_available_models_team_key"; import { KeyResponse, Team } from "../key_team_helpers/key_list"; -import FilterComponent, { FilterOption } from "../molecules/filter"; import { Organization } from "../networking"; import KeyInfoView from "../templates/key_info_view"; -import { useQuery } from "@tanstack/react-query"; -import { fetchTeamFilterOptions } from "../key_team_helpers/filter_helpers"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; interface TeamVirtualKeysTableProps { teamId: string; @@ -30,18 +34,29 @@ interface TeamVirtualKeysTableProps { const DEFAULT_SORTING: SortingState = [{ id: "created_at", desc: true }]; export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVirtualKeysTableProps) { - const { accessToken } = useAuthorized(); const [selectedKey, setSelectedKey] = useState(null); const [sorting, setSorting] = useState(DEFAULT_SORTING); const [tablePagination, setTablePagination] = useState({ pageIndex: 0, pageSize: 50, }); - const [filters, setFilters] = useState>({ - "Organization ID": "", - "Key Alias": "", - "User ID": "", - }); + const [columnFilters, setColumnFilters] = useState([]); + const [filtersOpen, setFiltersOpen] = useState(false); + const [searchInput, setSearchInput] = useState(""); + const [searchQuery] = useDebouncedValue(searchInput, { wait: 300 }); + + const handleSearchChange = useCallback((value: string) => { + setSearchInput(value); + setTablePagination((prev) => ({ ...prev, pageIndex: 0 })); + }, []); + + const getFilterValue = useCallback( + (columnId: string): string | undefined => { + const entry = columnFilters.find((filter) => filter.id === columnId); + return typeof entry?.value === "string" && entry.value.trim() ? entry.value.trim() : undefined; + }, + [columnFilters], + ); const sortBy = sorting.length > 0 ? sorting[0].id : "created_at"; const sortOrder = sorting.length > 0 ? (sorting[0].desc ? "desc" : "asc") : "desc"; @@ -56,9 +71,8 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi refetch, } = useKeys(pageIndex + 1, pageSize, { teamID: teamId, - organizationID: filters["Organization ID"]?.trim() || undefined, - selectedKeyAlias: filters["Key Alias"]?.trim() || undefined, - userID: filters["User ID"]?.trim() || undefined, + selectedKeyAlias: searchQuery.trim() || undefined, + userID: getFilterValue("user_id"), sortBy: sortBy || undefined, sortOrder: sortOrder || undefined, expand: "user", @@ -95,18 +109,6 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi [teamId, teamAlias, organization], ); - const teamFilterOptionsQuery = useQuery({ - queryKey: ["teamFilterOptions", teamId, accessToken], - queryFn: async () => fetchTeamFilterOptions(accessToken, teamId), - enabled: !!accessToken && !!teamId, - staleTime: 30000, // 30 seconds - align with useKeys - }); - const teamFilterOptions = teamFilterOptionsQuery.data || { - keyAliases: [], - organizationIds: [], - userIds: [], - }; - const handleStorageChange = useCallback(() => { refetch?.(); }, [refetch]); @@ -116,76 +118,17 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi return () => window.removeEventListener("storage", handleStorageChange); }, [handleStorageChange]); - const handleFilterChange = useCallback((newFilters: Record) => { - setFilters((prev) => ({ - ...prev, - "Organization ID": newFilters["Organization ID"] ?? prev["Organization ID"], - "Key Alias": newFilters["Key Alias"] ?? prev["Key Alias"], - "User ID": newFilters["User ID"] ?? prev["User ID"], - })); + const handleColumnFiltersChange = useCallback>((updaterOrValue) => { + setColumnFilters(updaterOrValue); setTablePagination((prev) => ({ ...prev, pageIndex: 0 })); }, []); - const handleFilterReset = useCallback(() => { - setFilters({ - "Organization ID": "", - "Key Alias": "", - "User ID": "", - }); - setSorting(DEFAULT_SORTING); - setTablePagination((prev) => ({ ...prev, pageIndex: 0 })); - }, []); - - const filterOptions: FilterOption[] = useMemo( - () => [ - { - name: "Organization ID", - label: "Organization ID", - isSearchable: true, - searchFn: async (searchText: string) => { - const { organizationIds } = teamFilterOptions; - if (!organizationIds.length) return []; - const lower = searchText.toLowerCase(); - const filtered = lower ? organizationIds.filter((id) => id.toLowerCase().includes(lower)) : organizationIds; - return filtered.map((id) => ({ label: id, value: id })); - }, - }, - { - name: "Key Alias", - label: "Key Alias", - isSearchable: true, - searchFn: async (searchText: string) => { - const { keyAliases } = teamFilterOptions; - const lower = searchText.toLowerCase(); - const filtered = lower ? keyAliases.filter((alias) => alias.toLowerCase().includes(lower)) : keyAliases; - return filtered.map((alias) => ({ label: alias, value: alias })); - }, - }, - { - name: "User ID", - label: "User ID", - isSearchable: true, - searchFn: async (searchText: string) => { - const { userIds } = teamFilterOptions; - const lower = searchText.toLowerCase(); - const filtered = lower - ? userIds.filter((u) => u.id.toLowerCase().includes(lower) || u.email.toLowerCase().includes(lower)) - : userIds; - return filtered.map((u) => ({ - label: u.email ? `${u.id} (${u.email})` : u.id, - value: u.id, - })); - }, - }, - ], - [teamFilterOptions], - ); - const columns: ColumnDef[] = useMemo( () => [ { id: "token", accessorKey: "token", + meta: { title: "Key ID" }, header: ({ column }) => , size: 120, enableSorting: true, @@ -196,6 +139,7 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi { id: "key_alias", accessorKey: "key_alias", + meta: { title: "Key Alias" }, header: ({ column }) => , size: 150, enableSorting: true, @@ -268,6 +212,7 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi { id: "created_at", accessorKey: "created_at", + meta: { title: "Created At" }, header: ({ column }) => , size: 120, enableSorting: true, @@ -335,6 +280,7 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi { id: "updated_at", accessorKey: "updated_at", + meta: { title: "Updated At" }, header: ({ column }) => , size: 120, enableSorting: true, @@ -359,6 +305,7 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi { id: "spend", accessorKey: "spend", + meta: { title: "Spend (USD)" }, header: ({ column }) => , size: 100, enableSorting: true, @@ -367,6 +314,7 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi { id: "max_budget", accessorKey: "max_budget", + meta: { title: "Budget (USD)" }, header: ({ column }) => , size: 110, enableSorting: true, @@ -503,27 +451,7 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi onDelete={refetch} /> ) : ( -
-
- -
- -
- setTablePagination((prev) => ({ ...prev, pageIndex: nextPage }))} - onPageSizeChange={(nextSize) => setTablePagination({ pageIndex: 0, pageSize: nextSize })} - isLoading={isLoading || isFetching} - /> -
- +
null} + filterMode="server" + columnFilters={columnFilters} + onColumnFiltersChange={handleColumnFiltersChange} enableColumnResizing columnResizeMode="onChange" isLoading={isLoading || isFetching} loadingMessage="Loading keys..." - noDataMessage="No keys found" maxBodyHeight="75vh" size="compact" + toolbar={(table) => ( + <> + refetch?.()} + isRefreshing={isFetching} + onOpenFilters={() => setFiltersOpen(true)} + filterLabels={{ user_id: "User ID" }} + /> + + {({ get, set }) => ( + + set("user_id", event.target.value)} + placeholder="Filter by user ID…" + /> + + )} + + + )} />
)} diff --git a/ui/litellm-dashboard/src/components/ui/sheet.tsx b/ui/litellm-dashboard/src/components/ui/sheet.tsx new file mode 100644 index 00000000000..b619c927bbb --- /dev/null +++ b/ui/litellm-dashboard/src/components/ui/sheet.tsx @@ -0,0 +1,100 @@ +"use client"; + +import * as React from "react"; +import { Dialog as SheetPrimitive } from "@base-ui/react/dialog"; + +import { cn } from "@/lib/cva.config"; +import { Button } from "@/components/ui/button"; +import { XIcon } from "lucide-react"; + +function Sheet({ ...props }: SheetPrimitive.Root.Props) { + return ; +} + +function SheetTrigger({ ...props }: SheetPrimitive.Trigger.Props) { + return ; +} + +function SheetClose({ ...props }: SheetPrimitive.Close.Props) { + return ; +} + +function SheetPortal({ ...props }: SheetPrimitive.Portal.Props) { + return ; +} + +function SheetOverlay({ className, ...props }: SheetPrimitive.Backdrop.Props) { + return ( + + ); +} + +function SheetContent({ + className, + children, + side = "right", + showCloseButton = true, + ...props +}: SheetPrimitive.Popup.Props & { + side?: "top" | "right" | "bottom" | "left"; + showCloseButton?: boolean; +}) { + return ( + + + + {children} + {showCloseButton && ( + } + > + + Close + + )} + + + ); +} + +function SheetHeader({ className, ...props }: React.ComponentProps<"div">) { + return
; +} + +function SheetFooter({ className, ...props }: React.ComponentProps<"div">) { + return
; +} + +function SheetTitle({ className, ...props }: SheetPrimitive.Title.Props) { + return ( + + ); +} + +function SheetDescription({ className, ...props }: SheetPrimitive.Description.Props) { + return ( + + ); +} + +export { Sheet, SheetTrigger, SheetClose, SheetContent, SheetHeader, SheetFooter, SheetTitle, SheetDescription }; diff --git a/ui/litellm-dashboard/src/lib/http/api.test.ts b/ui/litellm-dashboard/src/lib/http/api.test.ts new file mode 100644 index 00000000000..fba2b7f77dc --- /dev/null +++ b/ui/litellm-dashboard/src/lib/http/api.test.ts @@ -0,0 +1,104 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { fetchClient } from "./api"; +import { + registerAuthHeaderNameGetter, + registerAuthTokenGetter, + registerBaseUrlGetter, + registerErrorHandler, +} from "./runtime"; + +const jsonResponse = (status: number, body: unknown): Response => + new Response(JSON.stringify(body), { status, headers: { "Content-Type": "application/json" } }); + +const capturingFetch = (response: Response) => { + const requests: Request[] = []; + const fetch = vi.fn(async (request: Request) => { + requests.push(request); + return response; + }); + return { fetch, requests }; +}; + +describe("typed api client middleware", () => { + beforeEach(() => { + registerBaseUrlGetter(() => ""); + registerAuthHeaderNameGetter(() => "Authorization"); + registerErrorHandler(() => {}); + registerAuthTokenGetter(() => null); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("injects the bearer token under the registered auth header name", async () => { + registerAuthTokenGetter(() => "sk-test"); + registerAuthHeaderNameGetter(() => "x-litellm-key"); + const { fetch, requests } = capturingFetch(jsonResponse(200, { data: [] })); + + await fetchClient.GET("/model_group/info", { fetch }); + + expect(requests[0].headers.get("x-litellm-key")).toBe("Bearer sk-test"); + expect(requests[0].headers.get("Authorization")).toBeNull(); + }); + + it("omits the auth header when no token is set", async () => { + const { fetch, requests } = capturingFetch(jsonResponse(200, { data: [] })); + + await fetchClient.GET("/model_group/info", { fetch }); + + expect(requests[0].headers.get("Authorization")).toBeNull(); + }); + + it("rebases the request onto the registered base url, preserving path and query", async () => { + registerBaseUrlGetter(() => "https://proxy.example.com/"); + const { fetch, requests } = capturingFetch(jsonResponse(200, { data: [] })); + + await fetchClient.GET("/model_group/info", { fetch, params: { query: { model_group: "gpt-4o" } } }); + + const url = new URL(requests[0].url); + expect(url.origin).toBe("https://proxy.example.com"); + expect(url.pathname).toBe("/model_group/info"); + expect(url.searchParams.get("model_group")).toBe("gpt-4o"); + }); + + it("maps a non-2xx response to an ApiError carrying status and the derived message", async () => { + const { fetch } = capturingFetch(jsonResponse(403, { error: { message: "no access" } })); + + await expect(fetchClient.GET("/model_group/info", { fetch })).rejects.toMatchObject({ + name: "ApiError", + status: 403, + message: "no access", + }); + }); + + it("returns the parsed body on a successful response", async () => { + const body = { data: [{ model_group: "gpt-4o" }] }; + const { fetch } = capturingFetch(jsonResponse(200, body)); + + const { data, error } = await fetchClient.GET("/model_group/info", { fetch }); + + expect(error).toBeUndefined(); + expect(data).toEqual(body); + }); + + it("reports the derived message to the registered error handler on a non-2xx response", async () => { + const onError = vi.fn(); + registerErrorHandler(onError); + const { fetch } = capturingFetch(jsonResponse(401, { error: { message: "Authentication Error - Expired Key" } })); + + await expect(fetchClient.GET("/model_group/info", { fetch })).rejects.toBeInstanceOf(Error); + + expect(onError).toHaveBeenCalledWith("Authentication Error - Expired Key"); + }); + + it("does not call the error handler on a successful response", async () => { + const onError = vi.fn(); + registerErrorHandler(onError); + const { fetch } = capturingFetch(jsonResponse(200, { data: [] })); + + await fetchClient.GET("/model_group/info", { fetch }); + + expect(onError).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/litellm-dashboard/src/lib/http/api.ts b/ui/litellm-dashboard/src/lib/http/api.ts new file mode 100644 index 00000000000..2e8e1d36c5b --- /dev/null +++ b/ui/litellm-dashboard/src/lib/http/api.ts @@ -0,0 +1,48 @@ +import createFetchClient, { type Middleware } from "openapi-fetch"; +import type { paths } from "./schema"; +import { ApiError, deriveErrorMessage } from "./client"; +import { getAuthHeaderName, getAuthToken, getRequestBaseUrl, reportError } from "./runtime"; + +const rebaseUrl = (requestUrl: string, base: string): string => { + const { pathname, search } = new URL(requestUrl); + return `${base.replace(/\/+$/, "")}${pathname}${search}`; +}; + +const middleware: Middleware = { + onRequest({ request }) { + const base = getRequestBaseUrl(); + const next = new Request(base ? rebaseUrl(request.url, base) : request.url, request); + const token = getAuthToken(); + if (token) { + next.headers.set(getAuthHeaderName(), `Bearer ${token}`); + } + return next; + }, + async onResponse({ response }) { + if (response.ok) return response; + const raw = await response.clone().text(); + let body: unknown = raw; + let message: string; + try { + body = JSON.parse(raw); + message = deriveErrorMessage(body); + } catch { + message = raw || `HTTP ${response.status}`; + } + reportError(message); + throw new ApiError(message, response.status, body); + }, +}; + +/** + * The typed, schema-bound HTTP client. Use it inside TanStack Query hooks + * (`fetchClient.GET("/path", { params })`) and for imperative calls; path + * params, query params, and request bodies are inferred from schema.d.ts. + * + * The creation-time base is the current origin so request URLs are absolute; the + * middleware rebases each call onto the runtime base when one is registered (a + * split-origin proxy or worker URL), injects the auth header, and maps non-2xx + * responses to ApiError so query functions can just read `.data`. + */ +export const fetchClient = createFetchClient({ baseUrl: globalThis.location?.origin ?? "" }); +fetchClient.use(middleware); diff --git a/ui/litellm-dashboard/src/lib/http/runtime.test.ts b/ui/litellm-dashboard/src/lib/http/runtime.test.ts new file mode 100644 index 00000000000..a475cc2bb10 --- /dev/null +++ b/ui/litellm-dashboard/src/lib/http/runtime.test.ts @@ -0,0 +1,22 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { getAuthHeaderName, getRequestBaseUrl } from "./runtime"; + +describe("runtime request config defaults", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("resolves the default base URL from NEXT_PUBLIC_BASE_URL before a getter is registered", () => { + vi.stubEnv("NEXT_PUBLIC_BASE_URL", "https://proxy.example.com/"); + expect(getRequestBaseUrl()).toBe("https://proxy.example.com"); + }); + + it("defaults the base URL to same-origin when NEXT_PUBLIC_BASE_URL is unset", () => { + vi.stubEnv("NEXT_PUBLIC_BASE_URL", ""); + expect(getRequestBaseUrl()).toBe(""); + }); + + it("defaults the auth header name to Authorization", () => { + expect(getAuthHeaderName()).toBe("Authorization"); + }); +}); diff --git a/ui/litellm-dashboard/src/lib/http/runtime.ts b/ui/litellm-dashboard/src/lib/http/runtime.ts new file mode 100644 index 00000000000..dd105255fb1 --- /dev/null +++ b/ui/litellm-dashboard/src/lib/http/runtime.ts @@ -0,0 +1,46 @@ +import { resolveApiBase } from "./resolveApiBase"; + +/** + * Runtime request config the typed client reads on every call. The values are + * mutable at runtime (base URL can switch to a worker origin; the auth header + * name and token come from the logged-in session), and they are owned outside + * this module: networking.tsx registers the base URL / header-name / token + * getters and the error handler. The token getter reads the session cookie, the + * same source useAuthorized decodes, so the client's token and the gate that + * enables a query cannot diverge. Keeping the seam here (not importing from the + * component tree) lets api.ts stay in lib/http without a layering inversion. + * + * The base URL default resolves from NEXT_PUBLIC_BASE_URL so a request still + * hits the right origin if it fires before networking registers its fuller + * getter (which additionally folds in the server root path from the live UI + * config). The auth header name has no build-time source, so it defaults to + * "Authorization" until the session's JWT supplies a custom one. + */ + +type Getter = () => T; + +let baseUrlGetter: Getter = () => resolveApiBase({ explicitBase: process.env.NEXT_PUBLIC_BASE_URL }); +let authHeaderNameGetter: Getter = () => "Authorization"; +let authTokenGetter: Getter = () => null; +let errorHandler: (message: string) => void = () => {}; + +export const registerBaseUrlGetter = (getter: Getter): void => { + baseUrlGetter = getter; +}; + +export const registerAuthHeaderNameGetter = (getter: Getter): void => { + authHeaderNameGetter = getter; +}; + +export const registerAuthTokenGetter = (getter: Getter): void => { + authTokenGetter = getter; +}; + +export const registerErrorHandler = (handler: (message: string) => void): void => { + errorHandler = handler; +}; + +export const getRequestBaseUrl = (): string => baseUrlGetter(); +export const getAuthHeaderName = (): string => authHeaderNameGetter(); +export const getAuthToken = (): string | null => authTokenGetter(); +export const reportError = (message: string): void => errorHandler(message); diff --git a/uv.lock b/uv.lock index 692c1c2cb8f..38a662db07c 100644 --- a/uv.lock +++ b/uv.lock @@ -9,7 +9,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-07-06T18:05:33.611729Z" +exclude-newer = "2026-07-08T17:10:15.525149Z" exclude-newer-span = "P3D" [manifest] @@ -2252,14 +2252,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b0/03/84359833f7e1d49a883e92777637c592306030e30cee5e2b1e6476f95c88/greenlet-3.5.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:29ea813b2e1f45fa9649a17853b2b5465c4072fbcb072e5af6cd3a288216574a", size = 283502, upload-time = "2026-04-27T12:20:55.213Z" }, { url = "https://files.pythonhosted.org/packages/25/ce/6f9f008266273aa14a2e011945797ac5802b97b8b40efe7afe1ee6c1afc9/greenlet-3.5.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:804a70b328e706b785c6ef16187051c394a63dd1a906d89be24b6ad77759f13f", size = 600508, upload-time = "2026-04-27T12:52:37.876Z" }, { url = "https://files.pythonhosted.org/packages/e0/6d/b0f3272c2368ea2c1aa19a5ad70db0be8f8dff6e6d3d1eb82efa00cbcf19/greenlet-3.5.0-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:884f649de075b84739713d41dd4dfd41e2b910bfb769c4a3ea02ec1da52cd9bb", size = 613283, upload-time = "2026-04-27T12:59:37.957Z" }, + { url = "https://files.pythonhosted.org/packages/e5/ae/1db979ff6ae7958d80b288f63d5f6c30df96682700ea9fc340ce994d94a1/greenlet-3.5.0-cp310-cp310-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4d0eadc7e4d9ffb2af4247b606cae307be8e448911e5a0d0b16d72fc3d224cfd", size = 619894, upload-time = "2026-04-27T13:02:35.13Z" }, { url = "https://files.pythonhosted.org/packages/ed/ac/0b509b6fb93551ce5a01612ee1acda7f7dda4bbb66c99aeb2ab403d205dc/greenlet-3.5.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b28037cb07768933c54d81bfe47a85f9f402f57d7d69743b991a713b63954eb", size = 613418, upload-time = "2026-04-27T12:25:23.852Z" }, + { url = "https://files.pythonhosted.org/packages/ce/94/b0590e3d1978f02419f30502341c40d72f77eb0a2198119fe27df47714ee/greenlet-3.5.0-cp310-cp310-manylinux_2_39_riscv64.whl", hash = "sha256:f8c30c2225f40dd76c50790f0eb3b5c7c18431efb299e2782083e1981feed243", size = 415681, upload-time = "2026-04-27T13:05:11.494Z" }, { url = "https://files.pythonhosted.org/packages/03/03/2b2b680ec87aaa97998fb5b8d76658d4d3560386864f17efab33ba7c2e24/greenlet-3.5.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cda05425526240807408156b6960a17a79a0c760b813573b67027823be760977", size = 1572229, upload-time = "2026-04-27T12:53:23.509Z" }, { url = "https://files.pythonhosted.org/packages/61/e4/42b259e7a19aff1a270a4bd82caf6353109ed6860c9454e18f37162b83ae/greenlet-3.5.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9c615f869163e14bb1ced20322d8038fb680b08236521ac3f30cd4c1288785a0", size = 1639886, upload-time = "2026-04-27T12:25:22.325Z" }, { url = "https://files.pythonhosted.org/packages/6f/b4/733ca47b883b67c57f90d3ecb21055c9ec753597d10754ac201644061f9d/greenlet-3.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:ba8f0bdc2fae6ce915dfd0c16d2d00bca7e4247c1eae4416e06430e522137858", size = 237795, upload-time = "2026-04-27T12:21:40.118Z" }, { url = "https://files.pythonhosted.org/packages/8b/0f/a91f143f356523ff682309732b175765a9bc2836fd7c081c2c67fedc1ad4/greenlet-3.5.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:8f1cc966c126639cd152fdaa52624d2655f492faa79e013fea161de3e6dda082", size = 284726, upload-time = "2026-04-27T12:20:51.402Z" }, { url = "https://files.pythonhosted.org/packages/95/82/800646c7ffc5dbabd75ddd2f6b519bb898c0c9c969e5d0473bfe5d20bcce/greenlet-3.5.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:362624e6a8e5bca3b8233e45eef33903a100e9539a2b995c364d595dbc4018b3", size = 604264, upload-time = "2026-04-27T12:52:39.494Z" }, { url = "https://files.pythonhosted.org/packages/ca/ac/354867c0bba812fc33b15bc55aedafedd0aee3c7dd91dfca22444157dc0c/greenlet-3.5.0-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5ecd83806b0f4c2f53b1018e0005cd82269ea01d42befc0368730028d850ed1c", size = 616099, upload-time = "2026-04-27T12:59:39.623Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ab/192090c4a5b30df148c22bf4b8895457d739a7c7c5a7b9c41e5dd7f537f2/greenlet-3.5.0-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa94cb2288681e3a11645958f1871d48ee9211bd2f66628fdace505927d6e564", size = 623976, upload-time = "2026-04-27T13:02:37.363Z" }, { url = "https://files.pythonhosted.org/packages/ff/b0/815bece7399e01cadb69014219eebd0042339875c59a59b0820a46ece356/greenlet-3.5.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0ff251e9a0279522e62f6176412869395a64ddf2b5c5f782ff609a8216a4e662", size = 615198, upload-time = "2026-04-27T12:25:25.928Z" }, + { url = "https://files.pythonhosted.org/packages/24/11/05eb2b9b188c6df7d68a89c99134d644a7af616a40b9808e8e6ced315d5d/greenlet-3.5.0-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:64d6ac45f7271f48e45f67c95b54ef73534c52ec041fcda8edf520c6d811f4bc", size = 418379, upload-time = "2026-04-27T13:05:12.755Z" }, { url = "https://files.pythonhosted.org/packages/10/80/3b2c0a895d6698f6ddb31b07942ebfa982f3e30888bc5546a5b5990de8b2/greenlet-3.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d874e79afd41a96e11ff4c5d0bc90a80973e476fda1c2c64985667397df432b", size = 1574927, upload-time = "2026-04-27T12:53:25.81Z" }, { url = "https://files.pythonhosted.org/packages/44/0e/f354af514a4c61454dbc68e44d47544a5a4d6317e30b77ddfa3a09f4c5f3/greenlet-3.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0ed006e4b86c59de7467eb2601cd1b77b5a7d657d1ee55e30fe30d76451edba4", size = 1642683, upload-time = "2026-04-27T12:25:23.9Z" }, { url = "https://files.pythonhosted.org/packages/fa/6a/87f38255201e993a1915265ebb80cd7c2c78b04a45744995abbf6b259fd8/greenlet-3.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:703cb211b820dbffbbc55a16bfc6e4583a6e6e990f33a119d2cc8b83211119c8", size = 238115, upload-time = "2026-04-27T12:21:48.845Z" }, @@ -2267,7 +2271,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ef/32/f2ce6d4cac3e55bc6173f92dbe627e782e1850f89d986c3606feb63aafa7/greenlet-3.5.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:db2910d3c809444e0a20147361f343fe2798e106af8d9d8506f5305302655a9f", size = 286228, upload-time = "2026-04-27T12:20:34.421Z" }, { url = "https://files.pythonhosted.org/packages/b7/aa/caed9e5adf742315fc7be2a84196373aab4816e540e38ba0d76cb7584d68/greenlet-3.5.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ec9ea74e7268ace7f9aab1b1a4e730193fc661b39a993cd91c606c32d4a3628", size = 601775, upload-time = "2026-04-27T12:52:41.045Z" }, { url = "https://files.pythonhosted.org/packages/c7/af/90ae08497400a941595d12774447f752d3dfe0fbb012e35b76bc5c0ff37e/greenlet-3.5.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54d243512da35485fc7a6bf3c178fdda6327a9d6506fcdd62b1abd1e41b2927b", size = 614436, upload-time = "2026-04-27T12:59:41.595Z" }, + { url = "https://files.pythonhosted.org/packages/3f/e9/4eeadf8cb3403ac274245ba75f07844abc7fa5f6787583fc9156ba741e0f/greenlet-3.5.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:41353ec2ecedf7aa8f682753a41919f8718031a6edac46b8d3dc7ed9e1ceb136", size = 620610, upload-time = "2026-04-27T13:02:39.194Z" }, { url = "https://files.pythonhosted.org/packages/2b/e0/2e13df68f367e2f9960616927d60857dd7e56aaadd59a47c644216b2f920/greenlet-3.5.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d280a7f5c331622c69f97eb167f33577ff2d1df282c41cd15907fc0a3ca198c", size = 611388, upload-time = "2026-04-27T12:25:28.008Z" }, + { url = "https://files.pythonhosted.org/packages/ee/ef/f913b3c0eb7d26d86a2401c5e1546c9d46b657efee724b06f6f4ac5d8824/greenlet-3.5.0-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:58c1c374fe2b3d852f9b6b11a7dff4c85404e51b9a596fd9e89cf904eb09866d", size = 422775, upload-time = "2026-04-27T13:05:14.261Z" }, { url = "https://files.pythonhosted.org/packages/82/f7/393c64055132ac0d488ef6be549253b7e6274194863967ddc0bc8f5b87b8/greenlet-3.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1eb67d5adefb5bd2e182d42678a328979a209e4e82eb93575708185d31d1f588", size = 1570768, upload-time = "2026-04-27T12:53:28.099Z" }, { url = "https://files.pythonhosted.org/packages/b8/4b/eaf7735253522cf56d1b74d672a58f54fc114702ceaf05def59aae72f6e1/greenlet-3.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2628d6c86f6cb0cb45e0c3c54058bbec559f57eaae699447748cb3928150577e", size = 1635983, upload-time = "2026-04-27T12:25:26.903Z" }, { url = "https://files.pythonhosted.org/packages/4c/fe/4fb3a0805bd5165da5ebf858da7cc01cce8061674106d2cf5bdab32cbfde/greenlet-3.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:d4d9f0624c775f2dfc56ba54d515a8c771044346852a918b405914f6b19d7fd8", size = 238840, upload-time = "2026-04-27T12:23:54.806Z" }, @@ -2275,7 +2281,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/58/fc576f99037ce19c5aa16628e4c3226b6d1419f72a62c79f5f40576e6eb3/greenlet-3.5.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:5a5ed18de6a0f6cc7087f1563f6bd93fc7df1c19165ca01e9bde5a5dc281d106", size = 285066, upload-time = "2026-04-27T12:23:05.033Z" }, { url = "https://files.pythonhosted.org/packages/4a/ba/b28ddbe6bfad6a8ac196ef0e8cff37bc65b79735995b9e410923fffeeb70/greenlet-3.5.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a717fbc46d8a354fa675f7c1e813485b6ba3885f9bef0cd56e5ba27d758ff5b", size = 604414, upload-time = "2026-04-27T12:52:42.358Z" }, { url = "https://files.pythonhosted.org/packages/09/06/4b69f8f0b67603a8be2790e55107a190b376f2627fe0eaf5695d85ffb3cd/greenlet-3.5.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ddc090c5c1792b10246a78e8c2163ebbe04cf877f9d785c230a7b27b39ad038e", size = 617349, upload-time = "2026-04-27T12:59:43.32Z" }, + { url = "https://files.pythonhosted.org/packages/6a/15/a643b4ecd09969e30b8a150d5919960caae0abe4f5af75ab040b1ab85e78/greenlet-3.5.0-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4964101b8585c144cbda5532b1aa644255126c08a265dae90c16e7a0e63aaa9d", size = 623234, upload-time = "2026-04-27T13:02:40.611Z" }, { url = "https://files.pythonhosted.org/packages/8a/17/a3918541fd0ddefe024a69de6d16aa7b46d36ac19562adaa63c7fa180eff/greenlet-3.5.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2094acd54b272cb6eae8c03dd87b3fa1820a4cef18d6889c378d503500a1dc13", size = 613927, upload-time = "2026-04-27T12:25:30.28Z" }, + { url = "https://files.pythonhosted.org/packages/77/18/3b13d5ef1275b0ffaf933b05efa21408ac4ca95823c7411d79682e4fdcff/greenlet-3.5.0-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:7022615368890680e67b9965d33f5773aade330d5343bbe25560135aaa849eae", size = 425243, upload-time = "2026-04-27T13:05:15.689Z" }, { url = "https://files.pythonhosted.org/packages/ee/e1/bd0af6213c7dd33175d8a462d4c1fe1175124ebed4855bc1475a5b5242c2/greenlet-3.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5e05ba267789ea87b5a155cf0e810b1ab88bf18e9e8740813945ceb8ee4350ba", size = 1570893, upload-time = "2026-04-27T12:53:29.483Z" }, { url = "https://files.pythonhosted.org/packages/9b/2a/0789702f864f5382cb476b93d7a9c823c10472658102ccd65f415747d2e2/greenlet-3.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0ecec963079cd58cbd14723582384f11f166fd58883c15dcbfb342e0bc9b5846", size = 1636060, upload-time = "2026-04-27T12:25:28.845Z" }, { url = "https://files.pythonhosted.org/packages/b2/8f/22bf9df92bbff0eb07842b60f7e63bf7675a9742df628437a9f02d09137f/greenlet-3.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:728d9667d8f2f586644b748dbd9bb67e50d6a9381767d1357714ea6825bb3bf5", size = 238740, upload-time = "2026-04-27T12:24:01.341Z" }, @@ -3453,6 +3461,10 @@ dev = [ { name = "types-setuptools" }, { name = "vcrpy" }, ] +e2e-dev = [ + { name = "playwright" }, + { name = "websockets" }, +] healthcheck = [ { name = "httpx" }, { name = "pyyaml" }, @@ -3622,6 +3634,10 @@ dev = [ { name = "types-setuptools", specifier = "==75.8.0.20250225" }, { name = "vcrpy", specifier = "==8.2.1" }, ] +e2e-dev = [ + { name = "playwright", specifier = "==1.61.0" }, + { name = "websockets", specifier = ">=15.0.1,<16.0" }, +] healthcheck = [ { name = "httpx", specifier = "==0.28.1" }, { name = "pyyaml", specifier = "==6.0.3" }, @@ -5343,6 +5359,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/75/a6/a0a304dc33b49145b21f4808d763822111e67d1c3a32b524a1baf947b6e1/platformdirs-4.9.6-py3-none-any.whl", hash = "sha256:e61adb1d5e5cb3441b4b7710bea7e4c12250ca49439228cc1021c00dcfac0917", size = 21348, upload-time = "2026-04-09T00:04:09.463Z" }, ] +[[package]] +name = "playwright" +version = "1.61.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet" }, + { name = "pyee" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/ee/31e4e0db36588b817a10b299a0285082545fde7d36543c2abe498bb3d61a/playwright-1.61.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:ff138c3a604f69911e9d42fd036e55c2a171e5616edf04c1e7f60a2a285540b0", size = 43421877, upload-time = "2026-06-29T10:32:48.428Z" }, + { url = "https://files.pythonhosted.org/packages/42/35/71395dd3ecc798965be4a3ef8c443217d4abca168e7cb34536304f9489e6/playwright-1.61.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:009588c2a7e499bc5a8b425b61fa65490968bbda9cd69e0cf2cff10f8304659a", size = 42205016, upload-time = "2026-06-29T10:32:52.104Z" }, + { url = "https://files.pythonhosted.org/packages/f4/44/323164cf5cd1647bdefce76ffce27651aadb959d089b48f53ea40918276e/playwright-1.61.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:9f7de4536088d12037c13a52b7ea34b59270b78926bb56935070597ffac6b1af", size = 43421884, upload-time = "2026-06-29T10:32:55.773Z" }, + { url = "https://files.pythonhosted.org/packages/ab/f8/a35bf179e4ba2522c1893635094a64e407572547bd61528820fc0abc87fe/playwright-1.61.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:54f3b39f6eab832e33458c1dd7da0b5682aedab3b09ae731b5c59fa12fd2024e", size = 47421381, upload-time = "2026-06-29T10:32:59.903Z" }, + { url = "https://files.pythonhosted.org/packages/b7/eb/e3f922348ec17c315f98c463f72faa1181a1c3de0bfe31a8d2edf6561723/playwright-1.61.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:93454322ade8c11d5d6c211bfd91bdfb9ffb4810e3e026371bcbc4bec1b7ee4c", size = 47120545, upload-time = "2026-06-29T10:33:03.574Z" }, + { url = "https://files.pythonhosted.org/packages/c2/a6/5be4e52b40a9c0c8a073e7c5b0785c05cf5a9ea8f8a7b5b260e32d970342/playwright-1.61.0-py3-none-win32.whl", hash = "sha256:372d55a6f1248fa1dd47599686980cb8fb5bbe6fcda59eab793eb657c11d8a9b", size = 37844841, upload-time = "2026-06-29T10:33:07.361Z" }, + { url = "https://files.pythonhosted.org/packages/6c/fd/2b78036e5fbe9d5f5645bbe08a1eac7160c51243c0093963edbcf67c35d9/playwright-1.61.0-py3-none-win_amd64.whl", hash = "sha256:35c6cc4589a5d00964a59d7b3e59641e0aac0c02f15479a7af77d20f6bc79597", size = 37844846, upload-time = "2026-06-29T10:33:10.637Z" }, + { url = "https://files.pythonhosted.org/packages/27/0d/1b0f3c4ee4eb0514bc805b5c2f9a223e5b6de4f11a926f5235d51d0fc81b/playwright-1.61.0-py3-none-win_arm64.whl", hash = "sha256:e9fcbffcf557a8620fdedd92491eb59a32d18e23d6f3b4f6214b952be324fe51", size = 33955127, upload-time = "2026-06-29T10:33:14.008Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -5915,6 +5950,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/c1/6e422f34e569cf8e18df68d1939c81c099d2b61e4f7d9621c8a77560799c/pydantic_settings-2.14.2-py3-none-any.whl", hash = "sha256:a20c97b37910b6550d5ea50fbcc2d4187defe58cd57070b73863d069419c9440", size = 61715, upload-time = "2026-06-19T13:44:55.02Z" }, ] +[[package]] +name = "pyee" +version = "13.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c4/b4d4827c93ef43c01f599ef31453ccc1c132b353284fc6c87d535c233129/pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228", size = 15659, upload-time = "2026-02-14T21:12:26.263Z" }, +] + [[package]] name = "pyflakes" version = "3.4.0"