scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-06 08:16:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 79

Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_fix_azure_chat_anyof_tool_schema

Browse source
This commit is contained in:
mateo-berri 2026-09-01 14:08:34 -07:00
commit b311c24253
43 changed files with 1336 additions and 995 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
.circleci/config.yml
View file

@ -1483,7 +1483,7 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py -k "not legacy_resolver"
uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py -k "not v2_resolver"
installing_litellm_on_python_3_13:
docker:
@ -1507,9 +1507,9 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py -k "not legacy_resolver"
uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py -k "not v2_resolver"
installing_litellm_on_python_legacy_migration_resolver:
installing_litellm_on_python_v2_migration_resolver:
docker:
- *python312_image
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
@ -1536,10 +1536,10 @@ jobs:
url: tcp://localhost:5432
timeout: "60"
- run:
name: Run legacy migration resolver proxy smoke test
name: Run v2 migration resolver proxy smoke test
command: |
uv run --no-sync python -m pytest -vv \
tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings_legacy_resolver
tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings_v2_resolver
helm_chart_testing:
machine:
@ -2879,8 +2879,7 @@ jobs:
command: |
if grep -q "Error: P1001: Can't reach database server at" docker_output.log && \
(grep -q "Database setup failed after multiple retries" docker_output.log || \
grep -q "ERROR: Application startup failed. Exiting." docker_output.log || \
grep -q "Database migration cannot proceed" docker_output.log); then
grep -q "ERROR: Application startup failed. Exiting." docker_output.log); then
echo "Expected error found. Test passed."
else
echo "Expected error not found. Test failed."
@ -3012,7 +3011,7 @@ workflows:
filters: *main_branches
- installing_litellm_on_python_3_13:
filters: *main_branches
- installing_litellm_on_python_legacy_migration_resolver:
- installing_litellm_on_python_v2_migration_resolver:
filters: *main_branches
- helm_chart_testing:
requires:

2
CLAUDE.md
View file

@ -37,7 +37,7 @@ Same applies for filing bug reports and feature requests, with .github/ISSUE_TEM
If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just leave the section blank
Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload 2>&1 | tee litellm.log`; the Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR
Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`; the Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR
If you ever write any human-facing text (pull requests, issues, commit messages, discussion posts, github comments, release notes, docs, etc.), always follow these guidelines to sound less AI-y:
- don't use emojis

132
litellm-proxy-extras/litellm_proxy_extras/utils.py
View file

@ -7,8 +7,7 @@ import subprocess
import tempfile
import time
from pathlib import Path
from types import MappingProxyType
from typing import Final, Optional
from typing import Optional
from litellm_proxy_extras._logging import logger
from litellm_proxy_extras.replica_identity import (
@ -51,17 +50,6 @@ _SPEND_LOGS_PK_CLAUSE_RE = re.compile(
re.IGNORECASE,
)
_PRISMA_ATTEMPTS: Final = 4
_TRANSIENT_PRISMA_FAILURES: Final = MappingProxyType(
{
"deadlock detected": "a deadlock on the migration advisory lock (a concurrent migrate deploy)",
"P1001": "an unreachable database server",
"P1002": "a database server that timed out",
}
)
PARTITIONED_SPEND_LOGS_PUSH_ERROR = (
"LiteLLM_SpendLogs is a partitioned table (see db_scripts/partition_spend_logs.sql), "
"so its primary key must include the partition key (\"startTime\"). `prisma db push` "
@ -286,23 +274,6 @@ class ProxyExtrasDBManager:
env=prisma_env,
)
@staticmethod
def _transient_prisma_failure(stderr: str) -> str | None:
"""Why a failed prisma command is worth retrying, or None.
v1 retried every failure, so it absorbed a database that was not up yet
or another instance holding the migration lock. v2 fails fast, which is
right for a broken migration and wrong for these.
"""
return next(
(
reason
for marker, reason in _TRANSIENT_PRISMA_FAILURES.items()
if marker in stderr
),
None,
)
@staticmethod
def _is_permission_error(error_message: str) -> bool:
"""
@ -684,7 +655,7 @@ class ProxyExtrasDBManager:
@staticmethod
def _setup_database_v2(use_migrate: bool) -> bool:
"""
v2 migration resolver (what the proxy CLI selects by default).
v2 migration resolver (opt-in via --use_v2_migration_resolver).
Runs `prisma migrate deploy` and handles standard recovery paths
(P3005 baseline, P3009/P3018 idempotent errors). Critically, it does
@ -705,46 +676,20 @@ class ProxyExtrasDBManager:
original_dir = os.getcwd()
os.chdir(migrations_dir)
try:
for attempt in range(_PRISMA_ATTEMPTS):
try:
subprocess.run(
[_get_prisma_command(), "db", "push", "--accept-data-loss"],
timeout=prisma_command_timeout(),
check=True,
capture_output=True,
text=True,
env=_get_prisma_env(),
)
return True
except subprocess.TimeoutExpired:
logger.info(
"prisma db push attempt %s timed out, retrying",
attempt + 1,
)
time.sleep(random.randrange(5, 15))
except subprocess.CalledProcessError as e:
stderr = e.stderr or ""
transient = ProxyExtrasDBManager._transient_prisma_failure(
stderr
)
# Re-raise as RuntimeError so proxy_cli.py's
# `except RuntimeError` catches it and exits cleanly.
if transient is None or attempt == _PRISMA_ATTEMPTS - 1:
raise RuntimeError(
f"prisma db push failed.\n\nDetail: {e}"
f"\n\nPrisma error:\n{stderr}"
) from e
logger.info(
"prisma db push attempt %s failed on %s, retrying. "
"Prisma error:\n%s",
attempt + 1,
transient,
stderr,
)
time.sleep(random.randrange(5, 15))
raise RuntimeError(
f"prisma db push failed after {_PRISMA_ATTEMPTS} attempts."
subprocess.run(
[_get_prisma_command(), "db", "push", "--accept-data-loss"],
timeout=prisma_command_timeout(),
check=True,
env=_get_prisma_env(),
)
return True
except (
subprocess.CalledProcessError,
subprocess.TimeoutExpired,
) as e:
# Re-raise as RuntimeError so proxy_cli.py's
# `except RuntimeError` catches it and exits cleanly.
raise RuntimeError(f"prisma db push failed.\n\nDetail: {e}") from e
finally:
os.chdir(original_dir)
@ -754,7 +699,7 @@ class ProxyExtrasDBManager:
original_dir = os.getcwd()
os.chdir(migrations_dir)
try:
for attempt in range(_PRISMA_ATTEMPTS):
for attempt in range(4):
try:
result = subprocess.run(
[_get_prisma_command(), "migrate", "deploy"],
@ -869,36 +814,16 @@ class ProxyExtrasDBManager:
f"Manual intervention required.\n\nPrisma error:\n{stderr}"
) from e
transient = ProxyExtrasDBManager._transient_prisma_failure(stderr)
if transient is None:
raise RuntimeError(
"Database migration failed and cannot be auto-recovered. "
f"Manual intervention required.\n\nPrisma error:\n{stderr}"
) from e
if attempt == _PRISMA_ATTEMPTS - 1:
raise RuntimeError(
f"Database migration failed after "
f"{_PRISMA_ATTEMPTS} attempts on {transient}. "
"Check database connectivity and load."
f"\n\nPrisma error:\n{stderr}"
) from e
logger.info(
"prisma migrate deploy attempt %s failed on %s, retrying. "
"Prisma error:\n%s",
attempt + 1,
transient,
stderr,
)
time.sleep(random.randrange(5, 15))
continue
raise RuntimeError(
"Database migration failed and cannot be auto-recovered. "
f"Manual intervention required.\n\nPrisma error:\n{stderr}"
) from e
raise RuntimeError(
f"Database migration failed after {_PRISMA_ATTEMPTS} "
"attempts (retry loop exhausted by timeouts or repeated "
"idempotent-recovery continues). Check database connectivity, "
"load, and _prisma_migrations ledger state."
"Database migration failed after 4 attempts (retry loop "
"exhausted by timeouts or repeated idempotent-recovery "
"continues). Check database connectivity, load, and "
"_prisma_migrations ledger state."
)
finally:
os.chdir(original_dir)
@ -946,11 +871,10 @@ class ProxyExtrasDBManager:
Args:
use_migrate: Whether to use prisma migrate instead of db push
use_v2_resolver: Run the v2 migration resolver (safer during
use_v2_resolver: Opt into the v2 migration resolver (safer during
rolling deploys; does not run the diff-and-force recovery
that causes schema thrashing). Defaults to False here so
direct callers keep the old behavior; the proxy CLI passes
True, so the proxy's runtime default is v2.
that causes schema thrashing). Defaults to False for
backwards compatibility.
Returns:
bool: True if setup was successful, False otherwise
@ -968,7 +892,7 @@ class ProxyExtrasDBManager:
@staticmethod
def _run_migrations(use_migrate: bool, use_v2_resolver: bool) -> bool:
if use_v2_resolver:
logger.info("Using v2 migration resolver")
logger.info("Using v2 migration resolver (--use_v2_migration_resolver)")
return ProxyExtrasDBManager._setup_database_v2(use_migrate=use_migrate)
schema_path = ProxyExtrasDBManager._get_prisma_dir() + "/schema.prisma"

0
litellm-proxy-extras/tests/__init__.py Normal file
View file

242
litellm-proxy-extras/tests/test_setup_database_fail_fast.py Normal file
View file

@ -0,0 +1,242 @@
"""Regression tests for ProxyExtrasDBManager v2 migration resolver.
The v2 resolver is opt-in via `--use_v2_migration_resolver` / the
`use_v2_resolver=True` kwarg. These tests exercise the v2 path; the v1
(default) behavior is unchanged from pre-fix.
"""
import subprocess
from unittest.mock import patch
import pytest
from litellm_proxy_extras.utils import (
ProxyExtrasDBManager,
_max_migration_timestamp,
_migration_timestamp,
)
def _fake_migrate_deploy_failure(returncode: int, stderr: str):
def _run(*args, **kwargs):
raise subprocess.CalledProcessError(
returncode=returncode,
cmd=args[0],
stderr=stderr,
output="",
)
return _run
def test_v2_p3018_permission_error_raises_runtime_error(monkeypatch, tmp_path):
"""v2: a permission failure during migrate deploy raises RuntimeError."""
monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x")
monkeypatch.setattr(
ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
)
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
(tmp_path / "schema.prisma").write_text("// stub")
stderr = (
"Error: P3018\nMigration name: 20250326162113_baseline\n"
"Database error code: 42501\npermission denied for schema public"
)
with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
with pytest.raises(RuntimeError, match="permission"):
ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
def test_v2_non_idempotent_p3009_raises_runtime_error(monkeypatch, tmp_path):
"""v2: a non-idempotent migration failure raises (no silent recovery)."""
monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x")
monkeypatch.setattr(
ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
)
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
(tmp_path / "schema.prisma").write_text("// stub")
stderr = (
"Error: P3009\nMigration `20260101000000_genuinely_broken` failed\n"
'Reason: syntax error at or near "BRKN" LINE 42'
)
with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
def test_strip_prisma_query_params_removes_connection_limit():
"""DATABASE_URLs with Prisma-specific params should be parseable by psycopg."""
url = "postgresql://u:p@h:5432/db?connection_limit=100&pool_timeout=60&sslmode=require"
stripped = ProxyExtrasDBManager._strip_prisma_query_params(url)
assert "connection_limit" not in stripped
assert "pool_timeout" not in stripped
assert "sslmode=require" in stripped
def test_strip_prisma_query_params_passthrough_no_query():
"""URLs without query strings are returned unchanged."""
url = "postgresql://u:p@h:5432/db"
assert ProxyExtrasDBManager._strip_prisma_query_params(url) == url
def test_migration_timestamp_extracts_leading_digits():
assert _migration_timestamp("20260101000000_add_foo") == 20260101000000
assert _migration_timestamp("20250326162113_baseline") == 20250326162113
def test_migration_timestamp_returns_zero_on_malformed():
assert _migration_timestamp("0_init") == 0
assert _migration_timestamp("not_a_migration") == 0
def test_max_migration_timestamp():
names = {"20250326000000_a", "20260415000000_b", "20251115000000_c"}
assert _max_migration_timestamp(names) == 20260415000000
def test_max_migration_timestamp_empty_set():
assert _max_migration_timestamp(set()) == 0
def test_v1_default_still_calls_resolve_all_migrations(monkeypatch, tmp_path):
"""v1 (default) continues to call _resolve_all_migrations on the happy path.
This is the existing buggy behavior — we're not fixing it in v1, only
offering v2 as opt-in. This test pins the default so that a future
inadvertent default flip is caught.
"""
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
(tmp_path / "schema.prisma").write_text("// stub")
# Stub `prisma migrate deploy` to claim success with pending migrations
# applied, which is the code path that triggers the legacy post-migration
# sanity check (a call to _resolve_all_migrations).
class FakeResult:
stdout = "Applied migration.\n"
stderr = ""
def fake_run(cmd, *args, **kwargs):
return FakeResult()
resolve_called = {"n": 0}
def fake_resolve(*args, **kwargs):
resolve_called["n"] += 1
monkeypatch.setattr("subprocess.run", fake_run)
monkeypatch.setattr(ProxyExtrasDBManager, "_resolve_all_migrations", fake_resolve)
ok = ProxyExtrasDBManager.setup_database(use_migrate=True) # v2 flag NOT set
assert ok is True
assert resolve_called["n"] == 1, "v1 default should still invoke the legacy path"
def test_v2_db_push_wraps_subprocess_error_as_runtime_error(monkeypatch, tmp_path):
"""v2: a failing `prisma db push` must raise RuntimeError, not leak
CalledProcessError past proxy_cli.py's `except RuntimeError`."""
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
(tmp_path / "schema.prisma").write_text("// stub")
stderr = "db push error"
with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
with pytest.raises(RuntimeError, match="prisma db push failed"):
ProxyExtrasDBManager.setup_database(use_migrate=False, use_v2_resolver=True)
def test_v2_warn_ahead_of_head_swallows_db_errors(monkeypatch, tmp_path):
"""_warn_if_db_ahead_of_head must never raise — it's informational.
Non-connection DB errors (e.g. InsufficientPrivilege from a user
without SELECT on _prisma_migrations) must be caught, not propagated.
"""
import psycopg
monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x")
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
(tmp_path / "schema.prisma").write_text("// stub")
class _FakeConn:
def __enter__(self):
return self
def __exit__(self, *a):
return False
def execute(self, *a, **kw):
# Simulate an InsufficientPrivilege (subclass of DatabaseError).
raise psycopg.errors.InsufficientPrivilege("permission denied")
def _fake_connect(*a, **kw):
return _FakeConn()
monkeypatch.setattr("psycopg.connect", _fake_connect)
# Must not raise.
ProxyExtrasDBManager._warn_if_db_ahead_of_head(str(tmp_path))
def test_v2_resolve_specific_migration_failure_raises_runtime_error(
monkeypatch, tmp_path
):
"""If marking a migration as applied fails inside P3009 idempotent
recovery, the subprocess error must be re-raised as RuntimeError so
proxy_cli.py catches it cleanly (instead of leaking CalledProcessError)."""
monkeypatch.setattr(
ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
)
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
(tmp_path / "schema.prisma").write_text("// stub")
monkeypatch.setattr(
ProxyExtrasDBManager, "_roll_back_migration", lambda *a, **kw: None
)
# First call: migrate deploy -> P3009 idempotent error.
# Recovery path tries _resolve_specific_migration; that also raises.
def _failing_resolve(*a, **kw):
raise subprocess.CalledProcessError(
returncode=1,
cmd="prisma migrate resolve --applied",
stderr="resolve failed",
output="",
)
monkeypatch.setattr(
ProxyExtrasDBManager, "_resolve_specific_migration", _failing_resolve
)
stderr = (
"Error: P3009\nMigration `20260101000000_some_migration` failed\n"
"relation already exists"
)
with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
with pytest.raises(
RuntimeError, match="Failed to mark migration .* as applied"
):
ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
def test_v2_does_not_call_resolve_all_migrations(monkeypatch, tmp_path):
"""v2 must never call _resolve_all_migrations — that's the bug it fixes."""
monkeypatch.setattr(
ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
)
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
(tmp_path / "schema.prisma").write_text("// stub")
class FakeResult:
stdout = "Applied migration.\n"
stderr = ""
monkeypatch.setattr("subprocess.run", lambda *a, **kw: FakeResult())
resolve_called = {"n": 0}
monkeypatch.setattr(
ProxyExtrasDBManager,
"_resolve_all_migrations",
lambda *a, **kw: resolve_called.__setitem__("n", resolve_called["n"] + 1),
)
ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
assert ok is True
assert resolve_called["n"] == 0, "v2 must not invoke the diff-and-force recovery"

52
litellm/integrations/prometheus.py
View file

@ -59,6 +59,8 @@ from litellm.types.utils import (
if TYPE_CHECKING:
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from prometheus_client.metrics import MetricWrapperBase
from litellm.router import Router
else:
AsyncIOScheduler = Any
@ -67,6 +69,8 @@ _TableRowT: Final = TypeVar("_TableRowT", bound=BaseModel)
_DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT: Final = 5.0
UNRECOGNIZED_REQUESTED_MODEL_LABEL: Final = "other"
_NON_ENUM_METRIC_LABELS: Final[frozenset[str]] = frozenset(
(
"guardrail_name",
@ -154,6 +158,44 @@ def _get_budget_metrics_per_request_timeout() -> float:
return parsed
def _get_proxy_llm_router() -> Router | None:
try:
from litellm.proxy.proxy_server import llm_router
except Exception:
return None
return llm_router
def _bounded_requested_model_label(requested_model: str | None, router_originated: bool = False) -> str | None:
"""
Bound ``requested_model`` label cardinality: names the router recognizes
(model names, deployment ids, aliases, routing groups, team public model
names) or matches via a global or team wildcard/pattern route keep their
own label value; any other client-supplied string collapses into the
single ``other`` bucket. With no proxy router to vouch for the string,
client-supplied values collapse to ``other`` while ``router_originated``
values (emitted by an SDK ``Router``'s own deployment failure and
fallback events, where the proxy router never exists) pass through.
"""
if not requested_model:
return requested_model
llm_router: Final = _get_proxy_llm_router()
if llm_router is None:
return requested_model if router_originated else UNRECOGNIZED_REQUESTED_MODEL_LABEL
if llm_router.is_recognized_model(requested_model):
return requested_model
if requested_model in llm_router.team_public_model_names:
return requested_model
if llm_router.pattern_router.route(requested_model) is not None:
return requested_model
if any(
team_pattern_router.route(requested_model) is not None
for team_pattern_router in llm_router.team_pattern_routers.values()
):
return requested_model
return UNRECOGNIZED_REQUESTED_MODEL_LABEL
class PrometheusLogger(CustomLogger):
# Class variables or attributes
@ -2407,7 +2449,7 @@ class PrometheusLogger(CustomLogger):
team_alias=user_api_key_dict.team_alias,
org_id=user_api_key_dict.org_id,
org_alias=user_api_key_dict.organization_alias,
requested_model=request_data.get("model", ""),
requested_model=_bounded_requested_model_label(request_data.get("model", "")),
status_code=str(status_code),
exception_status=str(status_code),
exception_class=self._get_exception_class_name(original_exception),
@ -2627,7 +2669,9 @@ class PrometheusLogger(CustomLogger):
label_model_id = ""
label_api_base = ""
label_api_provider = ""
label_requested_model = litellm_model_name or model_group or ""
label_requested_model = (
_bounded_requested_model_label(litellm_model_name or model_group, router_originated=True) or ""
)
enum_values: Final = UserAPIKeyLabelValues(
litellm_model_name=label_litellm_model_name,
@ -3186,7 +3230,7 @@ class PrometheusLogger(CustomLogger):
_tags: Final = cast(list[str], kwargs.get("tags") or [])
enum_values: Final = UserAPIKeyLabelValues(
requested_model=original_model_group,
requested_model=_bounded_requested_model_label(original_model_group, router_originated=True),
fallback_model=_new_model,
hashed_api_key=standard_metadata["user_api_key_hash"],
api_key_alias=standard_metadata["user_api_key_alias"],
@ -3227,7 +3271,7 @@ class PrometheusLogger(CustomLogger):
)
enum_values: Final = UserAPIKeyLabelValues(
requested_model=original_model_group,
requested_model=_bounded_requested_model_label(original_model_group, router_originated=True),
fallback_model=_new_model,
hashed_api_key=standard_metadata["user_api_key_hash"],
api_key_alias=standard_metadata["user_api_key_alias"],

2
litellm/llms/anthropic/chat/transformation.py
View file

@ -1516,7 +1516,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
_tool = self.map_response_format_to_anthropic_tool(value, optional_params, is_thinking_enabled)
if _tool is None:
continue
if not is_thinking_enabled:
if not is_thinking_enabled and not AnthropicModelInfo.forced_tool_use_unsupported(model):
_tool_choice = {
"name": RESPONSE_FORMAT_TOOL_NAME,
"type": "tool",

6
litellm/llms/anthropic/common_utils.py
View file

@ -325,13 +325,17 @@ class AnthropicModelInfo(BaseLLMModelInfo):
status_code=400,
)
@staticmethod
def forced_tool_use_unsupported(model: str) -> bool:
return AnthropicModelInfo._get_model_capability(model, "supports_forced_tool_use") is False
@staticmethod
def forced_tool_use_downgraded(model: str, drop_params: bool) -> bool:
"""True when the model map flags the model with
``supports_forced_tool_use: false`` (Fable 5.1 / Mythos 5.1 400 on
``any``/``tool``) and ``drop_params`` asks for the ``auto`` downgrade;
raises a clean client-side 400 for such models without ``drop_params``."""
if AnthropicModelInfo._get_model_capability(model, "supports_forced_tool_use") is not False:
if not AnthropicModelInfo.forced_tool_use_unsupported(model):
return False
if not (litellm.drop_params or drop_params):
raise litellm.utils.UnsupportedParamsError(

1
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -1073,6 +1073,7 @@ class AmazonConverseConfig(BaseConfig):
if (
litellm.utils.supports_tool_choice(model=model, custom_llm_provider=self.custom_llm_provider)
and not is_thinking_enabled
and not AnthropicModelInfo.forced_tool_use_unsupported(model)
):
optional_params["tool_choice"] = ToolChoiceValuesBlock(
tool=SpecificToolChoiceBlock(name=RESPONSE_FORMAT_TOOL_NAME)

22
litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Final
import httpx
from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_anthropic_image_obj,
)
@ -11,6 +12,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
convert_url_to_base64,
)
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
AmazonInvokeConfig,
)
@ -74,10 +76,14 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
drop_params: bool,
) -> dict:
# Force tool-based structured outputs for Bedrock Invoke
# (similar to VertexAI fix in #19201)
# Bedrock Invoke doesn't support output_format parameter
# (similar to VertexAI fix in #19201) unless the model map advertises
# native structured output
from litellm.utils import supports_native_structured_output
original_model: Final = model
if "response_format" in non_default_params:
if "response_format" in non_default_params and not supports_native_structured_output(
model=model, custom_llm_provider="bedrock"
):
# Use a model name that forces tool-based approach
model = "claude-3-sonnet-20240229"
@ -101,6 +107,16 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
# Restore original model name
model = original_model
# The stub model hides the original model from the parent's forced-tool-use backstop
response_format_tool_choice: Final = optional_params.get("tool_choice")
if (
"response_format" in non_default_params
and isinstance(response_format_tool_choice, dict)
and response_format_tool_choice.get("name") == RESPONSE_FORMAT_TOOL_NAME
and AnthropicModelInfo.forced_tool_use_unsupported(original_model)
):
optional_params.pop("tool_choice")
return optional_params
@staticmethod

15
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py
View file

@ -153,14 +153,17 @@ class VertexAIAnthropicConfig(AnthropicConfig):
drop_params: bool,
) -> dict:
"""
Override parent method to ensure VertexAI always uses tool-based structured outputs.
VertexAI doesn't support the output_format parameter, so we force all models
to use the tool-based approach for structured outputs.
Override parent method so VertexAI uses tool-based structured outputs
unless the vertex map entry advertises native structured output
(``output_format``, which Vertex AI Claude forwards for those models).
"""
# Temporarily override model name to force tool-based approach
# This ensures Claude Sonnet 4.5 uses tools instead of output_format
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
original_model: Final = model
if "response_format" in non_default_params:
native_structured_output: Final = AnthropicModelInfo._get_provider_resolved_capability(
model, "supports_native_structured_output", "vertex_ai"
)
if "response_format" in non_default_params and native_structured_output is not True:
model = "claude-3-sonnet-20240229" # Use a model that will use tool-based approach
# Call parent method with potentially modified model name

11
litellm/model_prices_and_context_window_backup.json
View file

@ -1482,7 +1482,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -1557,7 +1557,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -1632,7 +1632,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -1707,7 +1707,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -3254,6 +3254,7 @@
"supports_computer_use": true,
"supports_forced_tool_use": false,
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
@ -44132,6 +44133,7 @@
"supports_computer_use": true,
"supports_forced_tool_use": false,
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
@ -44202,6 +44204,7 @@
"supports_computer_use": true,
"supports_forced_tool_use": false,
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,

48
litellm/proxy/management_endpoints/model_management_endpoints.py
View file

@ -54,7 +54,10 @@ from litellm.proxy.common_utils.config_sync_pubsub import (
coordination_redis_cache,
publish_config_change,
)
from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper
from litellm.proxy.common_utils.encrypt_decrypt_utils import (
decrypt_value_helper,
encrypt_value_helper,
)
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.management_endpoints.common_utils import _is_user_team_admin
from litellm.proxy.management_endpoints.team_endpoints import (
@ -701,6 +704,12 @@ async def patch_model(
param="blocked",
)
ModelManagementAuthChecks.can_user_attach_credential(
litellm_params=patch_data.litellm_params,
user_api_key_dict=user_api_key_dict,
existing_litellm_params=db_model.litellm_params,
)
_raise_on_strategy_router_write_violation(
incoming_params=patch_data.litellm_params,
existing_params=db_model.litellm_params,
@ -1464,6 +1473,32 @@ class ModelManagementAuthChecks:
)
return True
@staticmethod
def can_user_attach_credential(
litellm_params: GenericLiteLLMParams | None,
user_api_key_dict: UserAPIKeyAuth,
existing_litellm_params: GenericLiteLLMParams | None = None,
) -> Literal[True]:
if litellm_params is None or litellm_params.litellm_credential_name is None:
return True
if existing_litellm_params is not None and existing_litellm_params.litellm_credential_name is not None:
existing_credential_name: Final = decrypt_value_helper(
value=existing_litellm_params.litellm_credential_name,
key="litellm_credential_name",
exception_type="debug",
return_original_value=True,
)
if litellm_params.litellm_credential_name == existing_credential_name:
return True
if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN:
return True
raise ProxyException(
message=f"Only a proxy admin can attach a stored credential (litellm_credential_name) to a model. Your role={user_api_key_dict.user_role}.",
type=ProxyErrorTypes.auth_error.value,
code=status.HTTP_403_FORBIDDEN,
param="litellm_credential_name",
)
@staticmethod
async def allow_team_model_action(
model_params: Deployment | updateDeployment,
@ -1786,6 +1821,11 @@ async def add_new_model(
premium_user=premium_user,
)
ModelManagementAuthChecks.can_user_attach_credential(
litellm_params=model_params.litellm_params,
user_api_key_dict=user_api_key_dict,
)
_raise_on_strategy_router_write_violation(
incoming_params=model_params.litellm_params,
existing_params=None,
@ -1958,6 +1998,12 @@ async def update_model(
premium_user=premium_user,
)
ModelManagementAuthChecks.can_user_attach_credential(
litellm_params=model_params.litellm_params,
user_api_key_dict=user_api_key_dict,
existing_litellm_params=deployment.litellm_params,
)
_raise_on_strategy_router_write_violation(
incoming_params=model_params.litellm_params,
existing_params=deployment.litellm_params,

30
litellm/proxy/proxy_cli.py
View file

@ -913,14 +913,13 @@ class ProxyInitializationHelpers:
envvar="ENFORCE_PRISMA_MIGRATION_CHECK",
)
@click.option(
"--use_v2_migration_resolver/--use_legacy_migration_resolver",
default=True,
"--use_v2_migration_resolver",
is_flag=True,
default=False,
help=(
"Which database migration resolver to run at startup. The default v2 "
"resolver avoids the diff-and-force recovery path that can cause schema "
"thrashing during rolling deploys where two LiteLLM versions contend for "
"the same DB. Pass --use_legacy_migration_resolver, or set "
"USE_V2_MIGRATION_RESOLVER=false, to fall back to v1."
"Opt into the v2 migration resolver. Avoids the diff-and-force recovery "
"path that can cause schema thrashing during rolling deploys where two "
"LiteLLM versions contend for the same DB. Default is the v1 resolver."
),
envvar="USE_V2_MIGRATION_RESOLVER",
)
@ -1311,11 +1310,10 @@ def run_server(
else:
if not use_v2_migration_resolver:
print(
"\033[1;33mLiteLLM Proxy: Using the legacy (v1) migration resolver. "
"The default v2 resolver is safer: it avoids the diff-and-force "
"recovery that caused schema thrashing during rolling deploys. "
"Remove --use_legacy_migration_resolver / "
"USE_V2_MIGRATION_RESOLVER=false to switch back to it.\033[0m"
"\033[1;33mLiteLLM Proxy: Using default (v1) migration resolver. "
"If your deployment has seen schema thrashing during rolling "
"deploys, try --use_v2_migration_resolver (safer: avoids the "
"diff-and-force recovery that caused the thrash).\033[0m"
)
try:
setup_ok: Final = PrismaManager.setup_database(
@ -1323,10 +1321,10 @@ def run_server(
use_v2_resolver=use_v2_migration_resolver,
)
except RuntimeError as e:
# Raised on unrecoverable migration errors: permission
# failures from either resolver, the v2 resolver's
# non-idempotent failures, and any `prisma db push`
# against a partitioned LiteLLM_SpendLogs.
# Raised on unrecoverable migration errors: the v2
# resolver's non-idempotent failures and permission
# issues, and any `prisma db push` against a
# partitioned LiteLLM_SpendLogs.
print(
f"\033[1;31mLiteLLM Proxy: Database migration cannot proceed. {e}\033[0m",
file=sys.stderr,

11
model_prices_and_context_window.json
View file

@ -1482,7 +1482,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -1557,7 +1557,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -1632,7 +1632,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -1707,7 +1707,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -3254,6 +3254,7 @@
"supports_computer_use": true,
"supports_forced_tool_use": false,
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
@ -44132,6 +44133,7 @@
"supports_computer_use": true,
"supports_forced_tool_use": false,
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
@ -44202,6 +44204,7 @@
"supports_computer_use": true,
"supports_forced_tool_use": false,
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,

26
tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py
View file

@ -40,6 +40,24 @@ def prometheus_logger() -> PrometheusLogger:
return PrometheusLogger()
@pytest.fixture
def known_model_router():
router = litellm.Router(
model_list=[
{
"model_name": "gpt-5-mini",
"litellm_params": {"model": "openai/gpt-5-mini", "api_key": "fake-key"},
},
{
"model_name": "us/azure/openai/gpt-5-mini",
"litellm_params": {"model": "openai/gpt-5-mini", "api_key": "fake-key"},
},
]
)
with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam
yield router
def create_standard_logging_payload() -> StandardLoggingPayload:
return StandardLoggingPayload(
id="test_id",
@ -741,7 +759,7 @@ async def test_async_log_failure_event(prometheus_logger):
@pytest.mark.asyncio
async def test_async_log_failure_event_litellm_side_rate_limit(prometheus_logger):
async def test_async_log_failure_event_litellm_side_rate_limit(prometheus_logger, known_model_router):
"""LiteLLM-side reject (no deployment picked) routes the requested model
into `requested_model` and skips the partial-outage flag."""
standard_logging_object = create_standard_logging_payload()
@ -786,7 +804,7 @@ async def test_async_log_failure_event_litellm_side_rate_limit(prometheus_logger
@pytest.mark.asyncio
async def test_async_post_call_failure_hook(prometheus_logger):
async def test_async_post_call_failure_hook(prometheus_logger, known_model_router):
"""
Test for the async_post_call_failure_hook method
@ -1069,7 +1087,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger):
@pytest.mark.asyncio
async def test_log_success_fallback_event(prometheus_logger):
async def test_log_success_fallback_event(prometheus_logger, known_model_router):
prometheus_logger.litellm_deployment_successful_fallbacks = MagicMock()
original_model_group = "gpt-5-mini"
@ -1107,7 +1125,7 @@ async def test_log_success_fallback_event(prometheus_logger):
@pytest.mark.asyncio
async def test_log_failure_fallback_event(prometheus_logger):
async def test_log_failure_fallback_event(prometheus_logger, known_model_router):
prometheus_logger.litellm_deployment_failed_fallbacks = MagicMock()
original_model_group = "gpt-5-mini"

571
tests/litellm-proxy-extras/test_setup_database_fail_fast.py
View file

@ -1,571 +0,0 @@
"""Regression tests for ProxyExtrasDBManager's v2 migration resolver.
v2 is the proxy CLI default; v1 stays reachable via the `use_v2_resolver`
kwarg, which still defaults to False for direct callers.
"""
import subprocess
from unittest.mock import patch
import pytest
from litellm_proxy_extras.utils import (
_PRISMA_ATTEMPTS,
ProxyExtrasDBManager,
_max_migration_timestamp,
_migration_timestamp,
)
def _fake_migrate_deploy_failure(returncode: int, stderr: str):
def _run(*args, **kwargs):
raise subprocess.CalledProcessError(
returncode=returncode,
cmd=args[0],
stderr=stderr,
output="",
)
return _run
def test_v2_p3018_permission_error_raises_runtime_error(monkeypatch, tmp_path):
"""v2: a permission failure during migrate deploy raises RuntimeError."""
monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x")
monkeypatch.setattr(
ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
)
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
(tmp_path / "schema.prisma").write_text("// stub")
stderr = (
"Error: P3018\nMigration name: 20250326162113_baseline\n"
"Database error code: 42501\npermission denied for schema public"
)
with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
with pytest.raises(RuntimeError, match="permission"):
ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
def test_v2_non_idempotent_p3009_raises_runtime_error(monkeypatch, tmp_path):
"""v2: a non-idempotent migration failure raises (no silent recovery)."""
monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x")
monkeypatch.setattr(
ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
)
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
(tmp_path / "schema.prisma").write_text("// stub")
stderr = (
"Error: P3009\nMigration `20260101000000_genuinely_broken` failed\n"
'Reason: syntax error at or near "BRKN" LINE 42'
)
with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
def test_strip_prisma_query_params_removes_connection_limit():
"""DATABASE_URLs with Prisma-specific params should be parseable by psycopg."""
url = "postgresql://u:p@h:5432/db?connection_limit=100&pool_timeout=60&sslmode=require"
stripped = ProxyExtrasDBManager._strip_prisma_query_params(url)
assert "connection_limit" not in stripped
assert "pool_timeout" not in stripped
assert "sslmode=require" in stripped
def test_strip_prisma_query_params_passthrough_no_query():
"""URLs without query strings are returned unchanged."""
url = "postgresql://u:p@h:5432/db"
assert ProxyExtrasDBManager._strip_prisma_query_params(url) == url
def test_migration_timestamp_extracts_leading_digits():
assert _migration_timestamp("20260101000000_add_foo") == 20260101000000
assert _migration_timestamp("20250326162113_baseline") == 20250326162113
def test_migration_timestamp_returns_zero_on_malformed():
assert _migration_timestamp("0_init") == 0
assert _migration_timestamp("not_a_migration") == 0
def test_max_migration_timestamp():
names = {"20250326000000_a", "20260415000000_b", "20251115000000_c"}
assert _max_migration_timestamp(names) == 20260415000000
def test_max_migration_timestamp_empty_set():
assert _max_migration_timestamp(set()) == 0
def test_v1_default_still_calls_resolve_all_migrations(monkeypatch, tmp_path):
"""v1 (default) continues to call _resolve_all_migrations on the happy path.
This is the existing buggy behavior — we're not fixing it in v1, only
offering v2 as opt-in. This test pins the default so that a future
inadvertent default flip is caught.
"""
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
(tmp_path / "schema.prisma").write_text("// stub")
# Stub `prisma migrate deploy` to claim success with pending migrations
# applied, which is the code path that triggers the legacy post-migration
# sanity check (a call to _resolve_all_migrations).
class FakeResult:
stdout = "Applied migration.\n"
stderr = ""
def fake_run(cmd, *args, **kwargs):
return FakeResult()
resolve_called = {"n": 0}
def fake_resolve(*args, **kwargs):
resolve_called["n"] += 1
monkeypatch.setattr("subprocess.run", fake_run)
monkeypatch.setattr(ProxyExtrasDBManager, "_resolve_all_migrations", fake_resolve)
ok = ProxyExtrasDBManager.setup_database(use_migrate=True) # v2 flag NOT set
assert ok is True
assert resolve_called["n"] == 1, "v1 default should still invoke the legacy path"
def test_v2_db_push_wraps_subprocess_error_as_runtime_error(monkeypatch, tmp_path):
"""v2: a failing `prisma db push` must raise RuntimeError, not leak
CalledProcessError past proxy_cli.py's `except RuntimeError`."""
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
(tmp_path / "schema.prisma").write_text("// stub")
stderr = "db push error"
with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
with pytest.raises(RuntimeError, match="prisma db push failed"):
ProxyExtrasDBManager.setup_database(use_migrate=False, use_v2_resolver=True)
def test_v2_warn_ahead_of_head_swallows_db_errors(monkeypatch, tmp_path):
"""_warn_if_db_ahead_of_head must never raise — it's informational.
Non-connection DB errors (e.g. InsufficientPrivilege from a user
without SELECT on _prisma_migrations) must be caught, not propagated.
"""
import psycopg
monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x")
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
(tmp_path / "schema.prisma").write_text("// stub")
class _FakeConn:
def __enter__(self):
return self
def __exit__(self, *a):
return False
def execute(self, *a, **kw):
# Simulate an InsufficientPrivilege (subclass of DatabaseError).
raise psycopg.errors.InsufficientPrivilege("permission denied")
connects = {"n": 0}
def _fake_connect(*a, **kw):
connects["n"] += 1
return _FakeConn()
monkeypatch.setattr("psycopg.connect", _fake_connect)
assert ProxyExtrasDBManager._warn_if_db_ahead_of_head(str(tmp_path)) is None
assert connects["n"] == 1, "the failing query must actually have been reached"
def test_v2_resolve_specific_migration_failure_raises_runtime_error(
monkeypatch, tmp_path
):
"""If marking a migration as applied fails inside P3009 idempotent
recovery, the subprocess error must be re-raised as RuntimeError so
proxy_cli.py catches it cleanly (instead of leaking CalledProcessError)."""
monkeypatch.setattr(
ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
)
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
(tmp_path / "schema.prisma").write_text("// stub")
monkeypatch.setattr(
ProxyExtrasDBManager, "_roll_back_migration", lambda *a, **kw: None
)
# First call: migrate deploy -> P3009 idempotent error.
# Recovery path tries _resolve_specific_migration; that also raises.
def _failing_resolve(*a, **kw):
raise subprocess.CalledProcessError(
returncode=1,
cmd="prisma migrate resolve --applied",
stderr="resolve failed",
output="",
)
monkeypatch.setattr(
ProxyExtrasDBManager, "_resolve_specific_migration", _failing_resolve
)
stderr = (
"Error: P3009\nMigration `20260101000000_some_migration` failed\n"
"relation already exists"
)
with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
with pytest.raises(
RuntimeError, match=r"Failed to mark migration .* as applied"
):
ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
def test_v2_does_not_call_resolve_all_migrations(monkeypatch, tmp_path):
"""v2 must never call _resolve_all_migrations — that's the bug it fixes."""
monkeypatch.setattr(
ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
)
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
(tmp_path / "schema.prisma").write_text("// stub")
class FakeResult:
stdout = "Applied migration.\n"
stderr = ""
monkeypatch.setattr("subprocess.run", lambda *a, **kw: FakeResult())
resolve_called = {"n": 0}
monkeypatch.setattr(
ProxyExtrasDBManager,
"_resolve_all_migrations",
lambda *a, **kw: resolve_called.__setitem__("n", resolve_called["n"] + 1),
)
ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
assert ok is True
assert resolve_called["n"] == 0, "v2 must not invoke the diff-and-force recovery"
_DEADLOCK_STDERR = (
"Error: ERROR: deadlock detected\n"
"DETAIL: Process 277 waits for ExclusiveLock on advisory lock "
"[17556,0,72707369,1]; blocked by process 278.\n"
"Process 278 waits for ShareLock on virtual transaction 3/1041; "
"blocked by process 277."
)
class _DeployApplied:
stdout = "All migrations have been successfully applied."
stderr = ""
returncode = 0
def _deploy_only(deploy_side_effect):
"""subprocess.run stand-in that only intercepts `prisma migrate deploy`.
Scoped by argv so the Prisma toolchain check cannot consume the mock first.
"""
deploys = {"n": 0}
def _run(*args, **kwargs):
cmd = args[0] if args else kwargs.get("args", [])
if list(cmd)[-2:] == ["migrate", "deploy"]:
deploys["n"] += 1
return deploy_side_effect(deploys["n"], cmd)
return _DeployApplied()
return _run, deploys
def _prepare_v2_resolver(monkeypatch, tmp_path):
monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x")
monkeypatch.setattr(
ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
)
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
(tmp_path / "schema.prisma").write_text("// stub")
monkeypatch.setattr("time.sleep", lambda *_a, **_k: None)
def test_v2_retries_transient_advisory_lock_deadlock(monkeypatch, tmp_path):
"""v2: replicas racing `migrate deploy` deadlock on Prisma's advisory
lock, which is transient and must be retried rather than kill the boot."""
_prepare_v2_resolver(monkeypatch, tmp_path)
def _side_effect(n, cmd):
if n == 1:
raise subprocess.CalledProcessError(
returncode=1, cmd=cmd, stderr=_DEADLOCK_STDERR, output=""
)
return _DeployApplied()
run, deploys = _deploy_only(_side_effect)
with patch("subprocess.run", side_effect=run):
ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
assert ok is True
assert deploys["n"] == 2, "the deadlocked deploy must be retried, not raised"
def test_v2_persistent_advisory_lock_deadlock_eventually_raises(monkeypatch, tmp_path):
"""v2: the deadlock retry is bounded, so a deadlock that never clears
still raises instead of looping or reporting success."""
_prepare_v2_resolver(monkeypatch, tmp_path)
def _side_effect(n, cmd):
raise subprocess.CalledProcessError(
returncode=1, cmd=cmd, stderr=_DEADLOCK_STDERR, output=""
)
run, deploys = _deploy_only(_side_effect)
with patch("subprocess.run", side_effect=run):
with pytest.raises(RuntimeError, match="after 4 attempts"):
ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
assert deploys["n"] == 4
@pytest.mark.parametrize(
"stderr",
[
"Error: P1001: Can't reach database server at `db`:`5432`",
"Error: P1002: The database server was reached but timed out.",
],
)
def test_v2_retries_transient_database_connectivity_errors(monkeypatch, tmp_path, stderr):
"""v2: a database not accepting connections yet is retried, not fatal."""
_prepare_v2_resolver(monkeypatch, tmp_path)
def _side_effect(n, cmd):
if n == 1:
raise subprocess.CalledProcessError(
returncode=1, cmd=cmd, stderr=stderr, output=""
)
return _DeployApplied()
run, deploys = _deploy_only(_side_effect)
with patch("subprocess.run", side_effect=run):
ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
assert ok is True
assert deploys["n"] == 2, "an unreachable database must be retried, not raised"
def test_v2_unreachable_database_still_fails_after_the_retries(monkeypatch, tmp_path):
"""v2: a genuinely unreachable database still raises once the attempts
are spent, rather than passing as a successful migration."""
_prepare_v2_resolver(monkeypatch, tmp_path)
def _side_effect(n, cmd):
raise subprocess.CalledProcessError(
returncode=1,
cmd=cmd,
stderr="Error: P1001: Can't reach database server at `db`:`5432`",
output="",
)
run, deploys = _deploy_only(_side_effect)
with patch("subprocess.run", side_effect=run):
with pytest.raises(RuntimeError, match="after 4 attempts"):
ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
assert deploys["n"] == 4
def test_v2_exhausted_retries_report_the_prisma_error(monkeypatch, tmp_path, caplog):
"""v2: retrying must not swallow Prisma's stderr, which is captured and is
the only place the cause appears for an operator or a boot-log grep."""
_prepare_v2_resolver(monkeypatch, tmp_path)
stderr = "Error: P1001: Can't reach database server at `wrong`:`5432`"
def _side_effect(n, cmd):
raise subprocess.CalledProcessError(
returncode=1, cmd=cmd, stderr=stderr, output=""
)
run, _ = _deploy_only(_side_effect)
with caplog.at_level("INFO", logger="litellm_proxy_extras"):
with patch("subprocess.run", side_effect=run):
with pytest.raises(RuntimeError) as exc_info:
ProxyExtrasDBManager.setup_database(
use_migrate=True, use_v2_resolver=True
)
assert "P1001" in str(exc_info.value)
assert "P1001" in caplog.text
def test_v2_db_push_retries_transient_failures(monkeypatch, tmp_path):
"""v2: `prisma db push` retries a transient failure like v1 did.
Reached from the migrations Job (USE_PRISMA_DB_PUSH=true), not from the
proxy CLI, whose --use_prisma_db_push has its own loop in prisma_client.
"""
_prepare_v2_resolver(monkeypatch, tmp_path)
pushes = {"n": 0}
def _run(*args, **kwargs):
cmd = list(args[0] if args else kwargs.get("args", []))
if cmd[-3:] != ["db", "push", "--accept-data-loss"]:
return _DeployApplied()
pushes["n"] += 1
if pushes["n"] == 1:
raise subprocess.CalledProcessError(
returncode=1,
cmd=cmd,
stderr="Error: P1001: Can't reach database server at `db`:`5432`",
output="",
)
return _DeployApplied()
monkeypatch.setattr(
ProxyExtrasDBManager, "spend_logs_is_partitioned", lambda: False
)
with patch("subprocess.run", side_effect=_run):
ok = ProxyExtrasDBManager.setup_database(use_migrate=False, use_v2_resolver=True)
assert ok is True
assert pushes["n"] == 2
def test_v2_db_push_retries_are_bounded_and_report_the_prisma_error(
monkeypatch, tmp_path
):
"""v2: a database that never comes back stops after _PRISMA_ATTEMPTS and
surfaces the prisma error, rather than retrying the boot forever."""
_prepare_v2_resolver(monkeypatch, tmp_path)
pushes = {"n": 0}
def _run(*args, **kwargs):
cmd = list(args[0] if args else kwargs.get("args", []))
if cmd[-3:] != ["db", "push", "--accept-data-loss"]:
return _DeployApplied()
pushes["n"] += 1
raise subprocess.CalledProcessError(
returncode=1,
cmd=cmd,
stderr="Error: P1001: Can't reach database server at `db`:`5432`",
output="",
)
monkeypatch.setattr(
ProxyExtrasDBManager, "spend_logs_is_partitioned", lambda: False
)
with patch("subprocess.run", side_effect=_run):
with pytest.raises(RuntimeError) as exc:
ProxyExtrasDBManager.setup_database(
use_migrate=False, use_v2_resolver=True
)
assert pushes["n"] == _PRISMA_ATTEMPTS
assert "P1001" in str(exc.value)
def _db_push_only(push_side_effect):
"""subprocess.run stand-in that only intercepts `prisma db push`."""
pushes = {"n": 0}
def _run(*args, **kwargs):
cmd = list(args[0] if args else kwargs.get("args", []))
if cmd[-3:] != ["db", "push", "--accept-data-loss"]:
return _DeployApplied()
pushes["n"] += 1
return push_side_effect(pushes["n"], cmd)
return _run, pushes
def _timed_out_for_real():
"""Capture what subprocess.run really puts on a TimeoutExpired.
Under text=True it still leaves stderr as bytes, unlike CalledProcessError,
so hardcoding a str here would test a shape production never sees. Derived
at import, before any test patches subprocess.run.
"""
try:
subprocess.run(
["sh", "-c", "echo 'Error: P1001 unreachable' >&2; sleep 5"],
timeout=0.2,
check=True,
capture_output=True,
text=True,
)
except subprocess.TimeoutExpired as e:
return e
raise AssertionError("the helper command was supposed to time out")
_TIMEOUT_TEMPLATE = _timed_out_for_real()
def _real_timeout_expired(cmd):
return subprocess.TimeoutExpired(
cmd=cmd,
timeout=_TIMEOUT_TEMPLATE.timeout,
output=_TIMEOUT_TEMPLATE.stdout,
stderr=_TIMEOUT_TEMPLATE.stderr,
)
def test_v2_db_push_retries_a_timeout(monkeypatch, tmp_path):
"""v2: a `prisma db push` that times out is retried, not turned into a
TypeError by classifying its bytes stderr as if it were text."""
_prepare_v2_resolver(monkeypatch, tmp_path)
def _side_effect(n, cmd):
if n == 1:
raise _real_timeout_expired(cmd)
return _DeployApplied()
monkeypatch.setattr(
ProxyExtrasDBManager, "spend_logs_is_partitioned", lambda: False
)
run, pushes = _db_push_only(_side_effect)
with patch("subprocess.run", side_effect=run):
ok = ProxyExtrasDBManager.setup_database(use_migrate=False, use_v2_resolver=True)
assert ok is True
assert pushes["n"] == 2
def test_v2_db_push_timeouts_are_bounded(monkeypatch, tmp_path):
"""v2: a `prisma db push` that never stops timing out gives up as a
RuntimeError, which is the only exception proxy_cli.py exits cleanly on."""
_prepare_v2_resolver(monkeypatch, tmp_path)
def _side_effect(n, cmd):
raise _real_timeout_expired(cmd)
monkeypatch.setattr(
ProxyExtrasDBManager, "spend_logs_is_partitioned", lambda: False
)
run, pushes = _db_push_only(_side_effect)
with patch("subprocess.run", side_effect=run):
with pytest.raises(RuntimeError, match=r"prisma db push failed after \d+"):
ProxyExtrasDBManager.setup_database(use_migrate=False, use_v2_resolver=True)
assert pushes["n"] == _PRISMA_ATTEMPTS
def test_v2_unclassified_failure_is_not_treated_as_transient(monkeypatch, tmp_path):
"""v2: an unrecognised deploy failure still raises on the first attempt."""
_prepare_v2_resolver(monkeypatch, tmp_path)
def _side_effect(n, cmd):
raise subprocess.CalledProcessError(
returncode=1,
cmd=cmd,
stderr="Error: relation \"LiteLLM_SpendLogs\" does not exist",
output="",
)
run, deploys = _deploy_only(_side_effect)
with patch("subprocess.run", side_effect=run):
with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
assert deploys["n"] == 1

6
tests/llm_translation/conftest.py
View file

@ -44,7 +44,11 @@ _VCR_AUTO_MARKER_SKIP_FILES = frozenset(
{"test_vcr_redis_persister.py", "test_ws_vcr.py"}
)
_VCR_INCOMPATIBLE_NODEID_SUFFIXES: tuple[str, ...] = ()
_VCR_INCOMPATIBLE_NODEID_SUFFIXES: tuple[str, ...] = (
"test_nvidia_nim.py::test_embedding_nvidia_nim",
"test_litellm_proxy_provider.py::test_litellm_gateway_from_sdk_embedding[False]",
"test_litellm_proxy_provider.py::test_litellm_gateway_from_sdk_embedding[True]",
)
_verbose_state = VerboseReporterState()

1
tests/local_testing/conftest.py
View file

@ -90,6 +90,7 @@ _VCR_INCOMPATIBLE_FILES = frozenset(
# carry no real provider cost.
_VCR_INCOMPATIBLE_NODEID_SUFFIXES: tuple[str, ...] = (
"test_router.py::test_router_text_completion_client",
"test_embedding.py::test_encoding_format_omitted_by_default_for_openai_sdk",
)

14
tests/local_testing/test_basic_python_version.py
View file

@ -305,16 +305,14 @@ def _run_proxy_server_smoke_test(extra_proxy_args=None):
def test_litellm_proxy_server_config_no_general_settings():
"""Exercises the default (v2) migration resolver."""
"""Exercises the default (v1) migration resolver."""
_run_proxy_server_smoke_test()
def test_litellm_proxy_server_config_no_general_settings_legacy_resolver():
"""Exercises the legacy (v1) migration resolver against a real database.
def test_litellm_proxy_server_config_no_general_settings_v2_resolver():
"""Exercises the opt-in v2 migration resolver.
v2 is the default, so the no-arg test above already covers it. This one is
the only place the v1 opt-out gets real-DB migration plus proxy-boot
coverage, and it runs in a separate CI job against its own Postgres to
avoid collisions with the default variant.
Runs in a separate CI job against a local Postgres to avoid collisions
with the v1 variant when they share a database.
"""
_run_proxy_server_smoke_test(extra_proxy_args=["--use_legacy_migration_resolver"])
_run_proxy_server_smoke_test(extra_proxy_args=["--use_v2_migration_resolver"])

5
tests/local_testing/test_get_llm_provider.py
View file

@ -155,6 +155,11 @@ def test_default_api_base():
continue
elif provider == "github" and other_provider.value == "azure":
continue
elif (
provider in ("qwencloud", "qwen_ai_platform")
and other_provider.value == "dashscope"
):
continue
assert other_provider.value not in api_base.replace("/openai", "")

279
tests/test_litellm/integrations/test_prometheus_requested_model_cardinality.py Normal file
View file

@ -0,0 +1,279 @@
"""
LIT-6611: every unique client-supplied model name that fails routing used to
mint permanent Prometheus series carrying ``requested_model="<junk>"`` on the
proxy request metrics and the deployment metrics, with no eviction. The fix
collapses any requested model the router does not recognize (and no wildcard
pattern matches) into the single ``other`` label bucket, while recognized
names, aliases, and wildcard-matched names keep their own label values.
"""
import sys
import types
from unittest.mock import patch
import pytest
from prometheus_client import REGISTRY
import litellm
from litellm.integrations.prometheus import (
UNRECOGNIZED_REQUESTED_MODEL_LABEL,
PrometheusLogger,
)
from litellm.proxy._types import UserAPIKeyAuth
class _ClientSideError(Exception):
status_code = 400
@pytest.fixture(autouse=True)
def cleanup_prometheus_registry():
for collector in list(REGISTRY._collector_to_names.keys()):
try:
REGISTRY.unregister(collector)
except Exception:
pass
yield
for collector in list(REGISTRY._collector_to_names.keys()):
try:
REGISTRY.unregister(collector)
except Exception:
pass
@pytest.fixture
def router():
return litellm.Router(
model_list=[
{
"model_name": "gpt-4o-mini",
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "fake-key"},
},
{
"model_name": "openai/*",
"litellm_params": {"model": "openai/*", "api_key": "fake-key"},
},
],
model_group_alias={"gpt4o-alias": "gpt-4o-mini"},
)
@pytest.fixture
def team_router():
return litellm.Router(
model_list=[
{
"model_name": "team-internal-gpt",
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "fake-key"},
"model_info": {"team_id": "team-1", "team_public_model_name": "team-alias-gpt"},
},
{
"model_name": "team-internal-bedrock",
"litellm_params": {"model": "openai/*", "api_key": "fake-key"},
"model_info": {"team_id": "team-1", "team_public_model_name": "team-models/*"},
},
]
)
def _requested_model_values(metric) -> set[str]:
index = metric._labelnames.index("requested_model")
return {sample_key[index] for sample_key in metric._metrics}
def _series_count(metric) -> int:
return len(metric._metrics)
def _total_value(metric) -> float:
return sum(child._value.get() for child in metric._metrics.values())
async def _fire_proxy_failure(logger: PrometheusLogger, model: str) -> None:
await logger.async_post_call_failure_hook(
request_data={"model": model, "metadata": {}, "proxy_server_request": {}},
original_exception=_ClientSideError(f"model {model} does not exist"),
user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key-1"),
)
@pytest.mark.asyncio
async def test_unknown_models_collapse_to_one_series_on_proxy_request_metrics(router):
logger = PrometheusLogger()
with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam
for index in range(25):
await _fire_proxy_failure(logger, f"agent-typo-{index}")
for metric in (
logger.litellm_proxy_failed_requests_metric,
logger.litellm_proxy_total_requests_metric,
):
assert _requested_model_values(metric) == {UNRECOGNIZED_REQUESTED_MODEL_LABEL}
assert _series_count(metric) == 1
assert _total_value(metric) == 25
@pytest.mark.asyncio
async def test_known_alias_and_wildcard_models_keep_their_own_labels(router):
logger = PrometheusLogger()
with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam
await _fire_proxy_failure(logger, "gpt-4o-mini")
await _fire_proxy_failure(logger, "gpt4o-alias")
await _fire_proxy_failure(logger, "openai/gpt-4o-audio-preview")
await _fire_proxy_failure(logger, "agent-typo-hallucinated")
for metric in (
logger.litellm_proxy_failed_requests_metric,
logger.litellm_proxy_total_requests_metric,
):
assert _requested_model_values(metric) == {
"gpt-4o-mini",
"gpt4o-alias",
"openai/gpt-4o-audio-preview",
UNRECOGNIZED_REQUESTED_MODEL_LABEL,
}
@pytest.mark.asyncio
async def test_team_alias_and_team_wildcard_models_keep_their_own_labels(team_router):
logger = PrometheusLogger()
with patch("litellm.proxy.proxy_server.llm_router", team_router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam
await _fire_proxy_failure(logger, "team-alias-gpt")
await _fire_proxy_failure(logger, "team-models/gpt-4o-audio-preview")
await _fire_proxy_failure(logger, "agent-typo-hallucinated")
for metric in (
logger.litellm_proxy_failed_requests_metric,
logger.litellm_proxy_total_requests_metric,
):
assert _requested_model_values(metric) == {
"team-alias-gpt",
"team-models/gpt-4o-audio-preview",
UNRECOGNIZED_REQUESTED_MODEL_LABEL,
}
@pytest.mark.asyncio
async def test_unknown_models_collapse_to_other_when_router_is_unavailable():
logger = PrometheusLogger()
with patch("litellm.proxy.proxy_server.llm_router", None, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam
await _fire_proxy_failure(logger, "agent-typo-no-router")
await _fire_proxy_failure(logger, "gpt-4o-mini")
assert _requested_model_values(logger.litellm_proxy_failed_requests_metric) == {
UNRECOGNIZED_REQUESTED_MODEL_LABEL
}
@pytest.mark.asyncio
async def test_sdk_router_originated_metrics_keep_labels_without_proxy_router():
logger = PrometheusLogger()
with patch("litellm.proxy.proxy_server.llm_router", None, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam
logger.set_llm_deployment_failure_metrics(
request_kwargs={
"model": "sdk-deployment-group",
"litellm_params": {"metadata": {}},
"standard_logging_object": {},
"exception": _ClientSideError("model does not exist"),
}
)
await logger.log_failure_fallback_event(
original_model_group="sdk-fallback-group",
kwargs={"model": "sdk-fallback-group", "metadata": {}},
original_exception=_ClientSideError("upstream unavailable"),
)
assert _requested_model_values(logger.litellm_deployment_failure_responses) == {"sdk-deployment-group"}
assert _requested_model_values(logger.litellm_deployment_failed_fallbacks) == {"sdk-fallback-group"}
@pytest.mark.asyncio
async def test_sdk_fallback_labels_survive_non_import_errors_from_proxy_module(monkeypatch):
logger = PrometheusLogger()
broken_proxy_module = types.ModuleType("litellm.proxy.proxy_server")
def _raise_value_error(_name: str):
raise ValueError("bad proxy env var")
broken_proxy_module.__getattr__ = _raise_value_error # test-quality-ok: reproduces a proxy_server import raising non-ImportError, no injection seam
monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", broken_proxy_module) # test-quality-ok: reproduces a proxy_server import raising non-ImportError, no injection seam
await logger.log_failure_fallback_event(
original_model_group="sdk-fallback-group",
kwargs={"model": "sdk-fallback-group", "metadata": {}},
original_exception=_ClientSideError("upstream unavailable"),
)
assert _requested_model_values(logger.litellm_deployment_failed_fallbacks) == {"sdk-fallback-group"}
def test_unknown_models_collapse_to_one_series_on_deployment_metrics(router):
logger = PrometheusLogger()
with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam
for index in range(25):
logger.set_llm_deployment_failure_metrics(
request_kwargs={
"model": f"agent-typo-{index}",
"litellm_params": {"metadata": {}},
"standard_logging_object": {},
"exception": _ClientSideError("model does not exist"),
}
)
logger.set_llm_deployment_failure_metrics(
request_kwargs={
"model": "gpt-4o-mini",
"litellm_params": {"metadata": {}},
"standard_logging_object": {},
"exception": _ClientSideError("all deployments cooling down"),
}
)
for metric in (
logger.litellm_deployment_failure_responses,
logger.litellm_deployment_total_requests,
):
assert _requested_model_values(metric) == {
UNRECOGNIZED_REQUESTED_MODEL_LABEL,
"gpt-4o-mini",
}
assert _series_count(metric) == 2
assert _total_value(metric) == 26
@pytest.mark.asyncio
async def test_fallback_event_requested_model_is_bounded(router):
logger = PrometheusLogger()
kwargs = {"model": "gpt-4o-mini", "metadata": {}}
with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam
await logger.log_failure_fallback_event(
original_model_group="agent-typo-hallucinated",
kwargs=kwargs,
original_exception=_ClientSideError("model does not exist"),
)
await logger.log_success_fallback_event(
original_model_group="agent-typo-hallucinated",
kwargs=kwargs,
original_exception=_ClientSideError("model does not exist"),
)
await logger.log_failure_fallback_event(
original_model_group="gpt-4o-mini",
kwargs=kwargs,
original_exception=_ClientSideError("upstream unavailable"),
)
assert _requested_model_values(logger.litellm_deployment_failed_fallbacks) == {
UNRECOGNIZED_REQUESTED_MODEL_LABEL,
"gpt-4o-mini",
}
assert _requested_model_values(logger.litellm_deployment_successful_fallbacks) == {
UNRECOGNIZED_REQUESTED_MODEL_LABEL
}

30
tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py
View file

@ -6354,3 +6354,33 @@ def test_anthropic_drop_params_reduces_mixed_output_config_to_format(monkeypatch
)
assert result.get("output_config") == {"format": schema_format}
def test_response_format_tool_path_skips_forced_tool_choice_when_unsupported(local_model_cost_map, monkeypatch):
"""Backstop: on the tool-based structured-output path, a model flagged
``supports_forced_tool_use: false`` must not get the forced response-format
tool_choice the provider would 400 on."""
monkeypatch.setitem(
litellm.model_cost,
"claude-test-no-forced-tools",
{"litellm_provider": "anthropic", "mode": "chat", "supports_forced_tool_use": False},
)
config = AnthropicConfig()
result = config.map_openai_params(
non_default_params={
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "test_schema",
"schema": {"type": "object", "properties": {"result": {"type": "string"}}},
},
}
},
optional_params={},
model="claude-test-no-forced-tools",
drop_params=False,
)
assert "tools" in result
assert "tool_choice" not in result

28
tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py
View file

@ -643,3 +643,31 @@ def test_bedrock_chat_invoke_drop_params_still_inlines_for_non_native(local_mode
assert "output_config" not in result
last_content = result["messages"][-1]["content"]
assert json.loads(last_content[-1]["text"]) == schema
@pytest.mark.parametrize(
"model",
["us.anthropic.claude-fable-5-1", "anthropic.claude-fable-5-1"],
)
def test_bedrock_chat_invoke_fable_5_1_response_format_avoids_forced_tool_choice(local_model_cost_map, model):
"""Regression: Bedrock rejects both native ``output_config.format`` and forced
tool_choice for Fable 5.1, so invoke must use the tool-based path without a
forced ``tool_choice``."""
result = AmazonAnthropicClaudeConfig().map_openai_params(
non_default_params={
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "test_schema",
"schema": {"type": "object", "properties": {"result": {"type": "string"}}},
},
}
},
optional_params={},
model=model,
drop_params=False,
)
assert "output_format" not in result
assert "tools" in result
assert "tool_choice" not in result

33
tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py
View file

@ -6497,6 +6497,39 @@ def test_unforced_tool_choice_unaffected_on_fable_5_1_converse(local_model_cost_
assert result == ({"auto": {}} if tool_choice == "auto" else None)
@pytest.mark.parametrize(
"model",
["anthropic.claude-fable-5-1", "us.anthropic.claude-fable-5-1"],
)
def test_response_format_avoids_native_and_forced_tool_choice_on_fable_5_1_converse(
local_model_cost_map, model
):
"""Regression: Bedrock rejects both ``outputConfig`` structured output and forced
tool_choice for Fable 5.1, so response_format must map to a tool without a forced
tool_choice."""
config = AmazonConverseConfig()
result = config.map_openai_params(
non_default_params={
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "test_schema",
"schema": {"type": "object", "properties": {"result": {"type": "string"}}},
},
}
},
optional_params={},
model=model,
drop_params=False,
)
assert "outputConfig" not in result
assert "tools" in result
assert "tool_choice" not in result
assert result.get("json_mode") is True
def test_forced_tool_choice_forwarded_on_converse_models_that_support_it(
local_model_cost_map, monkeypatch
):

25
tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py
View file

@ -727,3 +727,28 @@ def test_sanitize_strips_effort_for_haiku_45():
data = {"output_config": {"effort": "high"}}
sanitize_vertex_anthropic_output_params(data, "vertex_ai/claude-opus-4-6")
assert data["output_config"] == {"effort": "high"}
def test_vertex_ai_fable_5_1_response_format_uses_native_output_format(local_model_cost_map):
"""Regression: Fable 5.1 rejects forced tool use, so the vertex map entry
advertises native structured output and ``response_format`` must map to
``output_format`` instead of the tool-based path's forced tool_choice."""
config = VertexAIAnthropicConfig()
response_format = {
"type": "json_schema",
"json_schema": {
"name": "test_schema",
"schema": {"type": "object", "properties": {"result": {"type": "string"}}},
},
}
result_params = config.map_openai_params(
non_default_params={"response_format": response_format},
optional_params={},
model="claude-fable-5-1",
drop_params=False,
)
assert "output_format" in result_params
assert "tool_choice" not in result_params
assert "tools" not in result_params

126
tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py
View file

@ -18,6 +18,7 @@ from litellm.proxy._types import (
ReconcileOutcome,
UserAPIKeyAuth,
)
from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper
from litellm.proxy.management_endpoints.model_management_endpoints import (
ModelManagementAuthChecks,
_get_team_deployments,
@ -263,6 +264,131 @@ class TestModelManagementAuthChecks:
)
assert "403" in str(exc_info.value)
def test_can_user_attach_credential_admin_success(self):
result = ModelManagementAuthChecks.can_user_attach_credential(
litellm_params=LiteLLM_Params(model="test_model", litellm_credential_name="shared-credential"),
user_api_key_dict=self.admin_user,
)
assert result is True
def test_can_user_attach_credential_without_credential_allows_any_role(self):
result = ModelManagementAuthChecks.can_user_attach_credential(
litellm_params=LiteLLM_Params(model="test_model"),
user_api_key_dict=self.team_admin_user,
)
assert result is True
def test_can_user_attach_credential_team_admin_fails(self):
with pytest.raises(Exception, match="Only a proxy admin can attach a stored credential") as exc_info:
ModelManagementAuthChecks.can_user_attach_credential(
litellm_params=LiteLLM_Params(model="test_model", litellm_credential_name="shared-credential"),
user_api_key_dict=self.team_admin_user,
)
assert exc_info.value.code == "403"
def test_can_user_attach_credential_unchanged_existing_allows_any_role(self):
result = ModelManagementAuthChecks.can_user_attach_credential(
litellm_params=LiteLLM_Params(model="test_model", litellm_credential_name="shared-credential"),
user_api_key_dict=self.team_admin_user,
existing_litellm_params=LiteLLM_Params(model="test_model", litellm_credential_name="shared-credential"),
)
assert result is True
def test_can_user_attach_credential_unchanged_encrypted_existing_allows_any_role(self, monkeypatch):
monkeypatch.setenv("LITELLM_SALT_KEY", "sk-1234")
encrypted_name = encrypt_value_helper(value="shared-credential")
assert encrypted_name != "shared-credential"
result = ModelManagementAuthChecks.can_user_attach_credential(
litellm_params=LiteLLM_Params(model="test_model", litellm_credential_name="shared-credential"),
user_api_key_dict=self.team_admin_user,
existing_litellm_params=LiteLLM_Params(model="test_model", litellm_credential_name=encrypted_name),
)
assert result is True
@pytest.mark.asyncio
async def test_add_new_model_rejects_credential_attach_for_non_admin(self):
from litellm.proxy._types import ProxyException
from litellm.proxy.management_endpoints.model_management_endpoints import (
add_new_model,
)
mock_prisma = MagicMock()
with (
patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), # test-quality-ok: endpoint reads proxy server globals with no injection seam
patch("litellm.proxy.proxy_server.store_model_in_db", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam
patch("litellm.proxy.proxy_server.premium_user", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam
patch( # test-quality-ok: prior auth check needs a live DB; only the credential check is under test
"litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call",
new=AsyncMock(return_value=None),
),
):
with pytest.raises(ProxyException) as exc_info:
await add_new_model(
model_params=Deployment(
model_name="credential-model",
litellm_params=LiteLLM_Params(
model="openai/gpt-4o", litellm_credential_name="shared-credential"
),
model_info={"id": "credential-create-test"},
),
user_api_key_dict=self.team_admin_user,
)
assert exc_info.value.code == "403"
mock_prisma.db.litellm_proxymodeltable.create.assert_not_called()
@pytest.mark.asyncio
async def test_patch_model_rejects_credential_attach_for_non_admin(self):
from litellm.proxy._types import ProxyException
from litellm.proxy.management_endpoints.model_management_endpoints import (
patch_model,
)
from litellm.types.router import updateLiteLLMParams
model_id = "credential-patch-test"
db_model = Deployment(
model_name="credential-model",
litellm_params=LiteLLM_Params(model="openai/gpt-4o"),
model_info={"id": model_id},
)
with (
patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), # test-quality-ok: endpoint reads proxy server globals with no injection seam
patch("litellm.proxy.proxy_server.llm_router", MagicMock()), # test-quality-ok: endpoint reads proxy server globals with no injection seam
patch("litellm.proxy.proxy_server.store_model_in_db", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam
patch("litellm.proxy.proxy_server.premium_user", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam
patch( # test-quality-ok: stubs the DB row fetch; only the credential check is under test
"litellm.proxy.management_endpoints.model_management_endpoints.get_db_model",
new=AsyncMock(return_value=db_model),
),
patch( # test-quality-ok: prior auth check needs a live DB; only the credential check is under test
"litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call",
new=AsyncMock(return_value=None),
),
patch( # test-quality-ok: asserts the DB write is never reached on rejection
"litellm.proxy.management_endpoints.model_management_endpoints._update_team_model_in_db",
new=AsyncMock(),
) as mock_update,
):
with pytest.raises(ProxyException) as exc_info:
await patch_model(
model_id=model_id,
patch_data=updateDeployment(
litellm_params=updateLiteLLMParams(
model="openai/gpt-4o", litellm_credential_name="shared-credential"
)
),
user_api_key_dict=self.team_admin_user,
)
assert exc_info.value.code == "403"
mock_update.assert_not_awaited()
def test_can_user_attach_credential_internal_user_fails(self):
with pytest.raises(Exception, match="Only a proxy admin can attach a stored credential") as exc_info:
ModelManagementAuthChecks.can_user_attach_credential(
litellm_params=LiteLLM_Params(model="test_model", litellm_credential_name="shared-credential"),
user_api_key_dict=self.normal_user,
)
assert exc_info.value.code == "403"
class MockModelTable:
def __init__(self, model_aliases: Dict[str, str], include: Optional[dict] = None):

39
tests/test_litellm/proxy/test_proxy_cli.py
View file

@ -1737,8 +1737,7 @@ class TestRunServerDbSetup:
mock_atexit_register,
mock_subprocess_run,
):
"""Which resolver and which migration mode run_server hands setup_database,
across the db push flag, the v2/legacy flag pair and USE_V2_MIGRATION_RESOLVER."""
"""Test that use_prisma_db_push flag correctly controls PrismaManager.setup_database use_migrate parameter"""
from litellm.proxy.proxy_cli import run_server
# Mock subprocess.run to simulate prisma being available
@ -1788,7 +1787,7 @@ class TestRunServerDbSetup:
# use_prisma_db_push should be False (default), so use_migrate should be True
run_server.main(["--local", "--skip_server_startup"], standalone_mode=False)
mock_setup_database.assert_called_with(
use_migrate=True, use_v2_resolver=True
use_migrate=True, use_v2_resolver=False
)
# Reset mocks
@ -1803,38 +1802,9 @@ class TestRunServerDbSetup:
standalone_mode=False,
)
mock_setup_database.assert_called_with(
use_migrate=False, use_v2_resolver=True
use_migrate=False, use_v2_resolver=False
)
for argv, env_value, expected_v2 in (
([], None, True),
(["--use_v2_migration_resolver"], None, True),
(["--use_legacy_migration_resolver"], None, False),
([], "false", False),
([], "true", True),
(["--use_v2_migration_resolver"], "false", True),
(["--use_legacy_migration_resolver"], "true", False),
):
mock_setup_database.reset_mock()
mock_should_update_schema.reset_mock()
mock_should_update_schema.return_value = True
resolver_env = (
{"USE_V2_MIGRATION_RESOLVER": env_value}
if env_value is not None
else {}
)
os.environ.pop("USE_V2_MIGRATION_RESOLVER", None)
with patch.dict(os.environ, resolver_env):
run_server.main(
["--local", "--skip_server_startup", *argv],
standalone_mode=False,
)
assert mock_setup_database.call_args.kwargs == {
"use_migrate": True,
"use_v2_resolver": expected_v2,
}, f"argv={argv} env={env_value}"
@patch("subprocess.run")
@patch("atexit.register")
@patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database")
@ -1899,7 +1869,7 @@ class TestRunServerDbSetup:
)
assert exc_info.value.code == 1
mock_setup_database.assert_called_once_with(
use_migrate=True, use_v2_resolver=True
use_migrate=True, use_v2_resolver=False
)
@patch("subprocess.run")
@ -2011,6 +1981,7 @@ class TestRunServerDbSetup:
use_migrate=True, use_v2_resolver=True
)
# --- Module-level helpers for worker startup hook tests ---
_dummy_hook_called = False

36
ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsTable.test.tsx
View file

@ -74,6 +74,42 @@ describe("AgentsTable", () => {
expect(onDeleteClick).toHaveBeenCalledWith("agent-9", "Doomed Agent");
});
it("filters agents by name or by agent card description", async () => {
const user = userEvent.setup();
render(
<AgentsTable
agents={[
makeAgent({ agent_id: "a1", agent_name: "Billing Router" }),
makeAgent({
agent_id: "a2",
agent_name: "Second Agent",
agent_card_params: { description: "handles support tickets" },
}),
]}
{...baseProps}
/>,
);
const search = screen.getByPlaceholderText("Search agent names or descriptions...");
await user.type(search, "billing");
expect(screen.getByText("Billing Router")).toBeInTheDocument();
expect(screen.queryByText("Second Agent")).not.toBeInTheDocument();
await user.clear(search);
await user.type(search, "support tickets");
expect(screen.getByText("Second Agent")).toBeInTheDocument();
expect(screen.queryByText("Billing Router")).not.toBeInTheDocument();
});
it("shows the no-match empty state when the search matches nothing", async () => {
const user = userEvent.setup();
render(<AgentsTable agents={[makeAgent()]} {...baseProps} />);
await user.type(screen.getByPlaceholderText("Search agent names or descriptions..."), "zzzz");
expect(screen.queryByText("Test Agent")).not.toBeInTheDocument();
expect(screen.getByText("No matching agents")).toBeInTheDocument();
});
it("hides the actions column entirely for non-admins", () => {
const agent = makeAgent({ agent_id: "agent-2" });
render(<AgentsTable agents={[agent]} {...baseProps} isAdmin={false} />);

42
ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsTable.tsx
View file

@ -1,13 +1,15 @@
"use client";
import { SortingState } from "@tanstack/react-table";
import { Bot, CircleCheck } from "lucide-react";
import { Bot, CircleCheck, Search as SearchIcon, X } from "lucide-react";
import React, { useMemo, useState } from "react";
import { Agent } from "@/components/agents/types";
import { DataTable } from "@/components/shared/DataTable";
import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from "@/components/ui/input-group";
import { Switch } from "@/components/ui/switch";
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
import { filterBySearchTerm } from "@/utils/searchUtils";
import { getAgentsTableColumns } from "./AgentsTableColumns";
@ -24,14 +26,18 @@ interface AgentsTableProps {
const DEFAULT_SORTING: SortingState = [{ id: "created_at", desc: true }];
function EmptyState() {
function EmptyState({ isFiltered }: { isFiltered: boolean }) {
return (
<div className="flex flex-col items-center gap-1 py-6">
<div className="mb-1 flex size-10 items-center justify-center rounded-lg bg-muted">
<Bot className="size-5 text-muted-foreground" />
</div>
<div className="text-sm font-medium text-foreground">No agents yet</div>
<div className="text-sm text-muted-foreground">Add an agent to make it available in your organization.</div>
<div className="text-sm font-medium text-foreground">{isFiltered ? "No matching agents" : "No agents yet"}</div>
<div className="text-sm text-muted-foreground">
{isFiltered
? "Adjust the search to see more agents."
: "Add an agent to make it available in your organization."}
</div>
</div>
);
}
@ -47,6 +53,11 @@ const AgentsTable: React.FC<AgentsTableProps> = ({
onDeleteClick,
}) => {
const [sorting, setSorting] = useState<SortingState>(DEFAULT_SORTING);
const [searchTerm, setSearchTerm] = useState("");
const filteredAgents = useMemo(
() => filterBySearchTerm(agents, searchTerm, (agent) => [agent.agent_name, agent.agent_card_params?.description]),
[agents, searchTerm],
);
const columns = useMemo(
() => getAgentsTableColumns({ isAdmin, onAgentClick, onDeleteClick }),
@ -55,7 +66,7 @@ const AgentsTable: React.FC<AgentsTableProps> = ({
return (
<DataTable
data={agents}
data={filteredAgents}
columns={columns}
getRowId={(agent, index) => agent.agent_id || String(index)}
sortingMode="client"
@ -63,10 +74,27 @@ const AgentsTable: React.FC<AgentsTableProps> = ({
onSortingChange={setSorting}
isLoading={isLoading}
loadingMessage="Loading agents…"
noDataMessage={<EmptyState />}
noDataMessage={<EmptyState isFiltered={agents.length > 0} />}
size="compact"
toolbar={() => (
<div className="flex items-center justify-end">
<div className="flex flex-wrap items-center justify-between gap-2">
<InputGroup className="max-w-sm">
<InputGroupAddon>
<SearchIcon className="size-4 text-muted-foreground" />
</InputGroupAddon>
<InputGroupInput
placeholder="Search agent names or descriptions..."
value={searchTerm}
onChange={(e) => setSearchTerm(e.target.value)}
/>
{searchTerm && (
<InputGroupAddon align="inline-end">
<InputGroupButton size="icon-xs" aria-label="Clear search" onClick={() => setSearchTerm("")}>
<X />
</InputGroupButton>
</InputGroupAddon>
)}
</InputGroup>
<TooltipProvider delay={300}>
<Tooltip>
<TooltipTrigger

1
ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx
View file

@ -87,7 +87,6 @@ const alwaysMounted = {
api_key: undefined,
api_base: undefined,
custom_llm_provider: "openai",
litellm_credential_name: null,
model: "gpt-4o",
};

35
ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx
View file

@ -203,12 +203,12 @@ describe("ModelHubTable", () => {
});
describe("hub tabs", () => {
const renderHub = async () => {
const renderHub = async (agents: object[] = []) => {
vi.mocked(networking.modelHubCall).mockResolvedValue({
data: [{ model_group: "claude-opus-4-8", providers: ["anthropic"], mode: "chat" }],
});
vi.mocked(networking.getConfigFieldSetting).mockResolvedValue({ field_value: false });
vi.mocked(networking.getAgentsList).mockResolvedValue({ agents: [] });
vi.mocked(networking.getAgentsList).mockResolvedValue({ agents });
vi.mocked(networking.fetchMCPServers).mockResolvedValue([]);
vi.mocked(networking.getUiSettings).mockResolvedValue({ values: {} });
mockUseUISettings.mockReturnValue({ data: { values: {} }, isLoading: false });
@ -230,6 +230,37 @@ describe("ModelHubTable", () => {
expect(await screen.findByPlaceholderText("Search model names...")).toHaveValue("opus");
});
it("filters the Agent Hub table by name or description and shows the no-match state", async () => {
const { user } = await renderHub([
{
agent_id: "a1",
agent_card_params: { name: "Billing Router", description: "routes billing questions" },
litellm_params: { is_public: false },
},
{
agent_id: "a2",
agent_card_params: { name: "Support Bot", description: "handles support tickets" },
litellm_params: { is_public: false },
},
]);
const agentCount = (expected: string) =>
screen.getByText((_, el) => el?.tagName === "P" && el.textContent === expected);
await user.click(screen.getByRole("tab", { name: "Agent Hub" }));
expect(await screen.findByText("Billing Router")).toBeInTheDocument();
const search = screen.getByPlaceholderText("Search agent names or descriptions...");
await user.type(search, "support tickets");
expect(screen.queryByText("Billing Router")).not.toBeInTheDocument();
expect(screen.getByText("Support Bot")).toBeInTheDocument();
expect(agentCount("Showing 1 of 2 agents")).toBeInTheDocument();
await user.clear(search);
await user.type(search, "zzzz");
expect(screen.getByText("No matching agents")).toBeInTheDocument();
expect(agentCount("Showing 0 of 2 agents")).toBeInTheDocument();
});
it("renders the hub strip as underlined tabs rather than a segmented pill", async () => {
await renderHub();

47
ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx
View file

@ -23,13 +23,15 @@ import {
import PublicModelHub from "@/components/public_model_hub";
import { copyToClipboard } from "@/utils/dataUtils";
import { isAdminRole, isProxyAdminRole } from "@/utils/roles";
import { filterBySearchTerm } from "@/utils/searchUtils";
import { SortingState } from "@tanstack/react-table";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { Card } from "@/components/ui/card";
import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog";
import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from "@/components/ui/input-group";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { Copy, Inbox } from "lucide-react";
import { Copy, Inbox, Search as SearchIcon, X } from "lucide-react";
import { useRouter } from "next/navigation";
import React, { useCallback, useEffect, useMemo, useState } from "react";
import { Prism as SyntaxHighlighter } from "react-syntax-highlighter";
@ -80,6 +82,7 @@ const ModelHubTable: React.FC<ModelHubTableProps> = ({ accessToken, publicPage,
const [agentLoading, setAgentLoading] = useState<boolean>(true);
const [selectedAgent, setSelectedAgent] = useState<null | AgentHubData>(null);
const [isAgentModalVisible, setIsAgentModalVisible] = useState(false);
const [agentSearchTerm, setAgentSearchTerm] = useState("");
// MCP Hub state
const [mcpHubData, setMcpHubData] = useState<MCPServerData[] | null>(null);
const [mcpLoading, setMcpLoading] = useState<boolean>(true);
@ -385,6 +388,10 @@ const ModelHubTable: React.FC<ModelHubTableProps> = ({ accessToken, publicPage,
const modelColumns = useMemo(() => getModelHubTableColumns({ onModelClick: showModal }), [showModal]);
const agentColumns = useMemo(() => getAgentHubTableColumns({ onAgentClick: showAgentModal }), [showAgentModal]);
const filteredAgentData = useMemo(
() => filterBySearchTerm(agentHubData ?? [], agentSearchTerm, (agent) => [agent.name, agent.description]),
[agentHubData, agentSearchTerm],
);
const mcpColumns = useMemo(() => getMCPHubTableColumns({ onServerClick: showMcpModal }), [showMcpModal]);
// If this is a public page, use the dedicated PublicModelHub component
@ -505,9 +512,34 @@ const ModelHubTable: React.FC<ModelHubTableProps> = ({ accessToken, publicPage,
</div>
)}
<div className="mb-4">
<p className="text-sm font-medium mb-2">Search Agents:</p>
<InputGroup className="max-w-sm">
<InputGroupAddon>
<SearchIcon className="size-4 text-muted-foreground" />
</InputGroupAddon>
<InputGroupInput
placeholder="Search agent names or descriptions..."
value={agentSearchTerm}
onChange={(e) => setAgentSearchTerm(e.target.value)}
/>
{agentSearchTerm && (
<InputGroupAddon align="inline-end">
<InputGroupButton
size="icon-xs"
aria-label="Clear search"
onClick={() => setAgentSearchTerm("")}
>
<X />
</InputGroupButton>
</InputGroupAddon>
)}
</InputGroup>
</div>
{/* Agent Table */}
<DataTable
data={agentHubData || []}
data={filteredAgentData}
columns={agentColumns}
getRowId={(agent, index) => agent.agent_id || agent.name || String(index)}
sortingMode="client"
@ -516,7 +548,14 @@ const ModelHubTable: React.FC<ModelHubTableProps> = ({ accessToken, publicPage,
isLoading={agentLoading}
loadingMessage="Loading agents…"
noDataMessage={
<HubEmptyState title="No agents yet" body="Agents added to this proxy will appear here." />
<HubEmptyState
title={agentHubData?.length ? "No matching agents" : "No agents yet"}
body={
agentHubData?.length
? "Adjust the search to see more agents."
: "Agents added to this proxy will appear here."
}
/>
}
size="compact"
/>
@ -524,7 +563,7 @@ const ModelHubTable: React.FC<ModelHubTableProps> = ({ accessToken, publicPage,
<div className="mt-4 text-center space-y-2">
<p className="text-sm text-muted-foreground">
Showing {agentHubData?.length || 0} agent{agentHubData?.length !== 1 ? "s" : ""}
Showing {filteredAgentData.length} of {agentHubData?.length || 0} agents
</p>
</div>
</TabsContent>

30
ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.test.tsx
View file

@ -1,6 +1,10 @@
import { describe, expect, it } from "vitest";
import { describe, expect, it, vi } from "vitest";
import { prepareModelAddRequest } from "./handle_add_model_submit";
vi.mock("../networking", () => ({
modelCreateCall: vi.fn(),
}));
describe("prepareModelAddRequest", () => {
it("returns deployment data for the most basic form", async () => {
const formValues = {
@ -73,4 +77,28 @@ describe("prepareModelAddRequest", () => {
expect(deployment.litellmParamsObj.litellm_credential_name).toBe("selected-credential");
expect(deployment.litellmParamsObj.timeout).toBe(5);
});
it("keeps litellm_credential_name from LiteLLM Params JSON when no credential is selected", async () => {
const formValues = {
model_mappings: [
{
public_name: "Public Model",
litellm_model: "litellm/public",
},
],
model_name: "custom-model-name",
litellm_extra_params: JSON.stringify({
litellm_credential_name: "from-json",
timeout: 5,
}),
litellm_credential_name: null,
};
const deployments = await prepareModelAddRequest({ ...formValues }, "token", null);
expect(deployments).toHaveLength(1);
const [deployment] = deployments!;
expect(deployment.litellmParamsObj.litellm_credential_name).toBe("from-json");
expect(deployment.litellmParamsObj.timeout).toBe(5);
});
});

9
ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx
View file

@ -91,6 +91,9 @@ export const prepareModelAddRequest = async (formValues: Record<string, any>, ac
if (value === "") {
continue;
}
if (key === "litellm_credential_name" && value == null) {
continue;
}
// Skip the custom_pricing and pricing_model fields as they're only used for UI control
if (key === "custom_pricing" || key === "pricing_model" || key === "cache_control") {
continue;
@ -124,13 +127,13 @@ export const prepareModelAddRequest = async (formValues: Record<string, any>, ac
if (value && value != undefined) {
try {
litellmExtraParams = JSON.parse(value);
if ("litellm_credential_name" in litellmExtraParams) {
delete litellmExtraParams.litellm_credential_name;
}
} catch (error) {
toast.fromError("Failed to parse LiteLLM Extra Params: " + error);
throw new Error("Failed to parse litellm_extra_params: " + error);
}
if ("litellm_credential_name" in litellmExtraParams && formValues.litellm_credential_name) {
delete litellmExtraParams.litellm_credential_name;
}
for (const [key, value] of Object.entries(litellmExtraParams)) {
litellmParamsObj[key] = value;
}

3
ui/litellm-dashboard/src/components/model_filters.tsx
View file

@ -1,5 +1,6 @@
import React, { useState, useEffect, useMemo, useRef } from "react";
import { Card } from "@/components/ui/card";
import { matchesSearchTerm } from "@/utils/searchUtils";
interface ModelGroupInfo {
model_group: string;
@ -76,7 +77,7 @@ const ModelFilters: React.FC<ModelFiltersProps> = ({
const filteredData = useMemo(() => {
return (
modelHubData?.filter((model) => {
const matchesSearch = model.model_group.toLowerCase().includes(searchTerm.toLowerCase());
const matchesSearch = matchesSearchTerm(searchTerm, [model.model_group]);
const matchesProvider = selectedProvider === "" || model.providers.includes(selectedProvider);
const matchesMode = selectedMode === "" || model.mode === selectedMode;

21
ui/litellm-dashboard/src/components/public_model_hub.test.tsx
View file

@ -134,6 +134,27 @@ describe("PublicModelHub", () => {
expect(within(gpt35Row as HTMLElement).getByText("Unknown")).toBeInTheDocument();
});
});
it("shows no models when the search has no matches (LIT-5230 regression)", async () => {
const networkingModule = await import("./networking");
vi.mocked(networkingModule.modelHubPublicModelsCall).mockResolvedValue([
{ model_group: "gpt-4", providers: ["openai"], mode: "chat" },
{ model_group: "claude-3", providers: ["anthropic"], mode: "chat" },
]);
render(<PublicModelHub />);
expect(await screen.findByText("gpt-4")).toBeInTheDocument();
fireEvent.change(screen.getByPlaceholderText("Search model names... (smart search enabled)"), {
target: { value: "zzzz" },
});
await waitFor(() => {
expect(screen.queryByText("gpt-4")).not.toBeInTheDocument();
expect(screen.queryByText("claude-3")).not.toBeInTheDocument();
expect(screen.getByText("No matching models")).toBeInTheDocument();
});
});
it("handles non-array response gracefully (regression test for e.filter crash)", async () => {
const networkingModule = await import("./networking");
// Mock the API to return an object (like an error response) instead of an array

136
ui/litellm-dashboard/src/components/public_model_hub.tsx
View file

@ -47,6 +47,7 @@ import { generateCodeSnippet } from "@/components/chat_ui/CodeSnippets";
import { getEndpointType } from "@/components/chat_ui/mode_endpoint_mapping";
import { MessageType } from "@/components/chat_ui/types";
import { getProviderLogoAndName } from "./provider_info_helpers";
import { filterBySearchTerm, rankBySearchRelevance } from "@/utils/searchUtils";
interface PublicModelHubProps {
accessToken?: string | null;
@ -236,52 +237,11 @@ const PublicModelHub: React.FC<PublicModelHubProps> = ({ accessToken, isEmbedded
const filteredData = useMemo(() => {
if (!modelHubData || !Array.isArray(modelHubData)) return [];
let searchResults = modelHubData;
// Apply search if there's a search term
if (searchTerm.trim()) {
const lowercaseSearch = searchTerm.toLowerCase();
const searchWords = lowercaseSearch.split(/\s+/);
// First, try flexible matching that handles different separators
const exactMatches = modelHubData.filter((model) => {
const modelName = model.model_group.toLowerCase();
// Check if it contains the exact search term
if (modelName.includes(lowercaseSearch)) {
return true;
}
// Check if it contains all search words (handles spaces vs slashes/dashes)
return searchWords.every((word) => modelName.includes(word));
});
// If we have exact matches, rank them by relevance
if (exactMatches.length > 0) {
searchResults = exactMatches.sort((a, b) => {
const aName = a.model_group.toLowerCase();
const bName = b.model_group.toLowerCase();
// Calculate relevance scores
const aExactMatch = aName === lowercaseSearch ? 1000 : 0;
const bExactMatch = bName === lowercaseSearch ? 1000 : 0;
const aStartsWith = aName.startsWith(lowercaseSearch) ? 100 : 0;
const bStartsWith = bName.startsWith(lowercaseSearch) ? 100 : 0;
const aContainsWords = lowercaseSearch.split(/\s+/).every((word) => aName.includes(word)) ? 50 : 0;
const bContainsWords = lowercaseSearch.split(/\s+/).every((word) => bName.includes(word)) ? 50 : 0;
const aLength = aName.length;
const bLength = bName.length;
const aScore = aExactMatch + aStartsWith + aContainsWords + (1000 - aLength);
const bScore = bExactMatch + bStartsWith + bContainsWords + (1000 - bLength);
return bScore - aScore; // Higher score first
});
}
}
const searchResults = rankBySearchRelevance(
filterBySearchTerm(modelHubData, searchTerm, (model) => [model.model_group]),
searchTerm,
(model) => model.model_group,
);
// Apply other filters
return searchResults.filter((model) => {
@ -310,43 +270,11 @@ const PublicModelHub: React.FC<PublicModelHubProps> = ({ accessToken, isEmbedded
const filteredAgentData = useMemo(() => {
if (!agentHubData || !Array.isArray(agentHubData)) return [];
let searchResults = agentHubData;
// Apply search if there's a search term
if (agentSearchTerm.trim()) {
const lowercaseSearch = agentSearchTerm.toLowerCase();
const searchWords = lowercaseSearch.split(/\s+/);
searchResults = agentHubData.filter((agent) => {
const agentName = agent.name.toLowerCase();
const agentDescription = agent.description.toLowerCase();
// Check if it contains the exact search term
if (agentName.includes(lowercaseSearch) || agentDescription.includes(lowercaseSearch)) {
return true;
}
// Check if it contains all search words
return searchWords.every((word) => agentName.includes(word) || agentDescription.includes(word));
});
// Sort by relevance
searchResults = searchResults.sort((a, b) => {
const aName = a.name.toLowerCase();
const bName = b.name.toLowerCase();
const aExactMatch = aName === lowercaseSearch ? 1000 : 0;
const bExactMatch = bName === lowercaseSearch ? 1000 : 0;
const aStartsWith = aName.startsWith(lowercaseSearch) ? 100 : 0;
const bStartsWith = bName.startsWith(lowercaseSearch) ? 100 : 0;
const aScore = aExactMatch + aStartsWith + (1000 - aName.length);
const bScore = bExactMatch + bStartsWith + (1000 - bName.length);
return bScore - aScore;
});
}
const searchResults = rankBySearchRelevance(
filterBySearchTerm(agentHubData, agentSearchTerm, (agent) => [agent.name, agent.description]),
agentSearchTerm,
(agent) => agent.name,
);
// Apply skill filters
return searchResults.filter((agent) => {
@ -361,43 +289,11 @@ const PublicModelHub: React.FC<PublicModelHubProps> = ({ accessToken, isEmbedded
const filteredMcpData = useMemo(() => {
if (!mcpHubData || !Array.isArray(mcpHubData)) return [];
let searchResults = mcpHubData;
// Apply search if there's a search term
if (mcpSearchTerm.trim()) {
const lowercaseSearch = mcpSearchTerm.toLowerCase();
const searchWords = lowercaseSearch.split(/\s+/);
searchResults = mcpHubData.filter((server) => {
const serverName = server.server_name.toLowerCase();
const serverDescription = (server.mcp_info?.description || "").toLowerCase();
// Check if it contains the exact search term
if (serverName.includes(lowercaseSearch) || serverDescription.includes(lowercaseSearch)) {
return true;
}
// Check if it contains all search words
return searchWords.every((word) => serverName.includes(word) || serverDescription.includes(word));
});
// Sort by relevance
searchResults = searchResults.sort((a, b) => {
const aName = a.server_name.toLowerCase();
const bName = b.server_name.toLowerCase();
const aExactMatch = aName === lowercaseSearch ? 1000 : 0;
const bExactMatch = bName === lowercaseSearch ? 1000 : 0;
const aStartsWith = aName.startsWith(lowercaseSearch) ? 100 : 0;
const bStartsWith = bName.startsWith(lowercaseSearch) ? 100 : 0;
const aScore = aExactMatch + aStartsWith + (1000 - aName.length);
const bScore = bExactMatch + bStartsWith + (1000 - bName.length);
return bScore - aScore;
});
}
const searchResults = rankBySearchRelevance(
filterBySearchTerm(mcpHubData, mcpSearchTerm, (server) => [server.server_name, server.mcp_info?.description]),
mcpSearchTerm,
(server) => server.server_name,
);
// Apply transport filters
return searchResults.filter((server) => {

53
ui/litellm-dashboard/src/components/view_logs/ToolsSection/FormattedToolView.tsx
View file

@ -25,31 +25,15 @@ export function FormattedToolView({ tool }: FormattedToolViewProps) {
<div>
{/* Description */}
{tool.description && (
<div style={{ marginBottom: 16 }}>
<span
style={{
lineHeight: 1.6,
whiteSpace: "pre-wrap",
}}
>
{tool.description}
</span>
<div className="mb-4">
<span className="whitespace-pre-wrap leading-relaxed">{tool.description}</span>
</div>
)}
{/* Parameters Table */}
{parameterRows.length > 0 && (
<div>
<span
className="text-muted-foreground"
style={{
fontSize: 12,
display: "block",
marginBottom: 8,
}}
>
Parameters
</span>
<span className="mb-2 block text-xs text-muted-foreground">Parameters</span>
<Table>
<TableHeader>
<TableRow>
@ -82,33 +66,10 @@ export function FormattedToolView({ tool }: FormattedToolViewProps) {
{/* If tool was called, show the arguments used */}
{tool.called && tool.callData && (
<div style={{ marginTop: 16 }}>
<span
className="text-muted-foreground"
style={{
fontSize: 12,
display: "block",
marginBottom: 8,
}}
>
Called With
</span>
<div
style={{
background: "#f6ffed",
border: "1px solid #b7eb8f",
borderRadius: 4,
padding: 12,
}}
>
<pre
style={{
margin: 0,
fontSize: 12,
whiteSpace: "pre-wrap",
wordBreak: "break-word",
}}
>
<div className="mt-4">
<span className="mb-2 block text-xs text-muted-foreground">Called With</span>
<div className="rounded border border-success/30 bg-success/10 p-3">
<pre className="m-0 whitespace-pre-wrap break-words text-xs text-foreground">
{JSON.stringify(tool.callData.arguments, null, 2)}
</pre>
</div>

14
ui/litellm-dashboard/src/components/view_logs/ToolsSection/JsonToolView.tsx
View file

@ -20,19 +20,7 @@ export function JsonToolView({ tool }: JsonToolViewProps) {
};
return (
<pre
style={{
margin: 0,
whiteSpace: "pre-wrap",
wordBreak: "break-word",
fontSize: 12,
background: "#fafafa",
padding: 12,
borderRadius: 4,
maxHeight: 300,
overflow: "auto",
}}
>
<pre className="m-0 max-h-[300px] overflow-auto whitespace-pre-wrap break-words rounded bg-muted p-3 text-xs text-foreground">
{JSON.stringify(toolJson, null, 2)}
</pre>
);

36
ui/litellm-dashboard/src/components/view_logs/ToolsSection/ToolItem.tsx
View file

@ -5,6 +5,7 @@
import { useState } from "react";
import { ChevronDown, ChevronRight, Wrench } from "lucide-react";
import { Badge } from "@/components/ui/badge";
import { cn } from "@/lib/cva.config";
import { ParsedTool } from "./types";
import { ToolExpandedContent } from "./ToolExpandedContent";
@ -16,34 +17,23 @@ export function ToolItem({ tool }: ToolItemProps) {
const [expanded, setExpanded] = useState(false);
return (
<div
style={{
border: "1px solid #f0f0f0",
borderRadius: 8,
overflow: "hidden",
}}
>
<div className="overflow-hidden rounded-lg border border-border">
{/* Header Row - Always Visible */}
<div
onClick={() => setExpanded(!expanded)}
style={{
display: "flex",
alignItems: "center",
justifyContent: "space-between",
padding: "12px 16px",
cursor: "pointer",
background: expanded ? "#fafafa" : "#fff",
transition: "background 0.2s",
}}
className={cn(
"flex cursor-pointer items-center justify-between gap-3 px-4 py-3 text-card-foreground transition-colors",
expanded ? "bg-muted" : "bg-card",
)}
>
<div style={{ display: "flex", alignItems: "center", gap: 10 }}>
<div className="flex items-center gap-2.5">
<Wrench className="size-3.5 text-muted-foreground" />
<span style={{ fontSize: 14 }}>
<span className="text-sm">
{tool.index}. {tool.name}
</span>
</div>
<div style={{ display: "flex", alignItems: "center", gap: 8 }}>
<div className="flex items-center gap-2">
<Badge variant={tool.called ? "default" : "secondary"}>{tool.called ? "called" : "not called"}</Badge>
{expanded ? (
<ChevronDown className="size-3 text-muted-foreground" />
@ -55,13 +45,7 @@ export function ToolItem({ tool }: ToolItemProps) {
{/* Expanded Content */}
{expanded && (
<div
style={{
padding: "16px",
borderTop: "1px solid #f0f0f0",
background: "#fff",
}}
>
<div className="border-t border-border bg-card p-4 text-card-foreground">
<ToolExpandedContent tool={tool} />
</div>
)}

64
ui/litellm-dashboard/src/utils/searchUtils.test.ts Normal file
View file

@ -0,0 +1,64 @@
import { describe, expect, it } from "vitest";
import { filterBySearchTerm, matchesSearchTerm, rankBySearchRelevance } from "./searchUtils";
describe("matchesSearchTerm", () => {
it("matches everything on an empty or whitespace-only term", () => {
expect(matchesSearchTerm("", ["anything"])).toBe(true);
expect(matchesSearchTerm(" ", ["anything"])).toBe(true);
});
it("matches a substring of any field, case-insensitively", () => {
expect(matchesSearchTerm("BILL", ["Billing Router", "routes invoices"])).toBe(true);
expect(matchesSearchTerm("invoice", ["Billing Router", "routes invoices"])).toBe(true);
});
it("matches when every word appears in some field", () => {
expect(matchesSearchTerm("router invoices", ["Billing Router", "routes invoices"])).toBe(true);
expect(matchesSearchTerm("router refunds", ["Billing Router", "routes invoices"])).toBe(false);
});
it("returns false when nothing matches", () => {
expect(matchesSearchTerm("zzzz", ["Billing Router", "routes invoices"])).toBe(false);
});
it("ignores null and undefined fields", () => {
expect(matchesSearchTerm("billing", [null, undefined, "Billing Router"])).toBe(true);
expect(matchesSearchTerm("billing", [null, undefined])).toBe(false);
});
});
describe("filterBySearchTerm", () => {
const agents = [
{ name: "Billing Router", description: "routes invoices" },
{ name: "Support Bot", description: "handles tickets" },
];
it("keeps only items whose fields match", () => {
expect(filterBySearchTerm(agents, "tickets", (a) => [a.name, a.description])).toEqual([agents[1]]);
});
it("returns an empty list when nothing matches", () => {
expect(filterBySearchTerm(agents, "zzzz", (a) => [a.name, a.description])).toEqual([]);
});
it("returns all items for an empty term", () => {
expect(filterBySearchTerm(agents, "", (a) => [a.name, a.description])).toEqual(agents);
});
});
describe("rankBySearchRelevance", () => {
it("orders exact match, then prefix match, then shorter names", () => {
const items = [{ name: "gpt-4o-mini-transcribe" }, { name: "gpt-4o" }, { name: "chatgpt-4o-latest" }];
expect(rankBySearchRelevance(items, "gpt-4o", (m) => m.name).map((m) => m.name)).toEqual([
"gpt-4o",
"gpt-4o-mini-transcribe",
"chatgpt-4o-latest",
]);
});
it("keeps the original order for an empty term", () => {
const items = [{ name: "b" }, { name: "a" }];
expect(rankBySearchRelevance(items, "", (m) => m.name)).toEqual(items);
});
});

32
ui/litellm-dashboard/src/utils/searchUtils.ts Normal file
View file

@ -0,0 +1,32 @@
type SearchField = string | null | undefined;
const normalizeTerm = (term: string): string => term.trim().toLowerCase();
export function matchesSearchTerm(term: string, fields: ReadonlyArray<SearchField>): boolean {
const needle = normalizeTerm(term);
if (needle === "") return true;
const haystacks = fields.filter((field): field is string => typeof field === "string").map((f) => f.toLowerCase());
if (haystacks.some((haystack) => haystack.includes(needle))) return true;
return needle.split(/\s+/).every((word) => haystacks.some((haystack) => haystack.includes(word)));
}
export function filterBySearchTerm<T>(
items: ReadonlyArray<T>,
term: string,
fields: (item: T) => ReadonlyArray<SearchField>,
): T[] {
return items.filter((item) => matchesSearchTerm(term, fields(item)));
}
export function rankBySearchRelevance<T>(items: ReadonlyArray<T>, term: string, name: (item: T) => string): T[] {
const needle = normalizeTerm(term);
if (needle === "") return [...items];
const score = (item: T): number => {
const candidate = name(item).toLowerCase();
return (candidate === needle ? 1000 : 0) + (candidate.startsWith(needle) ? 100 : 0) + (1000 - candidate.length);
};
return [...items].sort((a, b) => score(b) - score(a));
}