diff --git a/.circleci/config.yml b/.circleci/config.yml index 63012ce3fc0..55fa9410845 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1483,7 +1483,7 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py -k "not legacy_resolver" + uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py -k "not v2_resolver" installing_litellm_on_python_3_13: docker: @@ -1507,9 +1507,9 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py -k "not legacy_resolver" + uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py -k "not v2_resolver" - installing_litellm_on_python_legacy_migration_resolver: + installing_litellm_on_python_v2_migration_resolver: docker: - *python312_image - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 @@ -1536,10 +1536,10 @@ jobs: url: tcp://localhost:5432 timeout: "60" - run: - name: Run legacy migration resolver proxy smoke test + name: Run v2 migration resolver proxy smoke test command: | uv run --no-sync python -m pytest -vv \ - tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings_legacy_resolver + tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings_v2_resolver helm_chart_testing: machine: @@ -2879,8 +2879,7 @@ jobs: command: | if grep -q "Error: P1001: Can't reach database server at" docker_output.log && \ (grep -q "Database setup failed after multiple retries" docker_output.log || \ - grep -q "ERROR: Application startup failed. Exiting." docker_output.log || \ - grep -q "Database migration cannot proceed" docker_output.log); then + grep -q "ERROR: Application startup failed. Exiting." docker_output.log); then echo "Expected error found. Test passed." else echo "Expected error not found. Test failed." @@ -3012,7 +3011,7 @@ workflows: filters: *main_branches - installing_litellm_on_python_3_13: filters: *main_branches - - installing_litellm_on_python_legacy_migration_resolver: + - installing_litellm_on_python_v2_migration_resolver: filters: *main_branches - helm_chart_testing: requires: diff --git a/CLAUDE.md b/CLAUDE.md index 6af8390b1af..d9e9e8f1586 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -37,7 +37,7 @@ Same applies for filing bug reports and feature requests, with .github/ISSUE_TEM If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just leave the section blank -Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload 2>&1 | tee litellm.log`; the Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR +Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`; the Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR If you ever write any human-facing text (pull requests, issues, commit messages, discussion posts, github comments, release notes, docs, etc.), always follow these guidelines to sound less AI-y: - don't use emojis diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index c088609dad7..b8032dd0d28 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -7,8 +7,7 @@ import subprocess import tempfile import time from pathlib import Path -from types import MappingProxyType -from typing import Final, Optional +from typing import Optional from litellm_proxy_extras._logging import logger from litellm_proxy_extras.replica_identity import ( @@ -51,17 +50,6 @@ _SPEND_LOGS_PK_CLAUSE_RE = re.compile( re.IGNORECASE, ) -_PRISMA_ATTEMPTS: Final = 4 - -_TRANSIENT_PRISMA_FAILURES: Final = MappingProxyType( - { - "deadlock detected": "a deadlock on the migration advisory lock (a concurrent migrate deploy)", - "P1001": "an unreachable database server", - "P1002": "a database server that timed out", - } -) - - PARTITIONED_SPEND_LOGS_PUSH_ERROR = ( "LiteLLM_SpendLogs is a partitioned table (see db_scripts/partition_spend_logs.sql), " "so its primary key must include the partition key (\"startTime\"). `prisma db push` " @@ -286,23 +274,6 @@ class ProxyExtrasDBManager: env=prisma_env, ) - @staticmethod - def _transient_prisma_failure(stderr: str) -> str | None: - """Why a failed prisma command is worth retrying, or None. - - v1 retried every failure, so it absorbed a database that was not up yet - or another instance holding the migration lock. v2 fails fast, which is - right for a broken migration and wrong for these. - """ - return next( - ( - reason - for marker, reason in _TRANSIENT_PRISMA_FAILURES.items() - if marker in stderr - ), - None, - ) - @staticmethod def _is_permission_error(error_message: str) -> bool: """ @@ -684,7 +655,7 @@ class ProxyExtrasDBManager: @staticmethod def _setup_database_v2(use_migrate: bool) -> bool: """ - v2 migration resolver (what the proxy CLI selects by default). + v2 migration resolver (opt-in via --use_v2_migration_resolver). Runs `prisma migrate deploy` and handles standard recovery paths (P3005 baseline, P3009/P3018 idempotent errors). Critically, it does @@ -705,46 +676,20 @@ class ProxyExtrasDBManager: original_dir = os.getcwd() os.chdir(migrations_dir) try: - for attempt in range(_PRISMA_ATTEMPTS): - try: - subprocess.run( - [_get_prisma_command(), "db", "push", "--accept-data-loss"], - timeout=prisma_command_timeout(), - check=True, - capture_output=True, - text=True, - env=_get_prisma_env(), - ) - return True - except subprocess.TimeoutExpired: - logger.info( - "prisma db push attempt %s timed out, retrying", - attempt + 1, - ) - time.sleep(random.randrange(5, 15)) - except subprocess.CalledProcessError as e: - stderr = e.stderr or "" - transient = ProxyExtrasDBManager._transient_prisma_failure( - stderr - ) - # Re-raise as RuntimeError so proxy_cli.py's - # `except RuntimeError` catches it and exits cleanly. - if transient is None or attempt == _PRISMA_ATTEMPTS - 1: - raise RuntimeError( - f"prisma db push failed.\n\nDetail: {e}" - f"\n\nPrisma error:\n{stderr}" - ) from e - logger.info( - "prisma db push attempt %s failed on %s, retrying. " - "Prisma error:\n%s", - attempt + 1, - transient, - stderr, - ) - time.sleep(random.randrange(5, 15)) - raise RuntimeError( - f"prisma db push failed after {_PRISMA_ATTEMPTS} attempts." + subprocess.run( + [_get_prisma_command(), "db", "push", "--accept-data-loss"], + timeout=prisma_command_timeout(), + check=True, + env=_get_prisma_env(), ) + return True + except ( + subprocess.CalledProcessError, + subprocess.TimeoutExpired, + ) as e: + # Re-raise as RuntimeError so proxy_cli.py's + # `except RuntimeError` catches it and exits cleanly. + raise RuntimeError(f"prisma db push failed.\n\nDetail: {e}") from e finally: os.chdir(original_dir) @@ -754,7 +699,7 @@ class ProxyExtrasDBManager: original_dir = os.getcwd() os.chdir(migrations_dir) try: - for attempt in range(_PRISMA_ATTEMPTS): + for attempt in range(4): try: result = subprocess.run( [_get_prisma_command(), "migrate", "deploy"], @@ -869,36 +814,16 @@ class ProxyExtrasDBManager: f"Manual intervention required.\n\nPrisma error:\n{stderr}" ) from e - transient = ProxyExtrasDBManager._transient_prisma_failure(stderr) - if transient is None: - raise RuntimeError( - "Database migration failed and cannot be auto-recovered. " - f"Manual intervention required.\n\nPrisma error:\n{stderr}" - ) from e - - if attempt == _PRISMA_ATTEMPTS - 1: - raise RuntimeError( - f"Database migration failed after " - f"{_PRISMA_ATTEMPTS} attempts on {transient}. " - "Check database connectivity and load." - f"\n\nPrisma error:\n{stderr}" - ) from e - - logger.info( - "prisma migrate deploy attempt %s failed on %s, retrying. " - "Prisma error:\n%s", - attempt + 1, - transient, - stderr, - ) - time.sleep(random.randrange(5, 15)) - continue + raise RuntimeError( + "Database migration failed and cannot be auto-recovered. " + f"Manual intervention required.\n\nPrisma error:\n{stderr}" + ) from e raise RuntimeError( - f"Database migration failed after {_PRISMA_ATTEMPTS} " - "attempts (retry loop exhausted by timeouts or repeated " - "idempotent-recovery continues). Check database connectivity, " - "load, and _prisma_migrations ledger state." + "Database migration failed after 4 attempts (retry loop " + "exhausted by timeouts or repeated idempotent-recovery " + "continues). Check database connectivity, load, and " + "_prisma_migrations ledger state." ) finally: os.chdir(original_dir) @@ -946,11 +871,10 @@ class ProxyExtrasDBManager: Args: use_migrate: Whether to use prisma migrate instead of db push - use_v2_resolver: Run the v2 migration resolver (safer during + use_v2_resolver: Opt into the v2 migration resolver (safer during rolling deploys; does not run the diff-and-force recovery - that causes schema thrashing). Defaults to False here so - direct callers keep the old behavior; the proxy CLI passes - True, so the proxy's runtime default is v2. + that causes schema thrashing). Defaults to False for + backwards compatibility. Returns: bool: True if setup was successful, False otherwise @@ -968,7 +892,7 @@ class ProxyExtrasDBManager: @staticmethod def _run_migrations(use_migrate: bool, use_v2_resolver: bool) -> bool: if use_v2_resolver: - logger.info("Using v2 migration resolver") + logger.info("Using v2 migration resolver (--use_v2_migration_resolver)") return ProxyExtrasDBManager._setup_database_v2(use_migrate=use_migrate) schema_path = ProxyExtrasDBManager._get_prisma_dir() + "/schema.prisma" diff --git a/litellm-proxy-extras/tests/__init__.py b/litellm-proxy-extras/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm-proxy-extras/tests/test_setup_database_fail_fast.py b/litellm-proxy-extras/tests/test_setup_database_fail_fast.py new file mode 100644 index 00000000000..8d66bf872de --- /dev/null +++ b/litellm-proxy-extras/tests/test_setup_database_fail_fast.py @@ -0,0 +1,242 @@ +"""Regression tests for ProxyExtrasDBManager v2 migration resolver. + +The v2 resolver is opt-in via `--use_v2_migration_resolver` / the +`use_v2_resolver=True` kwarg. These tests exercise the v2 path; the v1 +(default) behavior is unchanged from pre-fix. +""" + +import subprocess +from unittest.mock import patch + +import pytest + +from litellm_proxy_extras.utils import ( + ProxyExtrasDBManager, + _max_migration_timestamp, + _migration_timestamp, +) + + +def _fake_migrate_deploy_failure(returncode: int, stderr: str): + def _run(*args, **kwargs): + raise subprocess.CalledProcessError( + returncode=returncode, + cmd=args[0], + stderr=stderr, + output="", + ) + + return _run + + +def test_v2_p3018_permission_error_raises_runtime_error(monkeypatch, tmp_path): + """v2: a permission failure during migrate deploy raises RuntimeError.""" + monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x") + monkeypatch.setattr( + ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None + ) + monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path)) + (tmp_path / "schema.prisma").write_text("// stub") + + stderr = ( + "Error: P3018\nMigration name: 20250326162113_baseline\n" + "Database error code: 42501\npermission denied for schema public" + ) + with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)): + with pytest.raises(RuntimeError, match="permission"): + ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + + +def test_v2_non_idempotent_p3009_raises_runtime_error(monkeypatch, tmp_path): + """v2: a non-idempotent migration failure raises (no silent recovery).""" + monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x") + monkeypatch.setattr( + ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None + ) + monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path)) + (tmp_path / "schema.prisma").write_text("// stub") + + stderr = ( + "Error: P3009\nMigration `20260101000000_genuinely_broken` failed\n" + 'Reason: syntax error at or near "BRKN" LINE 42' + ) + with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)): + with pytest.raises(RuntimeError, match="cannot be auto-recovered"): + ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + + +def test_strip_prisma_query_params_removes_connection_limit(): + """DATABASE_URLs with Prisma-specific params should be parseable by psycopg.""" + url = "postgresql://u:p@h:5432/db?connection_limit=100&pool_timeout=60&sslmode=require" + stripped = ProxyExtrasDBManager._strip_prisma_query_params(url) + assert "connection_limit" not in stripped + assert "pool_timeout" not in stripped + assert "sslmode=require" in stripped + + +def test_strip_prisma_query_params_passthrough_no_query(): + """URLs without query strings are returned unchanged.""" + url = "postgresql://u:p@h:5432/db" + assert ProxyExtrasDBManager._strip_prisma_query_params(url) == url + + +def test_migration_timestamp_extracts_leading_digits(): + assert _migration_timestamp("20260101000000_add_foo") == 20260101000000 + assert _migration_timestamp("20250326162113_baseline") == 20250326162113 + + +def test_migration_timestamp_returns_zero_on_malformed(): + assert _migration_timestamp("0_init") == 0 + assert _migration_timestamp("not_a_migration") == 0 + + +def test_max_migration_timestamp(): + names = {"20250326000000_a", "20260415000000_b", "20251115000000_c"} + assert _max_migration_timestamp(names) == 20260415000000 + + +def test_max_migration_timestamp_empty_set(): + assert _max_migration_timestamp(set()) == 0 + + +def test_v1_default_still_calls_resolve_all_migrations(monkeypatch, tmp_path): + """v1 (default) continues to call _resolve_all_migrations on the happy path. + + This is the existing buggy behavior — we're not fixing it in v1, only + offering v2 as opt-in. This test pins the default so that a future + inadvertent default flip is caught. + """ + monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path)) + (tmp_path / "schema.prisma").write_text("// stub") + + # Stub `prisma migrate deploy` to claim success with pending migrations + # applied, which is the code path that triggers the legacy post-migration + # sanity check (a call to _resolve_all_migrations). + class FakeResult: + stdout = "Applied migration.\n" + stderr = "" + + def fake_run(cmd, *args, **kwargs): + return FakeResult() + + resolve_called = {"n": 0} + + def fake_resolve(*args, **kwargs): + resolve_called["n"] += 1 + + monkeypatch.setattr("subprocess.run", fake_run) + monkeypatch.setattr(ProxyExtrasDBManager, "_resolve_all_migrations", fake_resolve) + + ok = ProxyExtrasDBManager.setup_database(use_migrate=True) # v2 flag NOT set + assert ok is True + assert resolve_called["n"] == 1, "v1 default should still invoke the legacy path" + + +def test_v2_db_push_wraps_subprocess_error_as_runtime_error(monkeypatch, tmp_path): + """v2: a failing `prisma db push` must raise RuntimeError, not leak + CalledProcessError past proxy_cli.py's `except RuntimeError`.""" + monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path)) + (tmp_path / "schema.prisma").write_text("// stub") + + stderr = "db push error" + with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)): + with pytest.raises(RuntimeError, match="prisma db push failed"): + ProxyExtrasDBManager.setup_database(use_migrate=False, use_v2_resolver=True) + + +def test_v2_warn_ahead_of_head_swallows_db_errors(monkeypatch, tmp_path): + """_warn_if_db_ahead_of_head must never raise — it's informational. + + Non-connection DB errors (e.g. InsufficientPrivilege from a user + without SELECT on _prisma_migrations) must be caught, not propagated. + """ + import psycopg + + monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x") + monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path)) + (tmp_path / "schema.prisma").write_text("// stub") + + class _FakeConn: + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def execute(self, *a, **kw): + # Simulate an InsufficientPrivilege (subclass of DatabaseError). + raise psycopg.errors.InsufficientPrivilege("permission denied") + + def _fake_connect(*a, **kw): + return _FakeConn() + + monkeypatch.setattr("psycopg.connect", _fake_connect) + + # Must not raise. + ProxyExtrasDBManager._warn_if_db_ahead_of_head(str(tmp_path)) + + +def test_v2_resolve_specific_migration_failure_raises_runtime_error( + monkeypatch, tmp_path +): + """If marking a migration as applied fails inside P3009 idempotent + recovery, the subprocess error must be re-raised as RuntimeError so + proxy_cli.py catches it cleanly (instead of leaking CalledProcessError).""" + monkeypatch.setattr( + ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None + ) + monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path)) + (tmp_path / "schema.prisma").write_text("// stub") + monkeypatch.setattr( + ProxyExtrasDBManager, "_roll_back_migration", lambda *a, **kw: None + ) + + # First call: migrate deploy -> P3009 idempotent error. + # Recovery path tries _resolve_specific_migration; that also raises. + def _failing_resolve(*a, **kw): + raise subprocess.CalledProcessError( + returncode=1, + cmd="prisma migrate resolve --applied", + stderr="resolve failed", + output="", + ) + + monkeypatch.setattr( + ProxyExtrasDBManager, "_resolve_specific_migration", _failing_resolve + ) + + stderr = ( + "Error: P3009\nMigration `20260101000000_some_migration` failed\n" + "relation already exists" + ) + with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)): + with pytest.raises( + RuntimeError, match="Failed to mark migration .* as applied" + ): + ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + + +def test_v2_does_not_call_resolve_all_migrations(monkeypatch, tmp_path): + """v2 must never call _resolve_all_migrations — that's the bug it fixes.""" + monkeypatch.setattr( + ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None + ) + monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path)) + (tmp_path / "schema.prisma").write_text("// stub") + + class FakeResult: + stdout = "Applied migration.\n" + stderr = "" + + monkeypatch.setattr("subprocess.run", lambda *a, **kw: FakeResult()) + + resolve_called = {"n": 0} + monkeypatch.setattr( + ProxyExtrasDBManager, + "_resolve_all_migrations", + lambda *a, **kw: resolve_called.__setitem__("n", resolve_called["n"] + 1), + ) + + ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + assert ok is True + assert resolve_called["n"] == 0, "v2 must not invoke the diff-and-force recovery" diff --git a/litellm/constants.py b/litellm/constants.py index b9bea466247..9a50797f517 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -13,6 +13,12 @@ DEFAULT_BATCH_SIZE: Final = int(os.getenv("DEFAULT_BATCH_SIZE", 512)) DEFAULT_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5)) DEFAULT_S3_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_S3_FLUSH_INTERVAL_SECONDS", 10)) DEFAULT_S3_BATCH_SIZE: Final = int(os.getenv("DEFAULT_S3_BATCH_SIZE", 512)) +# https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html +MAX_S3_OBJECT_KEY_BYTES: Final = 1024 +S3_BOUNDED_OBJECT_KEY_HEAD_BYTES: Final = 64 +S3_PREFIX_DIGEST_CHARS: Final = 16 +# s3 allows 2048 bytes of combined metadata headers, which Content-Disposition counts against +MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES: Final = 1024 DEFAULT_SQS_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10)) DEFAULT_NUM_WORKERS_LITELLM_PROXY: Final = int(os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1)) DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE = int(os.getenv("DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE", 1)) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 467ec72dc4a..add91033ff3 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -59,6 +59,8 @@ from litellm.types.utils import ( if TYPE_CHECKING: from apscheduler.schedulers.asyncio import AsyncIOScheduler from prometheus_client.metrics import MetricWrapperBase + + from litellm.router import Router else: AsyncIOScheduler = Any @@ -67,6 +69,8 @@ _TableRowT: Final = TypeVar("_TableRowT", bound=BaseModel) _DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT: Final = 5.0 +UNRECOGNIZED_REQUESTED_MODEL_LABEL: Final = "other" + _NON_ENUM_METRIC_LABELS: Final[frozenset[str]] = frozenset( ( "guardrail_name", @@ -154,6 +158,44 @@ def _get_budget_metrics_per_request_timeout() -> float: return parsed +def _get_proxy_llm_router() -> Router | None: + try: + from litellm.proxy.proxy_server import llm_router + except Exception: + return None + return llm_router + + +def _bounded_requested_model_label(requested_model: str | None, router_originated: bool = False) -> str | None: + """ + Bound ``requested_model`` label cardinality: names the router recognizes + (model names, deployment ids, aliases, routing groups, team public model + names) or matches via a global or team wildcard/pattern route keep their + own label value; any other client-supplied string collapses into the + single ``other`` bucket. With no proxy router to vouch for the string, + client-supplied values collapse to ``other`` while ``router_originated`` + values (emitted by an SDK ``Router``'s own deployment failure and + fallback events, where the proxy router never exists) pass through. + """ + if not requested_model: + return requested_model + llm_router: Final = _get_proxy_llm_router() + if llm_router is None: + return requested_model if router_originated else UNRECOGNIZED_REQUESTED_MODEL_LABEL + if llm_router.is_recognized_model(requested_model): + return requested_model + if requested_model in llm_router.team_public_model_names: + return requested_model + if llm_router.pattern_router.route(requested_model) is not None: + return requested_model + if any( + team_pattern_router.route(requested_model) is not None + for team_pattern_router in llm_router.team_pattern_routers.values() + ): + return requested_model + return UNRECOGNIZED_REQUESTED_MODEL_LABEL + + class PrometheusLogger(CustomLogger): # Class variables or attributes @@ -2407,7 +2449,7 @@ class PrometheusLogger(CustomLogger): team_alias=user_api_key_dict.team_alias, org_id=user_api_key_dict.org_id, org_alias=user_api_key_dict.organization_alias, - requested_model=request_data.get("model", ""), + requested_model=_bounded_requested_model_label(request_data.get("model", "")), status_code=str(status_code), exception_status=str(status_code), exception_class=self._get_exception_class_name(original_exception), @@ -2627,7 +2669,9 @@ class PrometheusLogger(CustomLogger): label_model_id = "" label_api_base = "" label_api_provider = "" - label_requested_model = litellm_model_name or model_group or "" + label_requested_model = ( + _bounded_requested_model_label(litellm_model_name or model_group, router_originated=True) or "" + ) enum_values: Final = UserAPIKeyLabelValues( litellm_model_name=label_litellm_model_name, @@ -3186,7 +3230,7 @@ class PrometheusLogger(CustomLogger): _tags: Final = cast(list[str], kwargs.get("tags") or []) enum_values: Final = UserAPIKeyLabelValues( - requested_model=original_model_group, + requested_model=_bounded_requested_model_label(original_model_group, router_originated=True), fallback_model=_new_model, hashed_api_key=standard_metadata["user_api_key_hash"], api_key_alias=standard_metadata["user_api_key_alias"], @@ -3227,7 +3271,7 @@ class PrometheusLogger(CustomLogger): ) enum_values: Final = UserAPIKeyLabelValues( - requested_model=original_model_group, + requested_model=_bounded_requested_model_label(original_model_group, router_originated=True), fallback_model=_new_model, hashed_api_key=standard_metadata["user_api_key_hash"], api_key_alias=standard_metadata["user_api_key_alias"], diff --git a/litellm/integrations/s3.py b/litellm/integrations/s3.py index ddeb410c54a..8ce461eea5b 100644 --- a/litellm/integrations/s3.py +++ b/litellm/integrations/s3.py @@ -1,11 +1,18 @@ #### What this does #### # On success + failure, log events to Supabase +import hashlib from datetime import datetime from typing import Final, cast import litellm from litellm._logging import print_verbose, verbose_logger +from litellm.constants import ( + MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES, + MAX_S3_OBJECT_KEY_BYTES, + S3_BOUNDED_OBJECT_KEY_HEAD_BYTES, + S3_PREFIX_DIGEST_CHARS, +) from litellm.types.utils import StandardLoggingPayload @@ -133,9 +140,7 @@ class S3Logger: s3_file_name, ) - s3_object_download_filename: Final = ( - "time-" + start_time.strftime("%Y-%m-%dT%H-%M-%S-%f") + "_" + payload["id"] + ".json" - ) + s3_object_download_filename: Final = get_s3_object_download_filename(start_time, payload["id"]) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps @@ -198,6 +203,47 @@ def resolve_sse_params( return algorithm, valid_key_id +S3_MIN_BOUNDED_FILE_NAME_BYTES: Final = 64 + + +def _truncate_to_utf8_bytes(value: str, max_bytes: int) -> str: + """Trim `value` so its UTF-8 encoding fits `max_bytes`, never splitting a character.""" + if max_bytes <= 0: + return "" + encoded: Final = value.encode("utf-8") + if len(encoded) <= max_bytes: + return value + return encoded[:max_bytes].decode("utf-8", errors="ignore") + + +def get_s3_object_download_filename(start_time: datetime, response_id: str) -> str: + """Content-Disposition filename for the uploaded object, bounded to the metadata header cap.""" + sanitized_response_id: Final = response_id.replace("/", "_").replace('"', "_") + file_name: Final = f"time-{start_time.strftime('%Y-%m-%dT%H-%M-%S-%f')}_{response_id}" + sanitized_file_name: Final = f"time-{start_time.strftime('%Y-%m-%dT%H-%M-%S-%f')}_{sanitized_response_id}" + budget: Final = MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES - len(b".json") + if len(sanitized_file_name.encode("utf-8")) <= budget: + return sanitized_file_name + ".json" + return _bounded_s3_file_name(file_name, sanitized_file_name, budget) + ".json" + + +def _bounded_s3_file_name(s3_file_name: str, sanitized_s3_file_name: str, max_bytes: int) -> str: + """As much of the file name as `max_bytes` allows, then the sha256 of the whole name.""" + digest: Final = hashlib.sha256(s3_file_name.encode("utf-8")).hexdigest() + head_budget: Final = min(S3_BOUNDED_OBJECT_KEY_HEAD_BYTES, max_bytes - len(digest) - 1) + head: Final = _truncate_to_utf8_bytes(sanitized_s3_file_name, head_budget) + return f"{head}_{digest}" if head else digest + + +def _bounded_s3_prefix(configured_prefix: str, max_bytes: int) -> str: + """As much of the configured prefix as fits, then a digest segment naming the full prefix.""" + digest_segment: Final = hashlib.sha256(configured_prefix.encode("utf-8")).hexdigest()[:S3_PREFIX_DIGEST_CHARS] + "/" + if max_bytes < len(digest_segment): + return "" + head: Final = _truncate_to_utf8_bytes(configured_prefix, max_bytes - len(digest_segment) - 1).rstrip("/") + return f"{head}/{digest_segment}" if head else digest_segment + + def get_s3_object_key( s3_path: str, prefix: str, @@ -205,12 +251,23 @@ def get_s3_object_key( s3_file_name: str, ) -> str: sanitized_s3_file_name: Final = s3_file_name.replace("/", "_") - s3_object_key = ( - (s3_path.rstrip("/") + "/" if s3_path else "") - + prefix - + start_time.strftime("%Y-%m-%d") - + "/" - + sanitized_s3_file_name - ) # we need the s3 key to include the time, so we log cache hits too - s3_object_key += ".json" - return s3_object_key + configured_prefix: Final = (s3_path.rstrip("/") + "/" if s3_path else "") + prefix + date_segment: Final = start_time.strftime("%Y-%m-%d") + "/" + # we need the s3 key to include the time, so we log cache hits too + s3_object_key: Final = configured_prefix + date_segment + sanitized_s3_file_name + ".json" + if len(s3_object_key.encode("utf-8")) <= MAX_S3_OBJECT_KEY_BYTES: + return s3_object_key + + # shorten the response id first and only trim the configured prefix if that is what does not + # fit, so prefix scoped IAM policies and lifecycle rules keep matching + budget: Final = MAX_S3_OBJECT_KEY_BYTES - len(date_segment.encode("utf-8")) - len(b".json") + prefix_bytes: Final = len(configured_prefix.encode("utf-8")) + if prefix_bytes + S3_MIN_BOUNDED_FILE_NAME_BYTES <= budget: + bounded_file_name: Final = _bounded_s3_file_name(s3_file_name, sanitized_s3_file_name, budget - prefix_bytes) + return configured_prefix + date_segment + bounded_file_name + ".json" + + shortest_file_name: Final = _bounded_s3_file_name( + s3_file_name, sanitized_s3_file_name, S3_MIN_BOUNDED_FILE_NAME_BYTES + ) + bounded_prefix: Final = _bounded_s3_prefix(configured_prefix, budget - len(shortest_file_name.encode("utf-8"))) + return bounded_prefix + date_segment + shortest_file_name + ".json" diff --git a/litellm/integrations/s3_v2.py b/litellm/integrations/s3_v2.py index 9f6ae72fb3a..712ce41d09e 100644 --- a/litellm/integrations/s3_v2.py +++ b/litellm/integrations/s3_v2.py @@ -16,7 +16,11 @@ from urllib.parse import quote import litellm from litellm._logging import print_verbose, verbose_logger from litellm.constants import DEFAULT_S3_BATCH_SIZE, DEFAULT_S3_FLUSH_INTERVAL_SECONDS -from litellm.integrations.s3 import get_s3_object_key, resolve_sse_params +from litellm.integrations.s3 import ( + get_s3_object_download_filename, + get_s3_object_key, + resolve_sse_params, +) from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker @@ -259,11 +263,11 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): now: Final = datetime.now(timezone.utc) audit_log_id: Final = audit_log.get("id", "unknown") - s3_path = cast(str | None, self.s3_path) or "" - s3_path = s3_path.rstrip("/") + "/" if s3_path else "" - - s3_object_key: Final = ( - f"{s3_path}audit_logs/{now.strftime('%Y-%m-%d')}/{now.strftime('%H-%M-%S')}_{audit_log_id}.json" + s3_object_key: Final = get_s3_object_key( + cast(str | None, self.s3_path) or "", + "audit_logs/", + now, + f"{now.strftime('%H-%M-%S')}_{audit_log_id}", ) element: Final = s3BatchLoggingElement( @@ -463,9 +467,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): ) verbose_logger.debug("s3_object_key=%s", s3_object_key) - s3_object_download_filename: Final = ( - f"time-{start_time.strftime('%Y-%m-%dT%H-%M-%S-%f')}_{standard_logging_payload['id']}.json" - ) + s3_object_download_filename: Final = get_s3_object_download_filename(start_time, standard_logging_payload["id"]) return s3BatchLoggingElement( payload=dict(standard_logging_payload), diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 350e5403e4c..eb223c2f988 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -111,6 +111,7 @@ from litellm.types.mcp import MCPPostCallResponseObject from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.rerank import RerankResponse from litellm.types.utils import ( + DEPLOYMENT_SCOPED_PRICING_FIELDS, CachingDetails, CallTypes, CostBreakdown, @@ -255,6 +256,7 @@ _STANDARD_LOGGING_METADATA_KEYS: Final[frozenset[str]] = frozenset(StandardLoggi # Cache custom pricing keys as frozenset for O(1) lookups instead of looping through 49 keys _CUSTOM_PRICING_KEYS: Final[frozenset[str]] = frozenset(CustomPricingLiteLLMParams.model_fields.keys()) +_MODEL_INFO_CUSTOM_PRICING_KEYS: Final[frozenset[str]] = _CUSTOM_PRICING_KEYS | DEPLOYMENT_SCOPED_PRICING_FIELDS sentry_sdk_instance = None capture_exception = None @@ -5033,7 +5035,9 @@ def use_custom_pricing_for_model(litellm_params: dict | None) -> bool: """ Check if the model uses custom pricing - Returns True if any of `SPECIAL_MODEL_INFO_PARAMS` are present in `litellm_params` or `model_info` + Returns True if any custom pricing field is present in `litellm_params`, or if + any custom pricing or deployment-scoped pricing field (such as + ``off_peak_pricing``) is present in the metadata ``model_info`` """ if litellm_params is None: return False @@ -5051,7 +5055,7 @@ def use_custom_pricing_for_model(litellm_params: dict | None) -> bool: model_info: dict = metadata.get("model_info", {}) or {} if model_info: - matching_keys = _CUSTOM_PRICING_KEYS & model_info.keys() + matching_keys = _MODEL_INFO_CUSTOM_PRICING_KEYS & model_info.keys() for key in matching_keys: if model_info.get(key) is not None: return True diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 19e3f624268..b34c416cd40 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -2,10 +2,12 @@ ## Helper utilities for cost_per_token() import re -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from dataclasses import dataclass +from datetime import datetime, timezone, tzinfo from types import MappingProxyType from typing import Any, Final, Literal, TypedDict, cast +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import litellm from litellm._logging import verbose_logger @@ -290,10 +292,187 @@ def _get_tiered_base_costs(model_info: ModelInfo, usage: Usage) -> tuple[float, ) +def _is_within_off_peak_window(off_peak_hours_utc: str | Sequence[str], current_time: datetime | None = None) -> bool: + """Return True if current_time (UTC, defaulting to now) falls inside any off-peak window. + + off_peak_hours_utc is a "HH:MM-HH:MM" string in UTC, or a list of such strings for providers + with multiple daily windows (e.g. ["16:30-00:30", "04:00-06:00"]). A window may wrap past + midnight, and a window whose start equals its end covers the whole day. The start is + inclusive and the end is exclusive; malformed windows are ignored. + + An aware current_time is converted to UTC. A naive one is taken to already be UTC rather + than being localised, so callers must pass datetime.now(timezone.utc), never datetime.now(), + or every window shifts by the host's offset. + """ + reference: Final = current_time if current_time is not None else datetime.now(timezone.utc) + now: Final = (reference.astimezone(timezone.utc) if reference.tzinfo is not None else reference).time() + windows: Final = (off_peak_hours_utc,) if isinstance(off_peak_hours_utc, str) else off_peak_hours_utc + for window in windows: + try: + start_str, end_str = window.split("-") + start = datetime.strptime(start_str.strip(), "%H:%M").replace(tzinfo=timezone.utc).time() + end = datetime.strptime(end_str.strip(), "%H:%M").replace(tzinfo=timezone.utc).time() + except (ValueError, AttributeError): + continue + if start < end: + if start <= now < end: + return True + elif now >= start or now < end: + return True + return False + + +_WEEKDAY_NUMBERS: Final = MappingProxyType( + { + "mon": 1, + "monday": 1, + "tue": 2, + "tues": 2, + "tuesday": 2, + "wed": 3, + "wednesday": 3, + "thu": 4, + "thur": 4, + "thurs": 4, + "thursday": 4, + "fri": 5, + "friday": 5, + "sat": 6, + "saturday": 6, + "sun": 7, + "sunday": 7, + } +) + + +def _normalize_weekday(value: object) -> int | None: + if isinstance(value, bool): + return None + if isinstance(value, int): + return value if 1 <= value <= 7 else None + if isinstance(value, str): + return _WEEKDAY_NUMBERS.get(value.strip().lower()) + return None + + +def _weekday_calendar(weekday_timezone: object) -> tzinfo: + if isinstance(weekday_timezone, str) and weekday_timezone.strip(): + try: + return ZoneInfo(weekday_timezone.strip()) + except (ValueError, ZoneInfoNotFoundError): + return timezone.utc + return timezone.utc + + +def _matches_weekdays(reference_utc: datetime, weekdays: object, weekday_timezone: object) -> bool: + """Return True when reference_utc falls on one of the rule's weekdays, read on the calendar + named by weekday_timezone (default UTC). An absent weekdays means every day. The calendar + matters even when UTC and vendor-local weekdays agree at every currently priced hour: a + window past 16:00 UTC is where an Asia/Shanghai weekday diverges from the UTC one. + """ + if weekdays is None: + return True + if isinstance(weekdays, str) or not isinstance(weekdays, Sequence): + return False + allowed: Final = frozenset(day for day in map(_normalize_weekday, weekdays) if day is not None) + return reference_utc.astimezone(_weekday_calendar(weekday_timezone)).isoweekday() in allowed + + +def _as_window_strings(value: object) -> tuple[str, ...]: + if isinstance(value, str): + return (value,) + if isinstance(value, Sequence): + return tuple(entry for entry in value if isinstance(entry, str)) + return () + + +def _is_off_peak(off_peak: Mapping[str, object], current_time: datetime | None = None) -> bool: + """Return True when current_time (UTC, defaulting to now) is off-peak under the block's + rules: the flat hours_utc windows, which apply every day, or any entry in windows, whose + hours apply only on its weekdays. + """ + reference: Final = current_time if current_time is not None else datetime.now(timezone.utc) + reference_utc: Final = ( + reference.astimezone(timezone.utc) if reference.tzinfo is not None else reference.replace(tzinfo=timezone.utc) + ) + flat_windows: Final = _as_window_strings(off_peak.get("hours_utc")) + if flat_windows and _is_within_off_peak_window(flat_windows, reference_utc): + return True + windows: Final = off_peak.get("windows") + if isinstance(windows, str) or not isinstance(windows, Sequence): + return False + weekday_timezone: Final = off_peak.get("weekday_timezone") + for rule in windows: + if not isinstance(rule, Mapping): + continue + rule_windows = _as_window_strings(rule.get("hours_utc")) + if not rule_windows: + continue + if not _matches_weekdays(reference_utc, rule.get("weekdays"), weekday_timezone): + continue + if _is_within_off_peak_window(rule_windows, reference_utc): + return True + return False + + +def _coerce_off_peak_rate(value: object, default: float) -> float: + if isinstance(value, bool): + return default + if isinstance(value, (int, float)): + return float(value) + if isinstance(value, str): + try: + return float(value) + except ValueError: + return default + return default + + +def _apply_off_peak_pricing( + model_info: ModelInfo, + current_time: datetime | None, + prompt_base_cost: float, + completion_base_cost: float, + cache_read_cost: float, +) -> tuple[float, float, float]: + """Swap in off-peak per-token rates when the current UTC time is inside one of the model's + off_peak_pricing rules, the every-day hours_utc windows or a day-of-week-qualified entry in + windows. An off-peak rate replaces the rate that would otherwise apply rather than + discounting it, so a model that also has tiered or above-threshold pricing bills the flat + off-peak rate for the whole request while the window is open. Any rate left unset in + off_peak_pricing falls back to the standard rate. + """ + off_peak: Final = model_info.get("off_peak_pricing") + if not isinstance(off_peak, Mapping) or not _is_off_peak(off_peak, current_time): + return prompt_base_cost, completion_base_cost, cache_read_cost + return ( + _coerce_off_peak_rate(off_peak.get("input_cost_per_token"), prompt_base_cost), + _coerce_off_peak_rate(off_peak.get("output_cost_per_token"), completion_base_cost), + _coerce_off_peak_rate(off_peak.get("cache_read_input_token_cost"), cache_read_cost), + ) + + +def _apply_off_peak_to_base_costs( + model_info: ModelInfo, + current_time: datetime | None, + base_costs: tuple[float, float, float, float, float], +) -> tuple[float, float, float, float, float]: + """Apply off-peak rates to an already-resolved set of base costs, whichever pricing path + produced them. Cache-creation rates are passed through untouched, since off_peak_pricing + has no field for them. + """ + prompt, completion, cache_creation, cache_creation_above_1hr, cache_read = base_costs + off_peak_prompt, off_peak_completion, off_peak_cache_read = _apply_off_peak_pricing( + model_info, current_time, prompt, completion, cache_read + ) + return (off_peak_prompt, off_peak_completion, cache_creation, cache_creation_above_1hr, off_peak_cache_read) + + def _get_token_base_cost( model_info: ModelInfo, usage: Usage, service_tier: str | None = None, + current_time: datetime | None = None, *, threshold_is_inclusive: bool = False, ) -> tuple[float, float, float, float, float]: @@ -311,7 +490,7 @@ def _get_token_base_cost( """ tiered_base_costs: Final = _get_tiered_base_costs(model_info=model_info, usage=usage) if tiered_base_costs is not None: - return tiered_base_costs + return _apply_off_peak_to_base_costs(model_info, current_time, tiered_base_costs) # Get service tier aware cost keys input_cost_key: Final = _get_service_tier_cost_key("input_cost_per_token", service_tier) @@ -345,12 +524,16 @@ def _get_token_base_cost( k for k in model_info if k.startswith("input_cost_per_token_above_") and not k.endswith(_SERVICE_TIER_SUFFIXES) ] if not threshold_keys: - return ( - prompt_base_cost, - completion_base_cost, - cache_creation_cost, - cache_creation_cost_above_1hr, - cache_read_cost, + return _apply_off_peak_to_base_costs( + model_info, + current_time, + ( + prompt_base_cost, + completion_base_cost, + cache_creation_cost, + cache_creation_cost_above_1hr, + cache_read_cost, + ), ) # Only sort the threshold keys (typically 1-2 keys instead of 66+) @@ -451,12 +634,16 @@ def _get_token_base_cost( except Exception: continue - return ( - prompt_base_cost, - completion_base_cost, - cache_creation_cost, - cache_creation_cost_above_1hr, - cache_read_cost, + return _apply_off_peak_to_base_costs( + model_info, + current_time, + ( + prompt_base_cost, + completion_base_cost, + cache_creation_cost, + cache_creation_cost_above_1hr, + cache_read_cost, + ), ) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 0bfe7dddd7f..aa805ccea71 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1516,7 +1516,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): _tool = self.map_response_format_to_anthropic_tool(value, optional_params, is_thinking_enabled) if _tool is None: continue - if not is_thinking_enabled: + if not is_thinking_enabled and not AnthropicModelInfo.forced_tool_use_unsupported(model): _tool_choice = { "name": RESPONSE_FORMAT_TOOL_NAME, "type": "tool", diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index b1f927fd8f9..c60ebd844ba 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -325,13 +325,17 @@ class AnthropicModelInfo(BaseLLMModelInfo): status_code=400, ) + @staticmethod + def forced_tool_use_unsupported(model: str) -> bool: + return AnthropicModelInfo._get_model_capability(model, "supports_forced_tool_use") is False + @staticmethod def forced_tool_use_downgraded(model: str, drop_params: bool) -> bool: """True when the model map flags the model with ``supports_forced_tool_use: false`` (Fable 5.1 / Mythos 5.1 400 on ``any``/``tool``) and ``drop_params`` asks for the ``auto`` downgrade; raises a clean client-side 400 for such models without ``drop_params``.""" - if AnthropicModelInfo._get_model_capability(model, "supports_forced_tool_use") is not False: + if not AnthropicModelInfo.forced_tool_use_unsupported(model): return False if not (litellm.drop_params or drop_params): raise litellm.utils.UnsupportedParamsError( diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 6f99f572686..7fefeaeaf04 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1073,6 +1073,7 @@ class AmazonConverseConfig(BaseConfig): if ( litellm.utils.supports_tool_choice(model=model, custom_llm_provider=self.custom_llm_provider) and not is_thinking_enabled + and not AnthropicModelInfo.forced_tool_use_unsupported(model) ): optional_params["tool_choice"] = ToolChoiceValuesBlock( tool=SpecificToolChoiceBlock(name=RESPONSE_FORMAT_TOOL_NAME) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index 8e709349400..2a4c38e71ea 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers +from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_anthropic_image_obj, ) @@ -11,6 +12,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( convert_url_to_base64, ) from litellm.llms.anthropic.chat.transformation import AnthropicConfig +from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( AmazonInvokeConfig, ) @@ -74,10 +76,14 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): drop_params: bool, ) -> dict: # Force tool-based structured outputs for Bedrock Invoke - # (similar to VertexAI fix in #19201) - # Bedrock Invoke doesn't support output_format parameter + # (similar to VertexAI fix in #19201) unless the model map advertises + # native structured output + from litellm.utils import supports_native_structured_output + original_model: Final = model - if "response_format" in non_default_params: + if "response_format" in non_default_params and not supports_native_structured_output( + model=model, custom_llm_provider="bedrock" + ): # Use a model name that forces tool-based approach model = "claude-3-sonnet-20240229" @@ -101,6 +107,16 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): # Restore original model name model = original_model + # The stub model hides the original model from the parent's forced-tool-use backstop + response_format_tool_choice: Final = optional_params.get("tool_choice") + if ( + "response_format" in non_default_params + and isinstance(response_format_tool_choice, dict) + and response_format_tool_choice.get("name") == RESPONSE_FORMAT_TOOL_NAME + and AnthropicModelInfo.forced_tool_use_unsupported(original_model) + ): + optional_params.pop("tool_choice") + return optional_params @staticmethod diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 7b1710fdfc5..f3e41c054ef 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -4,7 +4,8 @@ Support for gpt model family import json import os -from collections.abc import AsyncIterator, Coroutine, Iterator +from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast, overload from urllib.parse import urlparse @@ -19,6 +20,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo ) from litellm.litellm_core_utils.prompt_templates.common_utils import ( drop_tool_reference_parts_from_tool_messages, + flatten_top_level_schema_combinators, get_tool_call_names, hoist_images_from_tool_messages, ) @@ -65,6 +67,22 @@ else: LiteLLMLoggingObj = Any +_NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({}) + + +def _tool_with_flattened_parameters(tool: Mapping[str, object]) -> Mapping[str, object]: + function: Final = tool.get("function") + if not isinstance(function, dict): + return tool + parameters: Final = function.get("parameters") + if not isinstance(parameters, dict): + return tool + flattened: Final = flatten_top_level_schema_combinators(parameters) + if flattened is parameters: + return tool + return {**tool, "function": {**function, "parameters": flattened}} # mutable-ok: request tools are JSON dicts + + class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): """ Reference: https://platform.openai.com/docs/api-reference/chat/create @@ -397,6 +415,21 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): ) return messages, tools + def _targets_openai_hosted_endpoint( + self, + custom_llm_provider: str | None, + api_base: str | None, + ) -> bool: + if custom_llm_provider != "openai": + return False + resolved_api_base = api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE") + if not resolved_api_base: + return True + hostname: Final = urlparse(resolved_api_base).hostname + if hostname is None: + return True + return hostname == "openai.com" or hostname.endswith(".openai.com") + def _should_preserve_cache_control_for_endpoint( self, custom_llm_provider: str | None, @@ -408,15 +441,34 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): api_base. Those can understand cache_control, so it must survive there. Real OpenAI cannot, so it is still stripped for an openai.com host. """ - if custom_llm_provider != "openai": - return False - resolved_api_base = api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE") - if not resolved_api_base: - return False - hostname: Final = urlparse(resolved_api_base).hostname - if hostname is None: - return False - return hostname != "openai.com" and not hostname.endswith(".openai.com") + return custom_llm_provider == "openai" and not self._targets_openai_hosted_endpoint( + custom_llm_provider, api_base + ) + + def _flattened_tools_update_for_openai( + self, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + ) -> Mapping[str, object]: + """ + OpenAI's chat completions validator rejects tool `parameters` carrying + 'oneOf'/'anyOf'/'allOf'/'enum'/'const'/'not' at the top level for every + model family, unlike the Responses API, where GPT-5+ accepts them. + """ + tools: Final = optional_params.get("tools") + if not isinstance(tools, list): + return _NO_TOOLS_UPDATE + provider: Final = litellm_params.get("custom_llm_provider") + raw_api_base: Final = litellm_params.get("api_base") + if not self._targets_openai_hosted_endpoint( + provider if isinstance(provider, str) else None, + raw_api_base if isinstance(raw_api_base, str) else None, + ): + return _NO_TOOLS_UPDATE + flattened: Final = [ # mutable-ok: request tools are a JSON list + _tool_with_flattened_parameters(tool) if isinstance(tool, dict) else tool for tool in tools + ] + return MappingProxyType({"tools": flattened}) def transform_request( self, @@ -450,6 +502,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): "model": model, "messages": messages, **optional_params, + **self._flattened_tools_update_for_openai(optional_params, litellm_params), } async def async_transform_request( @@ -481,6 +534,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): "model": model, "messages": transformed_messages, **optional_params, + **self._flattened_tools_update_for_openai(optional_params, litellm_params), } else: ## allow for any object specific behaviour to be handled diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index d7ad69593c6..ef03e61a858 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -153,14 +153,17 @@ class VertexAIAnthropicConfig(AnthropicConfig): drop_params: bool, ) -> dict: """ - Override parent method to ensure VertexAI always uses tool-based structured outputs. - VertexAI doesn't support the output_format parameter, so we force all models - to use the tool-based approach for structured outputs. + Override parent method so VertexAI uses tool-based structured outputs + unless the vertex map entry advertises native structured output + (``output_format``, which Vertex AI Claude forwards for those models). """ - # Temporarily override model name to force tool-based approach - # This ensures Claude Sonnet 4.5 uses tools instead of output_format + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + original_model: Final = model - if "response_format" in non_default_params: + native_structured_output: Final = AnthropicModelInfo._get_provider_resolved_capability( + model, "supports_native_structured_output", "vertex_ai" + ) + if "response_format" in non_default_params and native_structured_output is not True: model = "claude-3-sonnet-20240229" # Use a model that will use tool-based approach # Call parent method with potentially modified model name diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1b2001cdadd..55618d9f772 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1482,7 +1482,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -1557,7 +1557,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -1632,7 +1632,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -1707,7 +1707,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -3254,6 +3254,7 @@ "supports_computer_use": true, "supports_forced_tool_use": false, "supports_function_calling": true, + "supports_native_structured_output": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -44132,6 +44133,7 @@ "supports_computer_use": true, "supports_forced_tool_use": false, "supports_function_calling": true, + "supports_native_structured_output": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -44202,6 +44204,7 @@ "supports_computer_use": true, "supports_forced_tool_use": false, "supports_function_calling": true, + "supports_native_structured_output": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 14d2332a7eb..ca66640bf46 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -54,7 +54,10 @@ from litellm.proxy.common_utils.config_sync_pubsub import ( coordination_redis_cache, publish_config_change, ) -from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper +from litellm.proxy.common_utils.encrypt_decrypt_utils import ( + decrypt_value_helper, + encrypt_value_helper, +) from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.management_endpoints.common_utils import _is_user_team_admin from litellm.proxy.management_endpoints.team_endpoints import ( @@ -701,6 +704,12 @@ async def patch_model( param="blocked", ) + ModelManagementAuthChecks.can_user_attach_credential( + litellm_params=patch_data.litellm_params, + user_api_key_dict=user_api_key_dict, + existing_litellm_params=db_model.litellm_params, + ) + _raise_on_strategy_router_write_violation( incoming_params=patch_data.litellm_params, existing_params=db_model.litellm_params, @@ -1464,6 +1473,32 @@ class ModelManagementAuthChecks: ) return True + @staticmethod + def can_user_attach_credential( + litellm_params: GenericLiteLLMParams | None, + user_api_key_dict: UserAPIKeyAuth, + existing_litellm_params: GenericLiteLLMParams | None = None, + ) -> Literal[True]: + if litellm_params is None or litellm_params.litellm_credential_name is None: + return True + if existing_litellm_params is not None and existing_litellm_params.litellm_credential_name is not None: + existing_credential_name: Final = decrypt_value_helper( + value=existing_litellm_params.litellm_credential_name, + key="litellm_credential_name", + exception_type="debug", + return_original_value=True, + ) + if litellm_params.litellm_credential_name == existing_credential_name: + return True + if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN: + return True + raise ProxyException( + message=f"Only a proxy admin can attach a stored credential (litellm_credential_name) to a model. Your role={user_api_key_dict.user_role}.", + type=ProxyErrorTypes.auth_error.value, + code=status.HTTP_403_FORBIDDEN, + param="litellm_credential_name", + ) + @staticmethod async def allow_team_model_action( model_params: Deployment | updateDeployment, @@ -1786,6 +1821,11 @@ async def add_new_model( premium_user=premium_user, ) + ModelManagementAuthChecks.can_user_attach_credential( + litellm_params=model_params.litellm_params, + user_api_key_dict=user_api_key_dict, + ) + _raise_on_strategy_router_write_violation( incoming_params=model_params.litellm_params, existing_params=None, @@ -1958,6 +1998,12 @@ async def update_model( premium_user=premium_user, ) + ModelManagementAuthChecks.can_user_attach_credential( + litellm_params=model_params.litellm_params, + user_api_key_dict=user_api_key_dict, + existing_litellm_params=deployment.litellm_params, + ) + _raise_on_strategy_router_write_violation( incoming_params=model_params.litellm_params, existing_params=deployment.litellm_params, diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 86f6853a625..8ac63ba25c9 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -913,14 +913,13 @@ class ProxyInitializationHelpers: envvar="ENFORCE_PRISMA_MIGRATION_CHECK", ) @click.option( - "--use_v2_migration_resolver/--use_legacy_migration_resolver", - default=True, + "--use_v2_migration_resolver", + is_flag=True, + default=False, help=( - "Which database migration resolver to run at startup. The default v2 " - "resolver avoids the diff-and-force recovery path that can cause schema " - "thrashing during rolling deploys where two LiteLLM versions contend for " - "the same DB. Pass --use_legacy_migration_resolver, or set " - "USE_V2_MIGRATION_RESOLVER=false, to fall back to v1." + "Opt into the v2 migration resolver. Avoids the diff-and-force recovery " + "path that can cause schema thrashing during rolling deploys where two " + "LiteLLM versions contend for the same DB. Default is the v1 resolver." ), envvar="USE_V2_MIGRATION_RESOLVER", ) @@ -1311,11 +1310,10 @@ def run_server( else: if not use_v2_migration_resolver: print( - "\033[1;33mLiteLLM Proxy: Using the legacy (v1) migration resolver. " - "The default v2 resolver is safer: it avoids the diff-and-force " - "recovery that caused schema thrashing during rolling deploys. " - "Remove --use_legacy_migration_resolver / " - "USE_V2_MIGRATION_RESOLVER=false to switch back to it.\033[0m" + "\033[1;33mLiteLLM Proxy: Using default (v1) migration resolver. " + "If your deployment has seen schema thrashing during rolling " + "deploys, try --use_v2_migration_resolver (safer: avoids the " + "diff-and-force recovery that caused the thrash).\033[0m" ) try: setup_ok: Final = PrismaManager.setup_database( @@ -1323,10 +1321,10 @@ def run_server( use_v2_resolver=use_v2_migration_resolver, ) except RuntimeError as e: - # Raised on unrecoverable migration errors: permission - # failures from either resolver, the v2 resolver's - # non-idempotent failures, and any `prisma db push` - # against a partitioned LiteLLM_SpendLogs. + # Raised on unrecoverable migration errors: the v2 + # resolver's non-idempotent failures and permission + # issues, and any `prisma db push` against a + # partitioned LiteLLM_SpendLogs. print( f"\033[1;31mLiteLLM Proxy: Database migration cannot proceed. {e}\033[0m", file=sys.stderr, diff --git a/litellm/router.py b/litellm/router.py index edff8294c3e..471a1116f44 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8165,6 +8165,52 @@ class Router: if backend_value is not None: model_info[field] = backend_value + @staticmethod + def _inherit_builtin_base_rates_for_off_peak( + model_info: dict, # mutable-ok: cost-map entry filled in place + backend_model: str, + custom_llm_provider: str | None, + ) -> None: + """Fill missing pricing fields on a deployment entry that only sets + ``off_peak_pricing``, from the backend model's built-in cost map entry. + + Cost lookup selects the deployment-scoped entry over the shared backend + entry only when the deployment entry carries a base pricing field, and + ``off_peak_pricing`` is deliberately kept off the shared entry, so a + deployment spelling out only its off-peak schedule would otherwise + never receive the discount. The backend model's entire canonical cost + map entry is copied, field by field, so threshold, tiered, + service-tier, cache, character, and per-second rates as well as + companion billing fields like ``web_search_billing_unit`` and the + regional uplift multipliers all carry over, and peak-hour billing + through the deployment entry matches the shared backend entry exactly. + The raw ``litellm.model_cost`` entry is the copy source rather than + ``get_model_info``'s view of it, since that view synthesizes zero flat + token rates for backends without one and storing those would mark a + tiered-only backend explicitly priced free. Values are deep-copied to + keep the builtin entry isolated. User-specified fields always win; + no-op when any base pricing field is already set or the backend model + has no canonical entry. + """ + if not model_info.get("off_peak_pricing"): + return + if any( + model_info.get(field) is not None + for field in ("input_cost_per_token", "input_cost_per_second", "tiered_pricing") + ): + return + try: + backend_info: Final = litellm.get_model_info(model=backend_model, custom_llm_provider=custom_llm_provider) + except Exception: # noqa: BLE001 # get_model_info raises plain Exception for an unmapped backend model + return + backend_entry: Final = litellm.model_cost.get(backend_info.get("key") or "") + if not isinstance(backend_entry, dict): + return + for field, backend_value in backend_entry.items(): + if model_info.get(field) is not None or backend_value is None: + continue + model_info[field] = copy.deepcopy(backend_value) + @staticmethod def _inherit_builtin_tiered_output_rate( model_info: dict, backend_model: str, custom_llm_provider: str | None @@ -8253,6 +8299,11 @@ class Router: if deployment.litellm_params.get(field) is not None: _model_info[field] = deployment.litellm_params[field] + Router._inherit_builtin_base_rates_for_off_peak( + model_info=_model_info, + backend_model=deployment.litellm_params.model, + custom_llm_provider=deployment.litellm_params.custom_llm_provider, + ) if _model_info.get("input_cost_per_token") is not None: Router._inherit_builtin_cache_pricing( model_info=_model_info, @@ -8994,6 +9045,11 @@ class Router: if field_value is not None: _model_info_dict[field] = field_value + Router._inherit_builtin_base_rates_for_off_peak( + model_info=_model_info_dict, + backend_model=deployment.litellm_params.model, + custom_llm_provider=deployment.litellm_params.custom_llm_provider, + ) if _model_info_dict.get("input_cost_per_token") is not None: Router._inherit_builtin_cache_pricing( model_info=_model_info_dict, @@ -9249,6 +9305,11 @@ class Router: field_value = deployment.litellm_params.get(field) if field_value is not None: model_info[field] = field_value + Router._inherit_builtin_base_rates_for_off_peak( + model_info=model_info, + backend_model=deployment.litellm_params.model, + custom_llm_provider=deployment.litellm_params.custom_llm_provider, + ) if model_info.get("input_cost_per_token") is not None: Router._inherit_builtin_cache_pricing( model_info=model_info, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index addb7b730de..3a0883b6607 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -193,6 +193,38 @@ class AgenticLoopParams(TypedDict, total=False): """The LLM provider name (e.g., 'bedrock', 'anthropic')""" +class OffPeakWindow(TypedDict, total=False): + """One off-peak rule: UTC time-of-day windows, optionally restricted to weekdays. + + hours_utc is a "HH:MM-HH:MM" string in UTC, or a list of them; a window may wrap past + midnight and an equal-ended window covers the whole day. weekdays is a list of days the + rule applies on, as ISO-8601 numbers (1 = Monday .. 7 = Sunday) or English day names; + omitted means every day. The weekday is read on the calendar named by the block's + weekday_timezone. + """ + + hours_utc: ReadOnly[str | Sequence[str]] + weekdays: ReadOnly[Sequence[int | str]] + + +class OffPeakPricing(TypedDict, total=False): + """Time-windowed off-peak rates for providers that discount by time of day (e.g. DeepSeek). + + hours_utc is a "HH:MM-HH:MM" string in UTC, or a list of them for multiple daily windows, + applying on every day of the week; a window may wrap past midnight. windows adds + day-of-week-qualified rules (e.g. weekend-only whole-day off-peak), matched as a union + with hours_utc. weekday_timezone names the IANA calendar weekdays are read on, defaulting + to UTC. Any rate left unset falls back to the standard rate. + """ + + hours_utc: ReadOnly[str | Sequence[str]] + windows: ReadOnly[Sequence[OffPeakWindow]] + weekday_timezone: ReadOnly[str] + input_cost_per_token: ReadOnly[float] + output_cost_per_token: ReadOnly[float] + cache_read_input_token_cost: ReadOnly[float] + + class ModelInfoBase(ProviderSpecificModelInfo, total=False): key: Required[str] # the key in litellm.model_cost which is returned @@ -225,6 +257,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): # Smallest prefix this model will actually cache, whatever caching mechanism its provider uses. # Absent means the provider-agnostic default applies; see MINIMUM_PROMPT_CACHE_TOKEN_COUNT. prompt_cache_min_tokens: int | None + off_peak_pricing: ReadOnly[OffPeakPricing | None] # time-windowed off-peak rates input_cost_per_character: float | None # only for vertex ai models input_cost_per_audio_token: float | None input_cost_per_token_above_128k_tokens: float | None # only for vertex ai models @@ -3486,17 +3519,22 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): return {k: v for k, v in model_info.items() if k not in cls.model_fields} -SHARED_BACKEND_MODEL_INFO_FIELDS: Final[frozenset[str]] = frozenset( - ModelInfoBase.__required_keys__ | ModelInfoBase.__optional_keys__ -) - frozenset(CustomPricingLiteLLMParams.model_fields) +DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing"}) + +SHARED_BACKEND_MODEL_INFO_FIELDS: Final[frozenset[str]] = ( + frozenset(ModelInfoBase.__required_keys__ | ModelInfoBase.__optional_keys__) + - frozenset(CustomPricingLiteLLMParams.model_fields) + - DEPLOYMENT_SCOPED_PRICING_FIELDS +) def shared_backend_model_info(model_info: dict[str, Any]) -> dict[str, Any]: """Return only the fields safe to register under a shared ``{provider}/{model}`` key in ``litellm.model_cost``: cost-map schema fields (``ModelInfoBase``) minus - per-deployment pricing overrides. Per-deployment metadata (``id``, - ``access_via_team_ids``, arbitrary custom keys) never belongs on the shared key; - it stays under the deployment's unique model id. + per-deployment pricing overrides and deployment-scoped pricing blocks such as + ``off_peak_pricing``. Per-deployment metadata (``id``, ``access_via_team_ids``, + arbitrary custom keys) never belongs on the shared key; it stays under the + deployment's unique model id. """ return {k: v for k, v in model_info.items() if k in SHARED_BACKEND_MODEL_INFO_FIELDS} diff --git a/litellm/utils.py b/litellm/utils.py index 028682661f7..2fdc4a45148 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2869,10 +2869,9 @@ def _update_dictionary(existing_dict: dict, new_dict: dict) -> dict: elif isinstance(v, dict): existing_nested_dict = existing_dict.get(k) if isinstance(existing_nested_dict, dict): - existing_nested_dict.update(v) - existing_dict[k] = existing_nested_dict + existing_dict[k] = {**existing_nested_dict, **v} # mutable-ok: copy-on-write merge else: - existing_dict[k] = v + existing_dict[k] = dict(v) # mutable-ok: detached copy, never the caller's dict by reference else: existing_dict[k] = v @@ -5860,6 +5859,7 @@ def _get_model_info_helper( cache_creation_input_token_cost_above_1hr=_model_info.get( "cache_creation_input_token_cost_above_1hr", None ), + off_peak_pricing=_model_info.get("off_peak_pricing", None), input_cost_per_character=_model_info.get("input_cost_per_character", None), input_cost_per_token_above_128k_tokens=_model_info.get("input_cost_per_token_above_128k_tokens", None), input_cost_per_token_above_200k_tokens=_model_info.get("input_cost_per_token_above_200k_tokens", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1b2001cdadd..55618d9f772 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1482,7 +1482,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -1557,7 +1557,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -1632,7 +1632,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -1707,7 +1707,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -3254,6 +3254,7 @@ "supports_computer_use": true, "supports_forced_tool_use": false, "supports_function_calling": true, + "supports_native_structured_output": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -44132,6 +44133,7 @@ "supports_computer_use": true, "supports_forced_tool_use": false, "supports_function_calling": true, + "supports_native_structured_output": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -44202,6 +44204,7 @@ "supports_computer_use": true, "supports_forced_tool_use": false, "supports_function_calling": true, + "supports_native_structured_output": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, diff --git a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py index 05886e4b7f6..58cde4c8103 100644 --- a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py +++ b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py @@ -40,6 +40,24 @@ def prometheus_logger() -> PrometheusLogger: return PrometheusLogger() +@pytest.fixture +def known_model_router(): + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-5-mini", + "litellm_params": {"model": "openai/gpt-5-mini", "api_key": "fake-key"}, + }, + { + "model_name": "us/azure/openai/gpt-5-mini", + "litellm_params": {"model": "openai/gpt-5-mini", "api_key": "fake-key"}, + }, + ] + ) + with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + yield router + + def create_standard_logging_payload() -> StandardLoggingPayload: return StandardLoggingPayload( id="test_id", @@ -741,7 +759,7 @@ async def test_async_log_failure_event(prometheus_logger): @pytest.mark.asyncio -async def test_async_log_failure_event_litellm_side_rate_limit(prometheus_logger): +async def test_async_log_failure_event_litellm_side_rate_limit(prometheus_logger, known_model_router): """LiteLLM-side reject (no deployment picked) routes the requested model into `requested_model` and skips the partial-outage flag.""" standard_logging_object = create_standard_logging_payload() @@ -786,7 +804,7 @@ async def test_async_log_failure_event_litellm_side_rate_limit(prometheus_logger @pytest.mark.asyncio -async def test_async_post_call_failure_hook(prometheus_logger): +async def test_async_post_call_failure_hook(prometheus_logger, known_model_router): """ Test for the async_post_call_failure_hook method @@ -1069,7 +1087,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger): @pytest.mark.asyncio -async def test_log_success_fallback_event(prometheus_logger): +async def test_log_success_fallback_event(prometheus_logger, known_model_router): prometheus_logger.litellm_deployment_successful_fallbacks = MagicMock() original_model_group = "gpt-5-mini" @@ -1107,7 +1125,7 @@ async def test_log_success_fallback_event(prometheus_logger): @pytest.mark.asyncio -async def test_log_failure_fallback_event(prometheus_logger): +async def test_log_failure_fallback_event(prometheus_logger, known_model_router): prometheus_logger.litellm_deployment_failed_fallbacks = MagicMock() original_model_group = "gpt-5-mini" diff --git a/tests/litellm-proxy-extras/test_setup_database_fail_fast.py b/tests/litellm-proxy-extras/test_setup_database_fail_fast.py deleted file mode 100644 index ef447315a8c..00000000000 --- a/tests/litellm-proxy-extras/test_setup_database_fail_fast.py +++ /dev/null @@ -1,571 +0,0 @@ -"""Regression tests for ProxyExtrasDBManager's v2 migration resolver. - -v2 is the proxy CLI default; v1 stays reachable via the `use_v2_resolver` -kwarg, which still defaults to False for direct callers. -""" - -import subprocess -from unittest.mock import patch - -import pytest - -from litellm_proxy_extras.utils import ( - _PRISMA_ATTEMPTS, - ProxyExtrasDBManager, - _max_migration_timestamp, - _migration_timestamp, -) - - -def _fake_migrate_deploy_failure(returncode: int, stderr: str): - def _run(*args, **kwargs): - raise subprocess.CalledProcessError( - returncode=returncode, - cmd=args[0], - stderr=stderr, - output="", - ) - - return _run - - -def test_v2_p3018_permission_error_raises_runtime_error(monkeypatch, tmp_path): - """v2: a permission failure during migrate deploy raises RuntimeError.""" - monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x") - monkeypatch.setattr( - ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None - ) - monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path)) - (tmp_path / "schema.prisma").write_text("// stub") - - stderr = ( - "Error: P3018\nMigration name: 20250326162113_baseline\n" - "Database error code: 42501\npermission denied for schema public" - ) - with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)): - with pytest.raises(RuntimeError, match="permission"): - ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) - - -def test_v2_non_idempotent_p3009_raises_runtime_error(monkeypatch, tmp_path): - """v2: a non-idempotent migration failure raises (no silent recovery).""" - monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x") - monkeypatch.setattr( - ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None - ) - monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path)) - (tmp_path / "schema.prisma").write_text("// stub") - - stderr = ( - "Error: P3009\nMigration `20260101000000_genuinely_broken` failed\n" - 'Reason: syntax error at or near "BRKN" LINE 42' - ) - with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)): - with pytest.raises(RuntimeError, match="cannot be auto-recovered"): - ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) - - -def test_strip_prisma_query_params_removes_connection_limit(): - """DATABASE_URLs with Prisma-specific params should be parseable by psycopg.""" - url = "postgresql://u:p@h:5432/db?connection_limit=100&pool_timeout=60&sslmode=require" - stripped = ProxyExtrasDBManager._strip_prisma_query_params(url) - assert "connection_limit" not in stripped - assert "pool_timeout" not in stripped - assert "sslmode=require" in stripped - - -def test_strip_prisma_query_params_passthrough_no_query(): - """URLs without query strings are returned unchanged.""" - url = "postgresql://u:p@h:5432/db" - assert ProxyExtrasDBManager._strip_prisma_query_params(url) == url - - -def test_migration_timestamp_extracts_leading_digits(): - assert _migration_timestamp("20260101000000_add_foo") == 20260101000000 - assert _migration_timestamp("20250326162113_baseline") == 20250326162113 - - -def test_migration_timestamp_returns_zero_on_malformed(): - assert _migration_timestamp("0_init") == 0 - assert _migration_timestamp("not_a_migration") == 0 - - -def test_max_migration_timestamp(): - names = {"20250326000000_a", "20260415000000_b", "20251115000000_c"} - assert _max_migration_timestamp(names) == 20260415000000 - - -def test_max_migration_timestamp_empty_set(): - assert _max_migration_timestamp(set()) == 0 - - -def test_v1_default_still_calls_resolve_all_migrations(monkeypatch, tmp_path): - """v1 (default) continues to call _resolve_all_migrations on the happy path. - - This is the existing buggy behavior — we're not fixing it in v1, only - offering v2 as opt-in. This test pins the default so that a future - inadvertent default flip is caught. - """ - monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path)) - (tmp_path / "schema.prisma").write_text("// stub") - - # Stub `prisma migrate deploy` to claim success with pending migrations - # applied, which is the code path that triggers the legacy post-migration - # sanity check (a call to _resolve_all_migrations). - class FakeResult: - stdout = "Applied migration.\n" - stderr = "" - - def fake_run(cmd, *args, **kwargs): - return FakeResult() - - resolve_called = {"n": 0} - - def fake_resolve(*args, **kwargs): - resolve_called["n"] += 1 - - monkeypatch.setattr("subprocess.run", fake_run) - monkeypatch.setattr(ProxyExtrasDBManager, "_resolve_all_migrations", fake_resolve) - - ok = ProxyExtrasDBManager.setup_database(use_migrate=True) # v2 flag NOT set - assert ok is True - assert resolve_called["n"] == 1, "v1 default should still invoke the legacy path" - - -def test_v2_db_push_wraps_subprocess_error_as_runtime_error(monkeypatch, tmp_path): - """v2: a failing `prisma db push` must raise RuntimeError, not leak - CalledProcessError past proxy_cli.py's `except RuntimeError`.""" - monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path)) - (tmp_path / "schema.prisma").write_text("// stub") - - stderr = "db push error" - with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)): - with pytest.raises(RuntimeError, match="prisma db push failed"): - ProxyExtrasDBManager.setup_database(use_migrate=False, use_v2_resolver=True) - - -def test_v2_warn_ahead_of_head_swallows_db_errors(monkeypatch, tmp_path): - """_warn_if_db_ahead_of_head must never raise — it's informational. - - Non-connection DB errors (e.g. InsufficientPrivilege from a user - without SELECT on _prisma_migrations) must be caught, not propagated. - """ - import psycopg - - monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x") - monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path)) - (tmp_path / "schema.prisma").write_text("// stub") - - class _FakeConn: - def __enter__(self): - return self - - def __exit__(self, *a): - return False - - def execute(self, *a, **kw): - # Simulate an InsufficientPrivilege (subclass of DatabaseError). - raise psycopg.errors.InsufficientPrivilege("permission denied") - - connects = {"n": 0} - - def _fake_connect(*a, **kw): - connects["n"] += 1 - return _FakeConn() - - monkeypatch.setattr("psycopg.connect", _fake_connect) - - assert ProxyExtrasDBManager._warn_if_db_ahead_of_head(str(tmp_path)) is None - assert connects["n"] == 1, "the failing query must actually have been reached" - - -def test_v2_resolve_specific_migration_failure_raises_runtime_error( - monkeypatch, tmp_path -): - """If marking a migration as applied fails inside P3009 idempotent - recovery, the subprocess error must be re-raised as RuntimeError so - proxy_cli.py catches it cleanly (instead of leaking CalledProcessError).""" - monkeypatch.setattr( - ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None - ) - monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path)) - (tmp_path / "schema.prisma").write_text("// stub") - monkeypatch.setattr( - ProxyExtrasDBManager, "_roll_back_migration", lambda *a, **kw: None - ) - - # First call: migrate deploy -> P3009 idempotent error. - # Recovery path tries _resolve_specific_migration; that also raises. - def _failing_resolve(*a, **kw): - raise subprocess.CalledProcessError( - returncode=1, - cmd="prisma migrate resolve --applied", - stderr="resolve failed", - output="", - ) - - monkeypatch.setattr( - ProxyExtrasDBManager, "_resolve_specific_migration", _failing_resolve - ) - - stderr = ( - "Error: P3009\nMigration `20260101000000_some_migration` failed\n" - "relation already exists" - ) - with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)): - with pytest.raises( - RuntimeError, match=r"Failed to mark migration .* as applied" - ): - ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) - - -def test_v2_does_not_call_resolve_all_migrations(monkeypatch, tmp_path): - """v2 must never call _resolve_all_migrations — that's the bug it fixes.""" - monkeypatch.setattr( - ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None - ) - monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path)) - (tmp_path / "schema.prisma").write_text("// stub") - - class FakeResult: - stdout = "Applied migration.\n" - stderr = "" - - monkeypatch.setattr("subprocess.run", lambda *a, **kw: FakeResult()) - - resolve_called = {"n": 0} - monkeypatch.setattr( - ProxyExtrasDBManager, - "_resolve_all_migrations", - lambda *a, **kw: resolve_called.__setitem__("n", resolve_called["n"] + 1), - ) - - ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) - assert ok is True - assert resolve_called["n"] == 0, "v2 must not invoke the diff-and-force recovery" - - -_DEADLOCK_STDERR = ( - "Error: ERROR: deadlock detected\n" - "DETAIL: Process 277 waits for ExclusiveLock on advisory lock " - "[17556,0,72707369,1]; blocked by process 278.\n" - "Process 278 waits for ShareLock on virtual transaction 3/1041; " - "blocked by process 277." -) - - -class _DeployApplied: - stdout = "All migrations have been successfully applied." - stderr = "" - returncode = 0 - - -def _deploy_only(deploy_side_effect): - """subprocess.run stand-in that only intercepts `prisma migrate deploy`. - - Scoped by argv so the Prisma toolchain check cannot consume the mock first. - """ - deploys = {"n": 0} - - def _run(*args, **kwargs): - cmd = args[0] if args else kwargs.get("args", []) - if list(cmd)[-2:] == ["migrate", "deploy"]: - deploys["n"] += 1 - return deploy_side_effect(deploys["n"], cmd) - return _DeployApplied() - - return _run, deploys - - -def _prepare_v2_resolver(monkeypatch, tmp_path): - monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x") - monkeypatch.setattr( - ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None - ) - monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path)) - (tmp_path / "schema.prisma").write_text("// stub") - monkeypatch.setattr("time.sleep", lambda *_a, **_k: None) - - -def test_v2_retries_transient_advisory_lock_deadlock(monkeypatch, tmp_path): - """v2: replicas racing `migrate deploy` deadlock on Prisma's advisory - lock, which is transient and must be retried rather than kill the boot.""" - _prepare_v2_resolver(monkeypatch, tmp_path) - - def _side_effect(n, cmd): - if n == 1: - raise subprocess.CalledProcessError( - returncode=1, cmd=cmd, stderr=_DEADLOCK_STDERR, output="" - ) - return _DeployApplied() - - run, deploys = _deploy_only(_side_effect) - with patch("subprocess.run", side_effect=run): - ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) - - assert ok is True - assert deploys["n"] == 2, "the deadlocked deploy must be retried, not raised" - - -def test_v2_persistent_advisory_lock_deadlock_eventually_raises(monkeypatch, tmp_path): - """v2: the deadlock retry is bounded, so a deadlock that never clears - still raises instead of looping or reporting success.""" - _prepare_v2_resolver(monkeypatch, tmp_path) - - def _side_effect(n, cmd): - raise subprocess.CalledProcessError( - returncode=1, cmd=cmd, stderr=_DEADLOCK_STDERR, output="" - ) - - run, deploys = _deploy_only(_side_effect) - with patch("subprocess.run", side_effect=run): - with pytest.raises(RuntimeError, match="after 4 attempts"): - ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) - - assert deploys["n"] == 4 - - -@pytest.mark.parametrize( - "stderr", - [ - "Error: P1001: Can't reach database server at `db`:`5432`", - "Error: P1002: The database server was reached but timed out.", - ], -) -def test_v2_retries_transient_database_connectivity_errors(monkeypatch, tmp_path, stderr): - """v2: a database not accepting connections yet is retried, not fatal.""" - _prepare_v2_resolver(monkeypatch, tmp_path) - - def _side_effect(n, cmd): - if n == 1: - raise subprocess.CalledProcessError( - returncode=1, cmd=cmd, stderr=stderr, output="" - ) - return _DeployApplied() - - run, deploys = _deploy_only(_side_effect) - with patch("subprocess.run", side_effect=run): - ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) - - assert ok is True - assert deploys["n"] == 2, "an unreachable database must be retried, not raised" - - -def test_v2_unreachable_database_still_fails_after_the_retries(monkeypatch, tmp_path): - """v2: a genuinely unreachable database still raises once the attempts - are spent, rather than passing as a successful migration.""" - _prepare_v2_resolver(monkeypatch, tmp_path) - - def _side_effect(n, cmd): - raise subprocess.CalledProcessError( - returncode=1, - cmd=cmd, - stderr="Error: P1001: Can't reach database server at `db`:`5432`", - output="", - ) - - run, deploys = _deploy_only(_side_effect) - with patch("subprocess.run", side_effect=run): - with pytest.raises(RuntimeError, match="after 4 attempts"): - ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) - - assert deploys["n"] == 4 - - -def test_v2_exhausted_retries_report_the_prisma_error(monkeypatch, tmp_path, caplog): - """v2: retrying must not swallow Prisma's stderr, which is captured and is - the only place the cause appears for an operator or a boot-log grep.""" - _prepare_v2_resolver(monkeypatch, tmp_path) - stderr = "Error: P1001: Can't reach database server at `wrong`:`5432`" - - def _side_effect(n, cmd): - raise subprocess.CalledProcessError( - returncode=1, cmd=cmd, stderr=stderr, output="" - ) - - run, _ = _deploy_only(_side_effect) - with caplog.at_level("INFO", logger="litellm_proxy_extras"): - with patch("subprocess.run", side_effect=run): - with pytest.raises(RuntimeError) as exc_info: - ProxyExtrasDBManager.setup_database( - use_migrate=True, use_v2_resolver=True - ) - - assert "P1001" in str(exc_info.value) - assert "P1001" in caplog.text - - -def test_v2_db_push_retries_transient_failures(monkeypatch, tmp_path): - """v2: `prisma db push` retries a transient failure like v1 did. - - Reached from the migrations Job (USE_PRISMA_DB_PUSH=true), not from the - proxy CLI, whose --use_prisma_db_push has its own loop in prisma_client. - """ - _prepare_v2_resolver(monkeypatch, tmp_path) - - pushes = {"n": 0} - - def _run(*args, **kwargs): - cmd = list(args[0] if args else kwargs.get("args", [])) - if cmd[-3:] != ["db", "push", "--accept-data-loss"]: - return _DeployApplied() - pushes["n"] += 1 - if pushes["n"] == 1: - raise subprocess.CalledProcessError( - returncode=1, - cmd=cmd, - stderr="Error: P1001: Can't reach database server at `db`:`5432`", - output="", - ) - return _DeployApplied() - - monkeypatch.setattr( - ProxyExtrasDBManager, "spend_logs_is_partitioned", lambda: False - ) - with patch("subprocess.run", side_effect=_run): - ok = ProxyExtrasDBManager.setup_database(use_migrate=False, use_v2_resolver=True) - - assert ok is True - assert pushes["n"] == 2 - - -def test_v2_db_push_retries_are_bounded_and_report_the_prisma_error( - monkeypatch, tmp_path -): - """v2: a database that never comes back stops after _PRISMA_ATTEMPTS and - surfaces the prisma error, rather than retrying the boot forever.""" - _prepare_v2_resolver(monkeypatch, tmp_path) - - pushes = {"n": 0} - - def _run(*args, **kwargs): - cmd = list(args[0] if args else kwargs.get("args", [])) - if cmd[-3:] != ["db", "push", "--accept-data-loss"]: - return _DeployApplied() - pushes["n"] += 1 - raise subprocess.CalledProcessError( - returncode=1, - cmd=cmd, - stderr="Error: P1001: Can't reach database server at `db`:`5432`", - output="", - ) - - monkeypatch.setattr( - ProxyExtrasDBManager, "spend_logs_is_partitioned", lambda: False - ) - with patch("subprocess.run", side_effect=_run): - with pytest.raises(RuntimeError) as exc: - ProxyExtrasDBManager.setup_database( - use_migrate=False, use_v2_resolver=True - ) - - assert pushes["n"] == _PRISMA_ATTEMPTS - assert "P1001" in str(exc.value) - - -def _db_push_only(push_side_effect): - """subprocess.run stand-in that only intercepts `prisma db push`.""" - pushes = {"n": 0} - - def _run(*args, **kwargs): - cmd = list(args[0] if args else kwargs.get("args", [])) - if cmd[-3:] != ["db", "push", "--accept-data-loss"]: - return _DeployApplied() - pushes["n"] += 1 - return push_side_effect(pushes["n"], cmd) - - return _run, pushes - - -def _timed_out_for_real(): - """Capture what subprocess.run really puts on a TimeoutExpired. - - Under text=True it still leaves stderr as bytes, unlike CalledProcessError, - so hardcoding a str here would test a shape production never sees. Derived - at import, before any test patches subprocess.run. - """ - try: - subprocess.run( - ["sh", "-c", "echo 'Error: P1001 unreachable' >&2; sleep 5"], - timeout=0.2, - check=True, - capture_output=True, - text=True, - ) - except subprocess.TimeoutExpired as e: - return e - raise AssertionError("the helper command was supposed to time out") - - -_TIMEOUT_TEMPLATE = _timed_out_for_real() - - -def _real_timeout_expired(cmd): - return subprocess.TimeoutExpired( - cmd=cmd, - timeout=_TIMEOUT_TEMPLATE.timeout, - output=_TIMEOUT_TEMPLATE.stdout, - stderr=_TIMEOUT_TEMPLATE.stderr, - ) - - -def test_v2_db_push_retries_a_timeout(monkeypatch, tmp_path): - """v2: a `prisma db push` that times out is retried, not turned into a - TypeError by classifying its bytes stderr as if it were text.""" - _prepare_v2_resolver(monkeypatch, tmp_path) - - def _side_effect(n, cmd): - if n == 1: - raise _real_timeout_expired(cmd) - return _DeployApplied() - - monkeypatch.setattr( - ProxyExtrasDBManager, "spend_logs_is_partitioned", lambda: False - ) - run, pushes = _db_push_only(_side_effect) - with patch("subprocess.run", side_effect=run): - ok = ProxyExtrasDBManager.setup_database(use_migrate=False, use_v2_resolver=True) - - assert ok is True - assert pushes["n"] == 2 - - -def test_v2_db_push_timeouts_are_bounded(monkeypatch, tmp_path): - """v2: a `prisma db push` that never stops timing out gives up as a - RuntimeError, which is the only exception proxy_cli.py exits cleanly on.""" - _prepare_v2_resolver(monkeypatch, tmp_path) - - def _side_effect(n, cmd): - raise _real_timeout_expired(cmd) - - monkeypatch.setattr( - ProxyExtrasDBManager, "spend_logs_is_partitioned", lambda: False - ) - run, pushes = _db_push_only(_side_effect) - with patch("subprocess.run", side_effect=run): - with pytest.raises(RuntimeError, match=r"prisma db push failed after \d+"): - ProxyExtrasDBManager.setup_database(use_migrate=False, use_v2_resolver=True) - - assert pushes["n"] == _PRISMA_ATTEMPTS - - -def test_v2_unclassified_failure_is_not_treated_as_transient(monkeypatch, tmp_path): - """v2: an unrecognised deploy failure still raises on the first attempt.""" - _prepare_v2_resolver(monkeypatch, tmp_path) - - def _side_effect(n, cmd): - raise subprocess.CalledProcessError( - returncode=1, - cmd=cmd, - stderr="Error: relation \"LiteLLM_SpendLogs\" does not exist", - output="", - ) - - run, deploys = _deploy_only(_side_effect) - with patch("subprocess.run", side_effect=run): - with pytest.raises(RuntimeError, match="cannot be auto-recovered"): - ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) - - assert deploys["n"] == 1 - - diff --git a/tests/local_testing/test_basic_python_version.py b/tests/local_testing/test_basic_python_version.py index 506c58d26b4..fb06ed6b69d 100644 --- a/tests/local_testing/test_basic_python_version.py +++ b/tests/local_testing/test_basic_python_version.py @@ -305,16 +305,14 @@ def _run_proxy_server_smoke_test(extra_proxy_args=None): def test_litellm_proxy_server_config_no_general_settings(): - """Exercises the default (v2) migration resolver.""" + """Exercises the default (v1) migration resolver.""" _run_proxy_server_smoke_test() -def test_litellm_proxy_server_config_no_general_settings_legacy_resolver(): - """Exercises the legacy (v1) migration resolver against a real database. +def test_litellm_proxy_server_config_no_general_settings_v2_resolver(): + """Exercises the opt-in v2 migration resolver. - v2 is the default, so the no-arg test above already covers it. This one is - the only place the v1 opt-out gets real-DB migration plus proxy-boot - coverage, and it runs in a separate CI job against its own Postgres to - avoid collisions with the default variant. + Runs in a separate CI job against a local Postgres to avoid collisions + with the v1 variant when they share a database. """ - _run_proxy_server_smoke_test(extra_proxy_args=["--use_legacy_migration_resolver"]) + _run_proxy_server_smoke_test(extra_proxy_args=["--use_v2_migration_resolver"]) diff --git a/tests/test_litellm/integrations/test_prometheus_requested_model_cardinality.py b/tests/test_litellm/integrations/test_prometheus_requested_model_cardinality.py new file mode 100644 index 00000000000..519a13751f1 --- /dev/null +++ b/tests/test_litellm/integrations/test_prometheus_requested_model_cardinality.py @@ -0,0 +1,279 @@ +""" +LIT-6611: every unique client-supplied model name that fails routing used to +mint permanent Prometheus series carrying ``requested_model=""`` on the +proxy request metrics and the deployment metrics, with no eviction. The fix +collapses any requested model the router does not recognize (and no wildcard +pattern matches) into the single ``other`` label bucket, while recognized +names, aliases, and wildcard-matched names keep their own label values. +""" + +import sys +import types +from unittest.mock import patch + +import pytest +from prometheus_client import REGISTRY + +import litellm +from litellm.integrations.prometheus import ( + UNRECOGNIZED_REQUESTED_MODEL_LABEL, + PrometheusLogger, +) +from litellm.proxy._types import UserAPIKeyAuth + + +class _ClientSideError(Exception): + status_code = 400 + + +@pytest.fixture(autouse=True) +def cleanup_prometheus_registry(): + for collector in list(REGISTRY._collector_to_names.keys()): + try: + REGISTRY.unregister(collector) + except Exception: + pass + + yield + + for collector in list(REGISTRY._collector_to_names.keys()): + try: + REGISTRY.unregister(collector) + except Exception: + pass + + +@pytest.fixture +def router(): + return litellm.Router( + model_list=[ + { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "fake-key"}, + }, + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "api_key": "fake-key"}, + }, + ], + model_group_alias={"gpt4o-alias": "gpt-4o-mini"}, + ) + + +@pytest.fixture +def team_router(): + return litellm.Router( + model_list=[ + { + "model_name": "team-internal-gpt", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "fake-key"}, + "model_info": {"team_id": "team-1", "team_public_model_name": "team-alias-gpt"}, + }, + { + "model_name": "team-internal-bedrock", + "litellm_params": {"model": "openai/*", "api_key": "fake-key"}, + "model_info": {"team_id": "team-1", "team_public_model_name": "team-models/*"}, + }, + ] + ) + + +def _requested_model_values(metric) -> set[str]: + index = metric._labelnames.index("requested_model") + return {sample_key[index] for sample_key in metric._metrics} + + +def _series_count(metric) -> int: + return len(metric._metrics) + + +def _total_value(metric) -> float: + return sum(child._value.get() for child in metric._metrics.values()) + + +async def _fire_proxy_failure(logger: PrometheusLogger, model: str) -> None: + await logger.async_post_call_failure_hook( + request_data={"model": model, "metadata": {}, "proxy_server_request": {}}, + original_exception=_ClientSideError(f"model {model} does not exist"), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key-1"), + ) + + +@pytest.mark.asyncio +async def test_unknown_models_collapse_to_one_series_on_proxy_request_metrics(router): + logger = PrometheusLogger() + + with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + for index in range(25): + await _fire_proxy_failure(logger, f"agent-typo-{index}") + + for metric in ( + logger.litellm_proxy_failed_requests_metric, + logger.litellm_proxy_total_requests_metric, + ): + assert _requested_model_values(metric) == {UNRECOGNIZED_REQUESTED_MODEL_LABEL} + assert _series_count(metric) == 1 + assert _total_value(metric) == 25 + + +@pytest.mark.asyncio +async def test_known_alias_and_wildcard_models_keep_their_own_labels(router): + logger = PrometheusLogger() + + with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + await _fire_proxy_failure(logger, "gpt-4o-mini") + await _fire_proxy_failure(logger, "gpt4o-alias") + await _fire_proxy_failure(logger, "openai/gpt-4o-audio-preview") + await _fire_proxy_failure(logger, "agent-typo-hallucinated") + + for metric in ( + logger.litellm_proxy_failed_requests_metric, + logger.litellm_proxy_total_requests_metric, + ): + assert _requested_model_values(metric) == { + "gpt-4o-mini", + "gpt4o-alias", + "openai/gpt-4o-audio-preview", + UNRECOGNIZED_REQUESTED_MODEL_LABEL, + } + + +@pytest.mark.asyncio +async def test_team_alias_and_team_wildcard_models_keep_their_own_labels(team_router): + logger = PrometheusLogger() + + with patch("litellm.proxy.proxy_server.llm_router", team_router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + await _fire_proxy_failure(logger, "team-alias-gpt") + await _fire_proxy_failure(logger, "team-models/gpt-4o-audio-preview") + await _fire_proxy_failure(logger, "agent-typo-hallucinated") + + for metric in ( + logger.litellm_proxy_failed_requests_metric, + logger.litellm_proxy_total_requests_metric, + ): + assert _requested_model_values(metric) == { + "team-alias-gpt", + "team-models/gpt-4o-audio-preview", + UNRECOGNIZED_REQUESTED_MODEL_LABEL, + } + + +@pytest.mark.asyncio +async def test_unknown_models_collapse_to_other_when_router_is_unavailable(): + logger = PrometheusLogger() + + with patch("litellm.proxy.proxy_server.llm_router", None, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + await _fire_proxy_failure(logger, "agent-typo-no-router") + await _fire_proxy_failure(logger, "gpt-4o-mini") + + assert _requested_model_values(logger.litellm_proxy_failed_requests_metric) == { + UNRECOGNIZED_REQUESTED_MODEL_LABEL + } + + +@pytest.mark.asyncio +async def test_sdk_router_originated_metrics_keep_labels_without_proxy_router(): + logger = PrometheusLogger() + + with patch("litellm.proxy.proxy_server.llm_router", None, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + logger.set_llm_deployment_failure_metrics( + request_kwargs={ + "model": "sdk-deployment-group", + "litellm_params": {"metadata": {}}, + "standard_logging_object": {}, + "exception": _ClientSideError("model does not exist"), + } + ) + await logger.log_failure_fallback_event( + original_model_group="sdk-fallback-group", + kwargs={"model": "sdk-fallback-group", "metadata": {}}, + original_exception=_ClientSideError("upstream unavailable"), + ) + + assert _requested_model_values(logger.litellm_deployment_failure_responses) == {"sdk-deployment-group"} + assert _requested_model_values(logger.litellm_deployment_failed_fallbacks) == {"sdk-fallback-group"} + + +@pytest.mark.asyncio +async def test_sdk_fallback_labels_survive_non_import_errors_from_proxy_module(monkeypatch): + logger = PrometheusLogger() + broken_proxy_module = types.ModuleType("litellm.proxy.proxy_server") + + def _raise_value_error(_name: str): + raise ValueError("bad proxy env var") + + broken_proxy_module.__getattr__ = _raise_value_error # test-quality-ok: reproduces a proxy_server import raising non-ImportError, no injection seam + monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", broken_proxy_module) # test-quality-ok: reproduces a proxy_server import raising non-ImportError, no injection seam + + await logger.log_failure_fallback_event( + original_model_group="sdk-fallback-group", + kwargs={"model": "sdk-fallback-group", "metadata": {}}, + original_exception=_ClientSideError("upstream unavailable"), + ) + + assert _requested_model_values(logger.litellm_deployment_failed_fallbacks) == {"sdk-fallback-group"} + + +def test_unknown_models_collapse_to_one_series_on_deployment_metrics(router): + logger = PrometheusLogger() + + with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + for index in range(25): + logger.set_llm_deployment_failure_metrics( + request_kwargs={ + "model": f"agent-typo-{index}", + "litellm_params": {"metadata": {}}, + "standard_logging_object": {}, + "exception": _ClientSideError("model does not exist"), + } + ) + logger.set_llm_deployment_failure_metrics( + request_kwargs={ + "model": "gpt-4o-mini", + "litellm_params": {"metadata": {}}, + "standard_logging_object": {}, + "exception": _ClientSideError("all deployments cooling down"), + } + ) + + for metric in ( + logger.litellm_deployment_failure_responses, + logger.litellm_deployment_total_requests, + ): + assert _requested_model_values(metric) == { + UNRECOGNIZED_REQUESTED_MODEL_LABEL, + "gpt-4o-mini", + } + assert _series_count(metric) == 2 + assert _total_value(metric) == 26 + + +@pytest.mark.asyncio +async def test_fallback_event_requested_model_is_bounded(router): + logger = PrometheusLogger() + kwargs = {"model": "gpt-4o-mini", "metadata": {}} + + with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + await logger.log_failure_fallback_event( + original_model_group="agent-typo-hallucinated", + kwargs=kwargs, + original_exception=_ClientSideError("model does not exist"), + ) + await logger.log_success_fallback_event( + original_model_group="agent-typo-hallucinated", + kwargs=kwargs, + original_exception=_ClientSideError("model does not exist"), + ) + await logger.log_failure_fallback_event( + original_model_group="gpt-4o-mini", + kwargs=kwargs, + original_exception=_ClientSideError("upstream unavailable"), + ) + + assert _requested_model_values(logger.litellm_deployment_failed_fallbacks) == { + UNRECOGNIZED_REQUESTED_MODEL_LABEL, + "gpt-4o-mini", + } + assert _requested_model_values(logger.litellm_deployment_successful_fallbacks) == { + UNRECOGNIZED_REQUESTED_MODEL_LABEL + } diff --git a/tests/test_litellm/integrations/test_s3.py b/tests/test_litellm/integrations/test_s3.py index 7e997870852..58b15b79e76 100644 --- a/tests/test_litellm/integrations/test_s3.py +++ b/tests/test_litellm/integrations/test_s3.py @@ -2,26 +2,27 @@ from datetime import datetime from unittest.mock import MagicMock, patch import litellm +from litellm.constants import MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES, MAX_S3_OBJECT_KEY_BYTES from litellm.integrations.s3 import S3Logger TEST_KMS_KEY_ARN = "arn:aws:kms:us-east-1:111122223333:key/test-key-id" -def _standard_logging_payload() -> dict: +def _standard_logging_payload(response_id: str = "chatcmpl-test-id") -> dict: return { - "id": "chatcmpl-test-id", + "id": response_id, "metadata": {"user_api_key_team_alias": None}, } -def _log_event_kwargs() -> dict: +def _log_event_kwargs(response_id: str = "chatcmpl-test-id") -> dict: return { "litellm_params": {"metadata": {}}, - "standard_logging_object": _standard_logging_payload(), + "standard_logging_object": _standard_logging_payload(response_id), } -def _run_log_event(callback_params: dict) -> MagicMock: +def _run_log_event(callback_params: dict, response_id: str = "chatcmpl-test-id") -> MagicMock: original = litellm.s3_callback_params litellm.s3_callback_params = callback_params try: @@ -30,8 +31,8 @@ def _run_log_event(callback_params: dict) -> MagicMock: mock_boto3_client.return_value = mock_s3_client logger = S3Logger() logger.log_event( - kwargs=_log_event_kwargs(), - response_obj={}, + kwargs=_log_event_kwargs(response_id), + response_obj={"id": response_id}, start_time=datetime(2026, 7, 30, 12, 0, 0), end_time=datetime(2026, 7, 30, 12, 0, 1), print_verbose=lambda *args, **kwargs: None, @@ -154,3 +155,30 @@ def test_non_string_key_id_is_dropped_and_valid_algorithm_is_kept(): put_object_kwargs = mock_s3_client.put_object.call_args.kwargs assert put_object_kwargs["ServerSideEncryption"] == "aws:kms" assert "SSEKMSKeyId" not in put_object_kwargs + + +def test_put_object_key_and_filename_are_bounded_for_an_oversized_response_id(): + """The sync logger bounds both the key and the Content-Disposition filename.""" + mock_s3_client = _run_log_event( + {"s3_bucket_name": "test-bucket", "s3_region_name": "us-west-2", "s3_path": "logs"}, + response_id="resp_" + "A" * 1100, + ) + + put_object_kwargs = mock_s3_client.put_object.call_args.kwargs + assert len(put_object_kwargs["Key"].encode("utf-8")) <= MAX_S3_OBJECT_KEY_BYTES + assert put_object_kwargs["Key"].startswith("logs/2026-07-30/time-12-00-00-000000_resp_") + filename = put_object_kwargs["ContentDisposition"].removeprefix('inline; filename="').removesuffix('"') + assert len(filename.encode("utf-8")) <= MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES + + +def test_put_object_keeps_the_configured_path_intact_when_only_the_id_has_to_shrink(): + """A long configured s3_path survives whole when the id can be shortened instead.""" + long_path = "litellm-prod-logs/" + "t" * 921 + mock_s3_client = _run_log_event( + {"s3_bucket_name": "test-bucket", "s3_region_name": "us-west-2", "s3_path": long_path}, + response_id="resp_" + "B" * 100, + ) + + key = mock_s3_client.put_object.call_args.kwargs["Key"] + assert key.startswith(long_path + "/2026-07-30/") + assert len(key.encode("utf-8")) == MAX_S3_OBJECT_KEY_BYTES diff --git a/tests/test_litellm/integrations/test_s3_v2.py b/tests/test_litellm/integrations/test_s3_v2.py index 51671d5101e..a037284d7c1 100644 --- a/tests/test_litellm/integrations/test_s3_v2.py +++ b/tests/test_litellm/integrations/test_s3_v2.py @@ -1170,6 +1170,294 @@ def test_create_s3_batch_logging_element_flat_key_for_arn_response_id(): assert file_segment.endswith("model-invocation-job_gl18r6skk9yy.json") +# -------------------------------------------------------------- +# object keys bounded to S3's 1024 UTF-8 byte limit +# -------------------------------------------------------------- +def _oversized_response_id() -> str: + return "resp_" + "A" * 1100 + + +def test_s3_object_key_at_the_byte_limit_is_left_alone(): + """A key that still fits is left byte-identical.""" + from litellm.constants import MAX_S3_OBJECT_KEY_BYTES + from litellm.integrations.s3 import get_s3_object_key + + start_time = datetime(2026, 8, 24, 6, 18, 41, 948021) + fixed_len = len("input/2026-08-24/.json") + file_name = "x" * (MAX_S3_OBJECT_KEY_BYTES - fixed_len) + + key = get_s3_object_key(s3_path="input", prefix="", start_time=start_time, s3_file_name=file_name) + + assert key == f"input/2026-08-24/{file_name}.json" + assert len(key.encode("utf-8")) == MAX_S3_OBJECT_KEY_BYTES + + +def test_s3_object_key_is_bounded_for_oversized_response_id(): + """An oversized Responses API id is shortened to a readable head plus a digest.""" + import hashlib + + from litellm.constants import MAX_S3_OBJECT_KEY_BYTES + from litellm.integrations.s3 import get_s3_object_key + + start_time = datetime(2026, 8, 24, 6, 18, 41, 948021) + file_name = f"time-06-18-41-948021_{_oversized_response_id()}" + + key = get_s3_object_key(s3_path="input", prefix="DefaultTeamProd/", start_time=start_time, s3_file_name=file_name) + + assert len(key.encode("utf-8")) <= MAX_S3_OBJECT_KEY_BYTES + assert key.startswith("input/DefaultTeamProd/2026-08-24/time-06-18-41-948021_resp_") + assert key.endswith(f"_{hashlib.sha256(file_name.encode('utf-8')).hexdigest()}.json") + + +@pytest.mark.parametrize( + "s3_path,prefix", + [ + ("input", ""), + ("a" * 900, ""), + ("input", "team-" + "b" * 900 + "/"), + ("c" * 600, "team-" + "d" * 600 + "/key-" + "e" * 600 + "/"), + # many short segments, so the trim lands exactly on the budget edge + ("", "ssss/" * 200), + ], +) +def test_s3_object_key_is_bounded_for_long_paths_and_aliases(s3_path: str, prefix: str): + """Long paths, team aliases and key aliases stay within the cap.""" + from litellm.constants import MAX_S3_OBJECT_KEY_BYTES + from litellm.integrations.s3 import get_s3_object_key + + key = get_s3_object_key( + s3_path=s3_path, + prefix=prefix, + start_time=datetime(2026, 8, 24, 6, 18, 41, 948021), + s3_file_name=f"time-06-18-41-948021_{_oversized_response_id()}", + ) + + assert len(key.encode("utf-8")) <= MAX_S3_OBJECT_KEY_BYTES + assert key.endswith(".json") + assert "/2026-08-24/" in key or key.startswith("2026-08-24/") + assert "/" not in key.rsplit("2026-08-24/", 1)[1] + + +def test_s3_object_key_trimmed_prefixes_stay_distinct_per_operator(): + """Prefixes that differ only past the trim point keep separate folders.""" + from litellm.constants import MAX_S3_OBJECT_KEY_BYTES + from litellm.integrations.s3 import get_s3_object_key + + start_time = datetime(2026, 8, 24, 6, 18, 41, 948021) + keys = [ + get_s3_object_key( + s3_path="input", + prefix="team-" + "b" * 1000 + suffix + "/", + start_time=start_time, + s3_file_name=f"time-06-18-41-948021_{_oversized_response_id()}", + ) + for suffix in ("-one", "-two") + ] + + assert keys[0] != keys[1] + assert all(key.startswith("input/team-" + "b" * 900) for key in keys) + assert all(len(key.encode("utf-8")) == MAX_S3_OBJECT_KEY_BYTES for key in keys) + + +def test_s3_object_key_bounded_prefix_never_splits_a_multibyte_character(): + """A multibyte prefix is trimmed on a character boundary.""" + from litellm.constants import MAX_S3_OBJECT_KEY_BYTES + from litellm.integrations.s3 import get_s3_object_key + + s3_path = "\u65e5\u672c\u8a9e" * 200 + + key = get_s3_object_key( + s3_path=s3_path, + prefix="\u30c1\u30fc\u30e0" * 200 + "/", + start_time=datetime(2026, 8, 24, 6, 18, 41, 948021), + s3_file_name=f"time-06-18-41-948021_{_oversized_response_id()}", + ) + + assert len(key.encode("utf-8")) <= MAX_S3_OBJECT_KEY_BYTES + assert key.startswith(s3_path[:100]) + assert "\ufffd" not in key + + +def test_s3_object_key_stays_unique_for_ids_sharing_a_head(): + """Ids sharing a visible head still get distinct keys.""" + from litellm.integrations.s3 import get_s3_object_key + + start_time = datetime(2026, 8, 24, 6, 18, 41, 948021) + keys = { + get_s3_object_key( + s3_path="input", + prefix="", + start_time=start_time, + s3_file_name=f"time-06-18-41-948021_{_oversized_response_id()}{suffix}", + ) + for suffix in ("first", "second", "third") + } + + assert len(keys) == 3 + + +def test_s3_object_key_bounding_matches_the_documented_layout(): + """The bounded key is `//_.json`.""" + import hashlib + + from litellm.integrations.s3 import get_s3_object_key + + file_name = f"time-06-18-41-948021_{_oversized_response_id()}" + + key = get_s3_object_key( + s3_path="input", + prefix="team/", + start_time=datetime(2026, 8, 24, 6, 18, 41, 948021), + s3_file_name=file_name, + ) + + digest = hashlib.sha256(file_name.encode("utf-8")).hexdigest() + assert key == f"input/team/2026-08-24/{file_name[:64]}_{digest}.json" + + +def test_s3_object_key_keeps_the_configured_prefix_when_only_the_id_overflows(): + """A 940 byte configured prefix survives whole when only the id overflows.""" + from litellm.constants import MAX_S3_OBJECT_KEY_BYTES + from litellm.integrations.s3 import get_s3_object_key + + prefix = "team-" + "b" * 934 + "/" + + key = get_s3_object_key( + s3_path="", + prefix=prefix, + start_time=datetime(2026, 8, 24, 6, 18, 41, 948021), + s3_file_name=f"time-06-18-41-948021_{_oversized_response_id()}", + ) + + assert key.startswith(prefix + "2026-08-24/") + assert len(key.encode("utf-8")) == MAX_S3_OBJECT_KEY_BYTES + + +def test_s3_object_key_spends_the_whole_budget_when_the_prefix_must_be_trimmed(): + """A trimmed prefix keeps every byte the budget allows, not whole segments.""" + from litellm.constants import MAX_S3_OBJECT_KEY_BYTES + from litellm.integrations.s3 import get_s3_object_key + + s3_path = "p" * 400 + "/" + "q" * 600 + + key = get_s3_object_key( + s3_path=s3_path, + prefix="", + start_time=datetime(2026, 8, 24, 6, 18, 41, 948021), + s3_file_name="time-06-18-41-948021_abc", + ) + + assert len(key.encode("utf-8")) == MAX_S3_OBJECT_KEY_BYTES + assert key.startswith("p" * 400 + "/" + "q" * 500) + + +def test_s3_object_key_keeps_a_single_segment_path_as_far_as_it_fits(): + """A path with no separator is kept as far as it fits, never dropped to the bucket root.""" + from litellm.constants import MAX_S3_OBJECT_KEY_BYTES + from litellm.integrations.s3 import get_s3_object_key + + key = get_s3_object_key( + s3_path="a" * 1050, + prefix="", + start_time=datetime(2026, 8, 24, 6, 18, 41, 948021), + s3_file_name="time-06-18-41-948021_chatcmpl-xyz", + ) + + assert len(key.encode("utf-8")) == MAX_S3_OBJECT_KEY_BYTES + assert key.startswith("a" * 900) + + +def test_create_s3_batch_logging_element_bounds_key_and_keeps_full_response_id(): + """The batch element bounds the key and keeps the full response id in the payload.""" + from litellm.constants import MAX_S3_OBJECT_KEY_BYTES + + logger = S3Logger(s3_use_team_prefix=True, s3_use_key_prefix=True) + response_id = _oversized_response_id() + payload = StandardLoggingPayload( + id=response_id, + metadata={"user_api_key_team_alias": "DefaultTeamProd", "user_api_key_alias": "prod-key"}, + messages=[], + ) + + result = logger.create_s3_batch_logging_element(datetime(2026, 8, 24, 6, 18, 41, 948021), payload) + + assert result is not None + assert len(result.s3_object_key.encode("utf-8")) <= MAX_S3_OBJECT_KEY_BYTES + assert result.s3_object_key.startswith("DefaultTeamProd/prod-key/2026-08-24/") + assert result.payload["id"] == response_id + + +def test_s3_object_download_filename_is_bounded_for_oversized_response_id(): + """The Content-Disposition filename is bounded too, or the PUT fails with MetadataTooLarge.""" + from litellm.constants import MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES + from litellm.integrations.s3 import get_s3_object_download_filename + + file_name = get_s3_object_download_filename(datetime(2026, 8, 24, 6, 18, 41, 948021), _oversized_response_id()) + + assert len(file_name.encode("utf-8")) <= MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES + assert file_name.startswith("time-2026-08-24T06-18-41-948021_resp_") + assert file_name.endswith(".json") + + +def test_s3_object_download_filenames_stay_distinct_when_shortened(): + """Shortened filenames stay distinct.""" + from litellm.integrations.s3 import get_s3_object_download_filename + + start_time = datetime(2026, 8, 24, 6, 18, 41, 948021) + file_names = { + get_s3_object_download_filename(start_time, _oversized_response_id() + suffix) + for suffix in ("first", "second", "third") + } + + assert len(file_names) == 3 + + +def test_s3_object_download_filename_short_id_is_unchanged(): + """An ordinary response id keeps the filename it had before.""" + from litellm.integrations.s3 import get_s3_object_download_filename + + file_name = get_s3_object_download_filename(datetime(2026, 8, 24, 6, 18, 41, 948021), "resp_abc123") + + assert file_name == "time-2026-08-24T06-18-41-948021_resp_abc123.json" + + +def test_create_s3_batch_logging_element_bounds_the_download_filename(): + """The batch element carries a bounded Content-Disposition filename.""" + from litellm.constants import MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES + + logger = S3Logger() + payload = StandardLoggingPayload(id=_oversized_response_id(), metadata={}, messages=[]) + + result = logger.create_s3_batch_logging_element(datetime(2026, 8, 24, 6, 18, 41, 948021), payload) + + assert result is not None + assert len(result.s3_object_download_filename.encode("utf-8")) <= MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES + + +@pytest.mark.asyncio +async def test_audit_log_object_key_is_bounded_for_a_long_configured_path(): + """Audit log keys are bounded by the same builder.""" + from litellm.constants import MAX_S3_OBJECT_KEY_BYTES + + logger = S3Logger() + logger.s3_path = "audit-archive/" + "z" * 1100 + + await logger.async_log_audit_log_event({"id": "1a4f7bd0-6f1e-4d0a-9b3c-9f2e1d5a7c88"}) + + assert len(logger.log_queue) == 1 + assert len(logger.log_queue[0].s3_object_key.encode("utf-8")) <= MAX_S3_OBJECT_KEY_BYTES + assert logger.log_queue[0].s3_object_key.startswith("audit-archive/" + "z" * 900) + + +def test_s3_object_download_filename_drops_characters_that_break_the_header(): + """A quote or separator in the response id cannot escape the quoted header value.""" + from litellm.integrations.s3 import get_s3_object_download_filename + + file_name = get_s3_object_download_filename(datetime(2026, 8, 24, 6, 18, 41, 948021), 'resp_a"b/c') + + assert file_name == "time-2026-08-24T06-18-41-948021_resp_a_b_c.json" + + # -------------------------------------------------------------- # params_source / s3_callback_params_override (audit-log decoupling) # -------------------------------------------------------------- diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 3c4121977de..0e1c832ebf5 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -32,6 +32,8 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( TokenTypeCostBreakdown, _calculate_input_cost, _get_token_base_cost, + _is_off_peak, + _is_within_off_peak_window, calculate_cache_writing_cost, generic_cost_per_token, get_token_type_cost_breakdown, @@ -409,6 +411,377 @@ def test_get_token_base_cost_picks_highest_crossed_tier(): assert prompt_base_cost == 9e-6 +def test_is_within_off_peak_window_same_day(): + from datetime import datetime, timezone + + window = "09:00-17:00" + assert _is_within_off_peak_window(window, datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) is True + assert _is_within_off_peak_window(window, datetime(2026, 1, 1, 8, 59, tzinfo=timezone.utc)) is False + assert _is_within_off_peak_window(window, datetime(2026, 1, 1, 9, 0, tzinfo=timezone.utc)) is True + assert _is_within_off_peak_window(window, datetime(2026, 1, 1, 17, 0, tzinfo=timezone.utc)) is False + + +def test_is_within_off_peak_window_wraps_midnight(): + from datetime import datetime, timezone + + window = "16:30-00:30" + assert _is_within_off_peak_window(window, datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) is True + assert _is_within_off_peak_window(window, datetime(2026, 1, 1, 0, 15, tzinfo=timezone.utc)) is True + assert _is_within_off_peak_window(window, datetime(2026, 1, 1, 16, 30, tzinfo=timezone.utc)) is True + assert _is_within_off_peak_window(window, datetime(2026, 1, 1, 0, 30, tzinfo=timezone.utc)) is False + assert _is_within_off_peak_window(window, datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) is False + + +def test_is_within_off_peak_window_equal_start_and_end_covers_whole_day(): + """An equal start and end is the natural way to spell off-peak all day. It used to take the + non-wrap branch, where start <= now < end can never hold, so it matched nothing and billed at + standard rates around the clock without raising or logging anything.""" + from datetime import datetime, timezone + + for window in ("00:00-00:00", "10:00-10:00"): + for hour in range(24): + assert ( + _is_within_off_peak_window(window, datetime(2026, 1, 1, hour, 0, tzinfo=timezone.utc)) is True + ), f"{window} should cover {hour:02d}:00" + + +def test_is_within_off_peak_window_multiple_windows(): + from datetime import datetime, timezone + + # Providers like DeepSeek V4 have more than one daily peak/off-peak window. + windows = ["01:00-05:00", "13:00-16:00"] + assert _is_within_off_peak_window(windows, datetime(2026, 1, 1, 3, 0, tzinfo=timezone.utc)) is True + assert _is_within_off_peak_window(windows, datetime(2026, 1, 1, 14, 30, tzinfo=timezone.utc)) is True + assert _is_within_off_peak_window(windows, datetime(2026, 1, 1, 9, 0, tzinfo=timezone.utc)) is False + # a malformed entry in the list is ignored, valid entries still match + assert _is_within_off_peak_window(["bad", "13:00-16:00"], datetime(2026, 1, 1, 14, 0, tzinfo=timezone.utc)) is True + assert _is_within_off_peak_window([], datetime(2026, 1, 1, 14, 0, tzinfo=timezone.utc)) is False + + +def test_is_within_off_peak_window_normalizes_timezone_aware_input(): + from datetime import datetime, timedelta, timezone + + # A caller may pass a non-UTC aware datetime; the window is UTC and must be + # evaluated in UTC, not against the caller's wall-clock. 09:00 at UTC+8 is + # 01:00 UTC, inside the 01:00-05:00 window. + tz_plus_8 = timezone(timedelta(hours=8)) + assert _is_within_off_peak_window("01:00-05:00", datetime(2026, 1, 1, 9, 0, tzinfo=tz_plus_8)) is True + assert _is_within_off_peak_window("01:00-05:00", datetime(2026, 1, 1, 12, 0, tzinfo=tz_plus_8)) is True + # 06:00 at UTC+8 is 22:00 UTC the previous day, outside the window + assert _is_within_off_peak_window("01:00-05:00", datetime(2026, 1, 1, 6, 0, tzinfo=tz_plus_8)) is False + + +def test_is_within_off_peak_window_malformed_returns_false(): + from datetime import datetime, timezone + + now = datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc) + assert _is_within_off_peak_window("not-a-window", now) is False + assert _is_within_off_peak_window("16:30", now) is False + assert _is_within_off_peak_window("25:00-26:00", now) is False + + +def test_is_off_peak_weekday_qualified_windows_deepseek_schedule(): + """DeepSeek since 2026-08-23: peak is 01:00-04:00 and 06:00-10:00 UTC on weekdays only, with + weekends off-peak around the clock. The weekday axis is not a filter on one window set; on + two days of seven the off-peak window becomes the whole day, so the schedule needs two + day-qualified rules. The weekend instants inside would-be peak hours are the ones a + time-only implementation bills wrong.""" + from datetime import datetime, timezone + + deepseek = { + "windows": [ + {"hours_utc": ["00:00-01:00", "04:00-06:00", "10:00-00:00"], "weekdays": [1, 2, 3, 4, 5]}, + {"hours_utc": "00:00-00:00", "weekdays": [6, 7]}, + ], + } + peak_instants = [ + datetime(2026, 8, 24, 1, 30, tzinfo=timezone.utc), + datetime(2026, 8, 26, 7, 0, tzinfo=timezone.utc), + datetime(2026, 8, 28, 9, 59, tzinfo=timezone.utc), + ] + off_peak_instants = [ + datetime(2026, 8, 23, 1, 30, tzinfo=timezone.utc), + datetime(2026, 8, 29, 2, 0, tzinfo=timezone.utc), + datetime(2026, 8, 30, 8, 0, tzinfo=timezone.utc), + datetime(2026, 8, 26, 5, 0, tzinfo=timezone.utc), + datetime(2026, 8, 28, 16, 30, tzinfo=timezone.utc), + datetime(2026, 8, 24, 0, 30, tzinfo=timezone.utc), + ] + for when in peak_instants: + assert _is_off_peak(deepseek, when) is False, f"{when.isoformat()} should bill peak" + for when in off_peak_instants: + assert _is_off_peak(deepseek, when) is True, f"{when.isoformat()} should bill off-peak" + + +def test_is_off_peak_weekday_timezone_reads_vendor_calendar(): + """The UTC and Asia/Shanghai calendars only disagree about the date over 16:00-24:00 UTC, so + a window in that stretch is the one place a vendor-local weekday differs from a UTC one: + 2026-08-28T16:30Z is Friday in UTC but already Saturday in Beijing.""" + from datetime import datetime, timezone + + shanghai_saturday = { + "weekday_timezone": "Asia/Shanghai", + "windows": [{"hours_utc": "16:00-17:00", "weekdays": [6]}], + } + assert _is_off_peak(shanghai_saturday, datetime(2026, 8, 28, 16, 30, tzinfo=timezone.utc)) is True + assert _is_off_peak(shanghai_saturday, datetime(2026, 8, 29, 16, 30, tzinfo=timezone.utc)) is False + + +def test_is_off_peak_weekdays_default_utc_calendar_and_accept_names(): + from datetime import datetime, timezone + + named_weekend = {"windows": [{"hours_utc": "00:00-00:00", "weekdays": ["Sat", "sunday"]}]} + assert _is_off_peak(named_weekend, datetime(2026, 8, 29, 12, 0, tzinfo=timezone.utc)) is True + assert _is_off_peak(named_weekend, datetime(2026, 8, 28, 12, 0, tzinfo=timezone.utc)) is False + + utc_friday = {"windows": [{"hours_utc": "16:00-17:00", "weekdays": [5]}]} + assert _is_off_peak(utc_friday, datetime(2026, 8, 28, 16, 30, tzinfo=timezone.utc)) is True + assert _is_off_peak(utc_friday, datetime(2026, 8, 29, 16, 30, tzinfo=timezone.utc)) is False + + +def test_is_off_peak_naive_current_time_read_as_utc(): + from datetime import datetime + + block = {"windows": [{"hours_utc": "16:00-17:00", "weekdays": [5]}]} + assert _is_off_peak(block, datetime(2026, 8, 28, 16, 30)) is True + assert _is_off_peak(block, datetime(2026, 8, 29, 16, 30)) is False + + +def test_is_off_peak_invalid_weekday_timezone_falls_back_to_utc(): + from datetime import datetime, timezone + + block = {"weekday_timezone": "Not/AZone", "windows": [{"hours_utc": "16:00-17:00", "weekdays": [5]}]} + assert _is_off_peak(block, datetime(2026, 8, 28, 16, 30, tzinfo=timezone.utc)) is True + assert _is_off_peak(block, datetime(2026, 8, 29, 16, 30, tzinfo=timezone.utc)) is False + + +def test_is_off_peak_ignores_malformed_weekday_rules(): + from datetime import datetime, timezone + + when = datetime(2026, 8, 29, 12, 0, tzinfo=timezone.utc) + assert _is_off_peak({"windows": [{"hours_utc": "00:00-00:00", "weekdays": []}]}, when) is False + assert _is_off_peak({"windows": [{"hours_utc": "00:00-00:00", "weekdays": [0, 8, "noday", True]}]}, when) is False + assert _is_off_peak({"windows": [{"weekdays": [6]}]}, when) is False + assert _is_off_peak({"windows": [{"hours_utc": 1630}]}, when) is False + assert _is_off_peak({"windows": ["00:00-00:00"]}, when) is False + assert _is_off_peak({"windows": "00:00-00:00"}, when) is False + assert _is_off_peak({"hours_utc": 1630}, when) is False + assert _is_off_peak({}, when) is False + + +def test_is_off_peak_flat_hours_and_windows_are_a_union(): + from datetime import datetime, timezone + + block = { + "hours_utc": "04:00-06:00", + "windows": [{"hours_utc": "00:00-00:00", "weekdays": [7]}], + } + assert _is_off_peak(block, datetime(2026, 8, 28, 5, 0, tzinfo=timezone.utc)) is True + assert _is_off_peak(block, datetime(2026, 8, 30, 20, 0, tzinfo=timezone.utc)) is True + assert _is_off_peak(block, datetime(2026, 8, 28, 20, 0, tzinfo=timezone.utc)) is False + + +def test_get_token_base_cost_weekend_only_off_peak_rate(): + from datetime import datetime, timezone + from typing import cast + + from litellm.types.utils import ModelInfo + + model_info = cast( + ModelInfo, + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "off_peak_pricing": { + "windows": [ + {"hours_utc": ["00:00-01:00", "04:00-06:00", "10:00-00:00"], "weekdays": [1, 2, 3, 4, 5]}, + {"hours_utc": "00:00-00:00", "weekdays": [6, 7]}, + ], + "input_cost_per_token": 5e-7, + "output_cost_per_token": 1e-6, + }, + }, + ) + usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) + + saturday_peak_hours = _get_token_base_cost( + model_info, usage, current_time=datetime(2026, 8, 29, 2, 0, tzinfo=timezone.utc) + ) + assert saturday_peak_hours[:2] == (5e-7, 1e-6) + + monday_same_hours = _get_token_base_cost( + model_info, usage, current_time=datetime(2026, 8, 24, 2, 0, tzinfo=timezone.utc) + ) + assert monday_same_hours[:2] == (1e-6, 2e-6) + + +def test_get_token_base_cost_applies_off_peak_pricing(): + from datetime import datetime, timezone + from typing import cast + + from litellm.types.utils import ModelInfo + + model_info = cast( + ModelInfo, + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "cache_read_input_token_cost": 1e-7, + "off_peak_pricing": { + "hours_utc": "16:30-00:30", + "input_cost_per_token": 5e-7, + "output_cost_per_token": 1e-6, + "cache_read_input_token_cost": 5e-8, + }, + }, + ) + usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) + + off_peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) + assert off_peak[0] == 5e-7 + assert off_peak[1] == 1e-6 + assert off_peak[4] == 5e-8 + + peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) + assert peak[0] == 1e-6 + assert peak[1] == 2e-6 + assert peak[4] == 1e-7 + + +def test_get_token_base_cost_non_mapping_off_peak_block_bills_standard_rates(): + """A truthy non-mapping off_peak_pricing value (a bare string or a list in + YAML) must bill standard rates rather than raising, matching how every + other malformed piece of the block behaves. + """ + from datetime import datetime, timezone + from typing import cast + + from litellm.types.utils import ModelInfo + + usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) + when = datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc) + + for malformed_block in ("16:00-19:00", ["16:00-19:00"], 5e-7, True): + model_info = cast( + ModelInfo, + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "off_peak_pricing": malformed_block, + }, + ) + result = _get_token_base_cost(model_info, usage, current_time=when) + assert result[0] == 1e-6 + assert result[1] == 2e-6 + + +def test_get_token_base_cost_off_peak_falls_back_to_standard_when_unset(): + from datetime import datetime, timezone + from typing import cast + + from litellm.types.utils import ModelInfo + + model_info = cast( + ModelInfo, + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "off_peak_pricing": {"hours_utc": "16:30-00:30", "input_cost_per_token": 5e-7}, + }, + ) + usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) + + result = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) + assert result[0] == 5e-7 + assert result[1] == 2e-6 + + +def test_get_token_base_cost_off_peak_wins_over_threshold(): + from datetime import datetime, timezone + from typing import cast + + from litellm.types.utils import ModelInfo + + model_info = cast( + ModelInfo, + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "input_cost_per_token_above_200k_tokens": 3e-6, + "output_cost_per_token_above_200k_tokens": 4e-6, + "off_peak_pricing": { + "hours_utc": "16:30-00:30", + "input_cost_per_token": 5e-7, + "output_cost_per_token": 1e-6, + }, + }, + ) + usage = Usage(prompt_tokens=250000, completion_tokens=250000, total_tokens=500000) + + off_peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) + assert off_peak[0] == 5e-7 + assert off_peak[1] == 1e-6 + + peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) + assert peak[0] == 3e-6 + assert peak[1] == 4e-6 + + +def test_get_model_info_propagates_off_peak_fields(): + model_name = "test-off-peak-model" + off_peak_pricing = { + "hours_utc": "16:30-00:30", + "input_cost_per_token": 5e-7, + "output_cost_per_token": 1e-6, + "cache_read_input_token_cost": 5e-8, + } + litellm.register_model( + { + model_name: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "off_peak_pricing": off_peak_pricing, + } + } + ) + info = litellm.get_model_info(model=model_name) + assert info["off_peak_pricing"] == off_peak_pricing + + +def test_get_token_base_cost_off_peak_wins_over_tiered_pricing(): + """Tiered pricing resolves base rates on its own path and returns early, so off-peak has to + be applied there too or a model carrying both would silently bill the tier rate all day.""" + from datetime import datetime, timezone + + model_name = "litellm-test-off-peak-tiered" + litellm.register_model( + { + model_name: { + "litellm_provider": "openai", + "mode": "chat", + "tiered_pricing": [ + {"range": [0, 128000], "input_cost_per_token": 3e-6, "output_cost_per_token": 6e-6}, + ], + "off_peak_pricing": { + "hours_utc": "16:30-00:30", + "input_cost_per_token": 5e-7, + "output_cost_per_token": 1e-6, + }, + } + } + ) + info = litellm.get_model_info(model=model_name) + usage = Usage(prompt_tokens=1_000, completion_tokens=100, total_tokens=1_100) + + inside = _get_token_base_cost(info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) + assert inside[:2] == (5e-7, 1e-6) + + outside = _get_token_base_cost(info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) + assert outside[:2] == (3e-6, 6e-6) + + def test_generic_cost_per_token_gpt54_above_272k_tokens(_local_model_cost_map): """GPT-5.4/5.4-pro: prompts >272K input tokens priced at 2x input, 1.5x output.""" model = "gpt-5.4" diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 2a955845861..0f9f8259bef 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -6354,3 +6354,33 @@ def test_anthropic_drop_params_reduces_mixed_output_config_to_format(monkeypatch ) assert result.get("output_config") == {"format": schema_format} + + +def test_response_format_tool_path_skips_forced_tool_choice_when_unsupported(local_model_cost_map, monkeypatch): + """Backstop: on the tool-based structured-output path, a model flagged + ``supports_forced_tool_use: false`` must not get the forced response-format + tool_choice the provider would 400 on.""" + monkeypatch.setitem( + litellm.model_cost, + "claude-test-no-forced-tools", + {"litellm_provider": "anthropic", "mode": "chat", "supports_forced_tool_use": False}, + ) + config = AnthropicConfig() + + result = config.map_openai_params( + non_default_params={ + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "test_schema", + "schema": {"type": "object", "properties": {"result": {"type": "string"}}}, + }, + } + }, + optional_params={}, + model="claude-test-no-forced-tools", + drop_params=False, + ) + + assert "tools" in result + assert "tool_choice" not in result diff --git a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py index a122d97a0f0..41d82e4f960 100644 --- a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py @@ -643,3 +643,31 @@ def test_bedrock_chat_invoke_drop_params_still_inlines_for_non_native(local_mode assert "output_config" not in result last_content = result["messages"][-1]["content"] assert json.loads(last_content[-1]["text"]) == schema + + +@pytest.mark.parametrize( + "model", + ["us.anthropic.claude-fable-5-1", "anthropic.claude-fable-5-1"], +) +def test_bedrock_chat_invoke_fable_5_1_response_format_avoids_forced_tool_choice(local_model_cost_map, model): + """Regression: Bedrock rejects both native ``output_config.format`` and forced + tool_choice for Fable 5.1, so invoke must use the tool-based path without a + forced ``tool_choice``.""" + result = AmazonAnthropicClaudeConfig().map_openai_params( + non_default_params={ + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "test_schema", + "schema": {"type": "object", "properties": {"result": {"type": "string"}}}, + }, + } + }, + optional_params={}, + model=model, + drop_params=False, + ) + + assert "output_format" not in result + assert "tools" in result + assert "tool_choice" not in result diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 37ca801a7a7..4f53d3481de 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -6497,6 +6497,39 @@ def test_unforced_tool_choice_unaffected_on_fable_5_1_converse(local_model_cost_ assert result == ({"auto": {}} if tool_choice == "auto" else None) +@pytest.mark.parametrize( + "model", + ["anthropic.claude-fable-5-1", "us.anthropic.claude-fable-5-1"], +) +def test_response_format_avoids_native_and_forced_tool_choice_on_fable_5_1_converse( + local_model_cost_map, model +): + """Regression: Bedrock rejects both ``outputConfig`` structured output and forced + tool_choice for Fable 5.1, so response_format must map to a tool without a forced + tool_choice.""" + config = AmazonConverseConfig() + + result = config.map_openai_params( + non_default_params={ + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "test_schema", + "schema": {"type": "object", "properties": {"result": {"type": "string"}}}, + }, + } + }, + optional_params={}, + model=model, + drop_params=False, + ) + + assert "outputConfig" not in result + assert "tools" in result + assert "tool_choice" not in result + assert result.get("json_mode") is True + + def test_forced_tool_choice_forwarded_on_converse_models_that_support_it( local_model_cost_map, monkeypatch ): diff --git a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py index ba7067322d8..9737d63cc26 100644 --- a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py @@ -1166,3 +1166,118 @@ class TestOpenAIPromptCacheBreakpointChatPath: assert request["messages"][1]["content"] == [{"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}] assert request["extra_body"] == {"prompt_cache_options": self.EXPLICIT} assert "prompt_cache_options" not in request + + +class TestToolSchemaCombinatorFlatteningForOpenAI: + """ + Regression tests for LIT-6488: OpenAI's chat completions validator rejects + tool parameters carrying a top-level anyOf/oneOf/allOf for every model + family, GPT-5 included, unlike the Responses API. + """ + + def setup_method(self): + self.config = OpenAIGPTConfig() + + @pytest.fixture(autouse=True) + def _clean_openai_base_env(self, monkeypatch): + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None, raising=False) + + @staticmethod + def _anyof_tool(): + return { + "type": "function", + "function": { + "name": "automation_update", + "description": "Update an automation", + "parameters": { + "type": "object", + "anyOf": [ + { + "properties": {"id": {"type": "string"}, "enabled": {"type": "boolean"}}, + "required": ["id", "enabled"], + }, + { + "properties": {"id": {"type": "string"}, "schedule": {"type": "string"}}, + "required": ["id", "schedule"], + }, + ], + "properties": {"id": {"type": "string"}}, + "required": ["id"], + }, + }, + } + + def _transform(self, config, model, litellm_params, tools): + return config.transform_request( + model=model, + messages=[{"role": "user", "content": "hi"}], + optional_params={"tools": tools}, + litellm_params=litellm_params, + headers={}, + ) + + def test_flattens_top_level_anyof_for_hosted_openai(self): + request = self._transform( + self.config, "gpt-4o", {"custom_llm_provider": "openai", "api_base": None}, [self._anyof_tool()] + ) + parameters = request["tools"][0]["function"]["parameters"] + assert "anyOf" not in parameters + assert parameters["type"] == "object" + assert set(parameters["properties"]) == {"id", "enabled", "schedule"} + assert parameters["required"] == ["id"] + assert request["tools"][0]["function"]["name"] == "automation_update" + + def test_gpt5_family_flattens_on_chat_completions(self): + request = self._transform( + OpenAIGPT5Config(), "gpt-5.6", {"custom_llm_provider": "openai", "api_base": None}, [self._anyof_tool()] + ) + assert "anyOf" not in request["tools"][0]["function"]["parameters"] + + def test_custom_api_base_keeps_union(self): + tool = self._anyof_tool() + request = self._transform( + self.config, + "gpt-4o", + {"custom_llm_provider": "openai", "api_base": "http://localhost:8000/v1"}, + [tool], + ) + assert request["tools"][0]["function"]["parameters"] == self._anyof_tool()["function"]["parameters"] + + def test_non_openai_provider_keeps_union(self): + request = self._transform( + self.config, "some-oss-model", {"custom_llm_provider": "groq", "api_base": None}, [self._anyof_tool()] + ) + assert request["tools"][0]["function"]["parameters"] == self._anyof_tool()["function"]["parameters"] + + def test_caller_tool_dict_is_not_mutated(self): + tool = self._anyof_tool() + self._transform(self.config, "gpt-4o", {"custom_llm_provider": "openai", "api_base": None}, [tool]) + assert tool == self._anyof_tool() + + def test_clean_object_schema_passes_through_as_same_object(self): + tool = { + "type": "function", + "function": { + "name": "lookup", + "parameters": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}, + }, + } + request = self._transform( + self.config, "gpt-4o", {"custom_llm_provider": "openai", "api_base": None}, [tool] + ) + assert request["tools"][0] is tool + + @pytest.mark.asyncio + async def test_async_transform_request_flattens_for_hosted_openai(self): + request = await self.config.async_transform_request( + model="gpt-4o", + messages=[{"role": "user", "content": "hi"}], + optional_params={"tools": [self._anyof_tool()]}, + litellm_params={"custom_llm_provider": "openai", "api_base": None}, + headers={}, + ) + parameters = request["tools"][0]["function"]["parameters"] + assert "anyOf" not in parameters + assert set(parameters["properties"]) == {"id", "enabled", "schedule"} diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index 552ca98441f..9419f88a981 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -727,3 +727,28 @@ def test_sanitize_strips_effort_for_haiku_45(): data = {"output_config": {"effort": "high"}} sanitize_vertex_anthropic_output_params(data, "vertex_ai/claude-opus-4-6") assert data["output_config"] == {"effort": "high"} + + +def test_vertex_ai_fable_5_1_response_format_uses_native_output_format(local_model_cost_map): + """Regression: Fable 5.1 rejects forced tool use, so the vertex map entry + advertises native structured output and ``response_format`` must map to + ``output_format`` instead of the tool-based path's forced tool_choice.""" + config = VertexAIAnthropicConfig() + response_format = { + "type": "json_schema", + "json_schema": { + "name": "test_schema", + "schema": {"type": "object", "properties": {"result": {"type": "string"}}}, + }, + } + + result_params = config.map_openai_params( + non_default_params={"response_format": response_format}, + optional_params={}, + model="claude-fable-5-1", + drop_params=False, + ) + + assert "output_format" in result_params + assert "tool_choice" not in result_params + assert "tools" not in result_params diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index 393953ccf68..4661cc17dbc 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -18,6 +18,7 @@ from litellm.proxy._types import ( ReconcileOutcome, UserAPIKeyAuth, ) +from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper from litellm.proxy.management_endpoints.model_management_endpoints import ( ModelManagementAuthChecks, _get_team_deployments, @@ -263,6 +264,131 @@ class TestModelManagementAuthChecks: ) assert "403" in str(exc_info.value) + def test_can_user_attach_credential_admin_success(self): + result = ModelManagementAuthChecks.can_user_attach_credential( + litellm_params=LiteLLM_Params(model="test_model", litellm_credential_name="shared-credential"), + user_api_key_dict=self.admin_user, + ) + assert result is True + + def test_can_user_attach_credential_without_credential_allows_any_role(self): + result = ModelManagementAuthChecks.can_user_attach_credential( + litellm_params=LiteLLM_Params(model="test_model"), + user_api_key_dict=self.team_admin_user, + ) + assert result is True + + def test_can_user_attach_credential_team_admin_fails(self): + with pytest.raises(Exception, match="Only a proxy admin can attach a stored credential") as exc_info: + ModelManagementAuthChecks.can_user_attach_credential( + litellm_params=LiteLLM_Params(model="test_model", litellm_credential_name="shared-credential"), + user_api_key_dict=self.team_admin_user, + ) + assert exc_info.value.code == "403" + + def test_can_user_attach_credential_unchanged_existing_allows_any_role(self): + result = ModelManagementAuthChecks.can_user_attach_credential( + litellm_params=LiteLLM_Params(model="test_model", litellm_credential_name="shared-credential"), + user_api_key_dict=self.team_admin_user, + existing_litellm_params=LiteLLM_Params(model="test_model", litellm_credential_name="shared-credential"), + ) + assert result is True + + def test_can_user_attach_credential_unchanged_encrypted_existing_allows_any_role(self, monkeypatch): + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-1234") + encrypted_name = encrypt_value_helper(value="shared-credential") + assert encrypted_name != "shared-credential" + result = ModelManagementAuthChecks.can_user_attach_credential( + litellm_params=LiteLLM_Params(model="test_model", litellm_credential_name="shared-credential"), + user_api_key_dict=self.team_admin_user, + existing_litellm_params=LiteLLM_Params(model="test_model", litellm_credential_name=encrypted_name), + ) + assert result is True + + @pytest.mark.asyncio + async def test_add_new_model_rejects_credential_attach_for_non_admin(self): + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.model_management_endpoints import ( + add_new_model, + ) + + mock_prisma = MagicMock() + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch("litellm.proxy.proxy_server.store_model_in_db", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch("litellm.proxy.proxy_server.premium_user", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch( # test-quality-ok: prior auth check needs a live DB; only the credential check is under test + "litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call", + new=AsyncMock(return_value=None), + ), + ): + with pytest.raises(ProxyException) as exc_info: + await add_new_model( + model_params=Deployment( + model_name="credential-model", + litellm_params=LiteLLM_Params( + model="openai/gpt-4o", litellm_credential_name="shared-credential" + ), + model_info={"id": "credential-create-test"}, + ), + user_api_key_dict=self.team_admin_user, + ) + assert exc_info.value.code == "403" + mock_prisma.db.litellm_proxymodeltable.create.assert_not_called() + + @pytest.mark.asyncio + async def test_patch_model_rejects_credential_attach_for_non_admin(self): + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.model_management_endpoints import ( + patch_model, + ) + from litellm.types.router import updateLiteLLMParams + + model_id = "credential-patch-test" + db_model = Deployment( + model_name="credential-model", + litellm_params=LiteLLM_Params(model="openai/gpt-4o"), + model_info={"id": model_id}, + ) + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch("litellm.proxy.proxy_server.llm_router", MagicMock()), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch("litellm.proxy.proxy_server.store_model_in_db", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch("litellm.proxy.proxy_server.premium_user", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch( # test-quality-ok: stubs the DB row fetch; only the credential check is under test + "litellm.proxy.management_endpoints.model_management_endpoints.get_db_model", + new=AsyncMock(return_value=db_model), + ), + patch( # test-quality-ok: prior auth check needs a live DB; only the credential check is under test + "litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call", + new=AsyncMock(return_value=None), + ), + patch( # test-quality-ok: asserts the DB write is never reached on rejection + "litellm.proxy.management_endpoints.model_management_endpoints._update_team_model_in_db", + new=AsyncMock(), + ) as mock_update, + ): + with pytest.raises(ProxyException) as exc_info: + await patch_model( + model_id=model_id, + patch_data=updateDeployment( + litellm_params=updateLiteLLMParams( + model="openai/gpt-4o", litellm_credential_name="shared-credential" + ) + ), + user_api_key_dict=self.team_admin_user, + ) + assert exc_info.value.code == "403" + mock_update.assert_not_awaited() + + def test_can_user_attach_credential_internal_user_fails(self): + with pytest.raises(Exception, match="Only a proxy admin can attach a stored credential") as exc_info: + ModelManagementAuthChecks.can_user_attach_credential( + litellm_params=LiteLLM_Params(model="test_model", litellm_credential_name="shared-credential"), + user_api_key_dict=self.normal_user, + ) + assert exc_info.value.code == "403" + class MockModelTable: def __init__(self, model_aliases: Dict[str, str], include: Optional[dict] = None): diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index 5e2dd358d75..6ea6f208bb5 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -1737,8 +1737,7 @@ class TestRunServerDbSetup: mock_atexit_register, mock_subprocess_run, ): - """Which resolver and which migration mode run_server hands setup_database, - across the db push flag, the v2/legacy flag pair and USE_V2_MIGRATION_RESOLVER.""" + """Test that use_prisma_db_push flag correctly controls PrismaManager.setup_database use_migrate parameter""" from litellm.proxy.proxy_cli import run_server # Mock subprocess.run to simulate prisma being available @@ -1788,7 +1787,7 @@ class TestRunServerDbSetup: # use_prisma_db_push should be False (default), so use_migrate should be True run_server.main(["--local", "--skip_server_startup"], standalone_mode=False) mock_setup_database.assert_called_with( - use_migrate=True, use_v2_resolver=True + use_migrate=True, use_v2_resolver=False ) # Reset mocks @@ -1803,38 +1802,9 @@ class TestRunServerDbSetup: standalone_mode=False, ) mock_setup_database.assert_called_with( - use_migrate=False, use_v2_resolver=True + use_migrate=False, use_v2_resolver=False ) - for argv, env_value, expected_v2 in ( - ([], None, True), - (["--use_v2_migration_resolver"], None, True), - (["--use_legacy_migration_resolver"], None, False), - ([], "false", False), - ([], "true", True), - (["--use_v2_migration_resolver"], "false", True), - (["--use_legacy_migration_resolver"], "true", False), - ): - mock_setup_database.reset_mock() - mock_should_update_schema.reset_mock() - mock_should_update_schema.return_value = True - - resolver_env = ( - {"USE_V2_MIGRATION_RESOLVER": env_value} - if env_value is not None - else {} - ) - os.environ.pop("USE_V2_MIGRATION_RESOLVER", None) - with patch.dict(os.environ, resolver_env): - run_server.main( - ["--local", "--skip_server_startup", *argv], - standalone_mode=False, - ) - assert mock_setup_database.call_args.kwargs == { - "use_migrate": True, - "use_v2_resolver": expected_v2, - }, f"argv={argv} env={env_value}" - @patch("subprocess.run") @patch("atexit.register") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") @@ -1899,7 +1869,7 @@ class TestRunServerDbSetup: ) assert exc_info.value.code == 1 mock_setup_database.assert_called_once_with( - use_migrate=True, use_v2_resolver=True + use_migrate=True, use_v2_resolver=False ) @patch("subprocess.run") @@ -2011,6 +1981,7 @@ class TestRunServerDbSetup: use_migrate=True, use_v2_resolver=True ) + # --- Module-level helpers for worker startup hook tests --- _dummy_hook_called = False diff --git a/tests/test_litellm/test_register_model_custom_pricing.py b/tests/test_litellm/test_register_model_custom_pricing.py index 39f498b4e58..452a15334ef 100644 --- a/tests/test_litellm/test_register_model_custom_pricing.py +++ b/tests/test_litellm/test_register_model_custom_pricing.py @@ -793,3 +793,203 @@ def test_embedding_direct_sdk_custom_pricing_still_registers_shared_key(): finally: litellm.model_cost.pop(model_key, None) _invalidate_model_cost_lowercase_map() + + +def test_update_dictionary_merges_nested_dicts_without_aliasing(): + """A nested dict must be merged copy-on-write: the pre-existing nested dict + object stays untouched, and the caller's incoming nested dict is never + inserted by reference into the merged result. + """ + from litellm.utils import _update_dictionary + + existing_nested = {"hours_utc": "01:00-02:00"} + existing = {"off_peak_pricing": existing_nested} + incoming_nested = {"windows": [{"hours_utc": "16:00-19:00", "weekdays": [2]}]} + incoming = {"off_peak_pricing": incoming_nested} + + merged = _update_dictionary(existing, incoming) + + assert merged["off_peak_pricing"] == { + "hours_utc": "01:00-02:00", + "windows": [{"hours_utc": "16:00-19:00", "weekdays": [2]}], + } + assert existing_nested == {"hours_utc": "01:00-02:00"} + assert merged["off_peak_pricing"] is not incoming_nested + + fresh = _update_dictionary({}, incoming) + assert fresh["off_peak_pricing"] == incoming_nested + assert fresh["off_peak_pricing"] is not incoming_nested + + +def test_router_deployments_sharing_backend_keep_their_own_off_peak_pricing(): + """Two deployments of the same backend model with different + ``off_peak_pricing`` blocks must each keep their own schedule under their + unique model id, and neither block may leak onto the shared backend keys. + + Before the fix, ``register_model`` inserted the first deployment's block by + reference into the built-in ``gpt-4o-mini`` entry, and the second + deployment's registration merged its keys into that same object, corrupting + the first deployment's schedule and polluting the built-in entry. + """ + from litellm import Router + + active_block = { + "windows": [{"hours_utc": "16:00-19:00", "weekdays": [2]}], + "input_cost_per_token": 5e-07, + "output_cost_per_token": 1e-06, + } + inactive_block = { + "hours_utc": "05:00-06:00", + "input_cost_per_token": 5e-07, + "output_cost_per_token": 1e-06, + } + shared_keys = ["gpt-4o-mini", "openai/gpt-4o-mini"] + deployment_ids = ["offpeak-alias-dep-1", "offpeak-alias-dep-2"] + original_entries = _snapshot_model_cost_entries(shared_keys) + + router = Router( + model_list=[ + { + "model_name": "offpeak-active-weekday", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "fake-key-for-registration", + }, + "model_info": { + "id": deployment_ids[0], + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "off_peak_pricing": dict(active_block), + }, + }, + { + "model_name": "offpeak-inactive-hours", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "fake-key-for-registration", + }, + "model_info": { + "id": deployment_ids[1], + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "off_peak_pricing": dict(inactive_block), + }, + }, + ] + ) + + try: + registered_first = litellm.model_cost[deployment_ids[0]]["off_peak_pricing"] + registered_second = litellm.model_cost[deployment_ids[1]]["off_peak_pricing"] + assert registered_first == active_block + assert registered_second == inactive_block + for shared_key in shared_keys: + shared_entry = litellm.model_cost.get(shared_key) or {} + assert not shared_entry.get("off_peak_pricing") + finally: + for deployment_id in deployment_ids: + litellm.model_cost.pop(deployment_id, None) + _restore_model_cost_entries(original_entries) + del router + + +def test_router_off_peak_only_deployment_inherits_builtin_base_rates(): + """A deployment that sets only ``off_peak_pricing`` on its model_info must + still be costed from its deployment-scoped entry: the base token rates are + inherited from the backend model's built-in cost map entry, since the + shared backend key deliberately never carries the off-peak block. + """ + from litellm import Router + + block = { + "hours_utc": "00:00-00:00", + "input_cost_per_token": 5e-05, + "output_cost_per_token": 1e-04, + } + shared_keys = ["gpt-4o-mini", "openai/gpt-4o-mini"] + deployment_id = "offpeak-only-dep-1" + original_entries = _snapshot_model_cost_entries(shared_keys + [deployment_id]) + builtin_info = litellm.get_model_info(model="openai/gpt-4o-mini") + + router = Router( + model_list=[ + { + "model_name": "offpeak-only", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "fake-key-for-registration", + }, + "model_info": {"id": deployment_id, "off_peak_pricing": dict(block)}, + } + ] + ) + + try: + entry = litellm.model_cost[deployment_id] + assert entry["off_peak_pricing"] == block + assert entry["input_cost_per_token"] is not None + assert entry["input_cost_per_token"] == builtin_info["input_cost_per_token"] + assert entry["output_cost_per_token"] == builtin_info["output_cost_per_token"] + for shared_key in shared_keys: + shared_entry = litellm.model_cost.get(shared_key) or {} + assert not shared_entry.get("off_peak_pricing") + finally: + _restore_model_cost_entries(original_entries) + del router + + +def test_use_custom_pricing_for_model_sees_off_peak_only_model_info(): + from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model + + block = {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-05} + assert use_custom_pricing_for_model({"metadata": {"model_info": {"off_peak_pricing": block}}}) is True + assert use_custom_pricing_for_model({"metadata": {"model_info": {"off_peak_pricing": None}}}) is False + assert use_custom_pricing_for_model({"metadata": {"model_info": {"id": "some-id"}}}) is False + + +def test_completion_cost_applies_off_peak_only_deployment_pricing(): + """End to end through the cost calculator: with ``custom_pricing`` set and + a ``router_model_id`` whose entry carries only an always-on off-peak block, + the request bills at the block's rates rather than the shared backend rate. + """ + from litellm import Router + from litellm.types.utils import ModelResponse, Usage + + block = { + "hours_utc": "00:00-00:00", + "input_cost_per_token": 5e-05, + "output_cost_per_token": 1e-04, + } + shared_keys = ["gpt-4o-mini", "openai/gpt-4o-mini"] + deployment_id = "offpeak-only-dep-2" + original_entries = _snapshot_model_cost_entries(shared_keys + [deployment_id]) + + router = Router( + model_list=[ + { + "model_name": "offpeak-only", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "fake-key-for-registration", + }, + "model_info": {"id": deployment_id, "off_peak_pricing": dict(block)}, + } + ] + ) + + try: + response = ModelResponse( + model="gpt-4o-mini", + usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150), + ) + cost = litellm.completion_cost( + completion_response=response, + model="openai/gpt-4o-mini", + custom_llm_provider="openai", + custom_pricing=True, + router_model_id=deployment_id, + ) + assert cost == pytest.approx(100 * 5e-05 + 50 * 1e-04) + finally: + _restore_model_cost_entries(original_entries) + del router diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index b580b03574e..7b7a962bf00 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -538,6 +538,157 @@ def test_inherit_builtin_cache_pricing_noop_for_unknown_backend(): assert model_info == {"input_cost_per_token": 0.000003} +def test_inherit_builtin_base_rates_for_off_peak_fills_missing_rates(): + """Direct unit test of the helper: an entry carrying only an + off_peak_pricing block inherits the backend model's built-in base token + rates, so cost lookup via the deployment id can bill standard rates + outside the windows. + """ + backend_model = "gpt-4o-mini" + builtin_info = litellm.get_model_info(model=backend_model, custom_llm_provider="openai") + off_peak_block = { + "hours_utc": "00:00-00:00", + "input_cost_per_token": 5e-07, + "output_cost_per_token": 1e-06, + } + model_info = {"off_peak_pricing": off_peak_block} + + Router._inherit_builtin_base_rates_for_off_peak( + model_info=model_info, + backend_model=backend_model, + custom_llm_provider="openai", + ) + + assert model_info["input_cost_per_token"] == builtin_info["input_cost_per_token"] + assert model_info["output_cost_per_token"] == builtin_info["output_cost_per_token"] + assert model_info["off_peak_pricing"] == off_peak_block + + +def test_inherit_builtin_base_rates_for_off_peak_carries_threshold_rates(): + """A backend with above-threshold pricing hands the whole rate structure to + the deployment entry, so peak-hour billing of large prompts through that + entry matches the shared backend entry instead of flattening to the base + rate. + """ + backend_model = "gemini/gemini-2.5-pro" + builtin_info = litellm.get_model_info(model=backend_model) + assert builtin_info["input_cost_per_token_above_200k_tokens"] is not None + + model_info = { + "off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07}, + } + + Router._inherit_builtin_base_rates_for_off_peak( + model_info=model_info, + backend_model=backend_model, + custom_llm_provider="gemini", + ) + + assert model_info["input_cost_per_token"] == builtin_info["input_cost_per_token"] + assert ( + model_info["input_cost_per_token_above_200k_tokens"] + == builtin_info["input_cost_per_token_above_200k_tokens"] + ) + assert ( + model_info["output_cost_per_token_above_200k_tokens"] + == builtin_info["output_cost_per_token_above_200k_tokens"] + ) + + +def test_inherit_builtin_base_rates_for_off_peak_carries_companion_billing_fields(): + """Billing rules that are not literal cost rates, like the web search + billing unit, must ride along, or grounding and regional uplifts would + bill differently through the deployment entry than through the shared + backend entry. + """ + backend_model = "gemini-3-pro-image" + raw_entry = litellm.model_cost[backend_model] + assert raw_entry.get("web_search_billing_unit") is not None + + model_info = { + "off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07}, + } + + Router._inherit_builtin_base_rates_for_off_peak( + model_info=model_info, + backend_model=backend_model, + custom_llm_provider=None, + ) + + assert model_info["web_search_billing_unit"] == raw_entry["web_search_billing_unit"] + assert model_info["input_cost_per_token"] == raw_entry["input_cost_per_token"] + + +def test_inherit_builtin_base_rates_for_off_peak_tiered_only_backend_stores_no_zero(): + """A tiered-only backend has no flat token rates; get_model_info synthesizes + zeros for them, and storing those would mark the deployment explicitly + priced free. The tier table itself must carry over as an isolated copy so + mutating the deployment entry never touches the shared cost map. + """ + backend_model = "dashscope/qwen-flash" + raw_tiers = litellm.model_cost[backend_model]["tiered_pricing"] + + model_info = { + "off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07}, + } + + Router._inherit_builtin_base_rates_for_off_peak( + model_info=model_info, + backend_model=backend_model, + custom_llm_provider="dashscope", + ) + + assert model_info.get("input_cost_per_token") != 0 + assert model_info.get("output_cost_per_token") != 0 + assert model_info["tiered_pricing"] == raw_tiers + assert model_info["tiered_pricing"] is not raw_tiers + assert model_info["tiered_pricing"][0] is not raw_tiers[0] + + original_first_tier = copy.deepcopy(raw_tiers[0]) + model_info["tiered_pricing"][0]["input_cost_per_token"] = 123.0 + assert raw_tiers[0] == original_first_tier + + +def test_inherit_builtin_base_rates_for_off_peak_leaves_explicit_rates_alone(): + """An entry that sets its own base rate beside the block already counts as + a full custom pricing entry; the helper must not mix builtin rates into it. + """ + model_info = { + "off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07}, + "input_cost_per_token": 3e-06, + } + + Router._inherit_builtin_base_rates_for_off_peak( + model_info=model_info, + backend_model="gpt-4o-mini", + custom_llm_provider="openai", + ) + + assert model_info["input_cost_per_token"] == 3e-06 + assert "output_cost_per_token" not in model_info + + +def test_inherit_builtin_base_rates_for_off_peak_noop_without_block_or_backend(): + """Nothing happens without an off_peak_pricing block, and an unmapped + backend model leaves the entry unchanged rather than raising. + """ + plain_info = {"id": "dep-1"} + Router._inherit_builtin_base_rates_for_off_peak( + model_info=plain_info, + backend_model="gpt-4o-mini", + custom_llm_provider="openai", + ) + assert plain_info == {"id": "dep-1"} + + off_peak_info = {"off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07}} + Router._inherit_builtin_base_rates_for_off_peak( + model_info=off_peak_info, + backend_model="this-backend-model-does-not-exist-x9y8z7", + custom_llm_provider=None, + ) + assert "input_cost_per_token" not in off_peak_info + + def test_custom_pricing_field_denylist_covers_all_builtin_pricing_fields(): """The shared-backend-key stripping in Router relies on CustomPricingLiteLLMParams enumerating every per-deployment pricing field. diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsTable.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsTable.test.tsx index 06099a9fc22..4d18ec2ef5f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsTable.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsTable.test.tsx @@ -74,6 +74,42 @@ describe("AgentsTable", () => { expect(onDeleteClick).toHaveBeenCalledWith("agent-9", "Doomed Agent"); }); + it("filters agents by name or by agent card description", async () => { + const user = userEvent.setup(); + render( + , + ); + + const search = screen.getByPlaceholderText("Search agent names or descriptions..."); + await user.type(search, "billing"); + expect(screen.getByText("Billing Router")).toBeInTheDocument(); + expect(screen.queryByText("Second Agent")).not.toBeInTheDocument(); + + await user.clear(search); + await user.type(search, "support tickets"); + expect(screen.getByText("Second Agent")).toBeInTheDocument(); + expect(screen.queryByText("Billing Router")).not.toBeInTheDocument(); + }); + + it("shows the no-match empty state when the search matches nothing", async () => { + const user = userEvent.setup(); + render(); + + await user.type(screen.getByPlaceholderText("Search agent names or descriptions..."), "zzzz"); + expect(screen.queryByText("Test Agent")).not.toBeInTheDocument(); + expect(screen.getByText("No matching agents")).toBeInTheDocument(); + }); + it("hides the actions column entirely for non-admins", () => { const agent = makeAgent({ agent_id: "agent-2" }); render(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsTable.tsx index 67c7ed74180..35ed6b66425 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsTable.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsTable.tsx @@ -1,13 +1,15 @@ "use client"; import { SortingState } from "@tanstack/react-table"; -import { Bot, CircleCheck } from "lucide-react"; +import { Bot, CircleCheck, Search as SearchIcon, X } from "lucide-react"; import React, { useMemo, useState } from "react"; import { Agent } from "@/components/agents/types"; import { DataTable } from "@/components/shared/DataTable"; +import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from "@/components/ui/input-group"; import { Switch } from "@/components/ui/switch"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; +import { filterBySearchTerm } from "@/utils/searchUtils"; import { getAgentsTableColumns } from "./AgentsTableColumns"; @@ -24,14 +26,18 @@ interface AgentsTableProps { const DEFAULT_SORTING: SortingState = [{ id: "created_at", desc: true }]; -function EmptyState() { +function EmptyState({ isFiltered }: { isFiltered: boolean }) { return (
-
No agents yet
-
Add an agent to make it available in your organization.
+
{isFiltered ? "No matching agents" : "No agents yet"}
+
+ {isFiltered + ? "Adjust the search to see more agents." + : "Add an agent to make it available in your organization."} +
); } @@ -47,6 +53,11 @@ const AgentsTable: React.FC = ({ onDeleteClick, }) => { const [sorting, setSorting] = useState(DEFAULT_SORTING); + const [searchTerm, setSearchTerm] = useState(""); + const filteredAgents = useMemo( + () => filterBySearchTerm(agents, searchTerm, (agent) => [agent.agent_name, agent.agent_card_params?.description]), + [agents, searchTerm], + ); const columns = useMemo( () => getAgentsTableColumns({ isAdmin, onAgentClick, onDeleteClick }), @@ -55,7 +66,7 @@ const AgentsTable: React.FC = ({ return ( agent.agent_id || String(index)} sortingMode="client" @@ -63,10 +74,27 @@ const AgentsTable: React.FC = ({ onSortingChange={setSorting} isLoading={isLoading} loadingMessage="Loading agents…" - noDataMessage={} + noDataMessage={ 0} />} size="compact" toolbar={() => ( -
+
+ + + + + setSearchTerm(e.target.value)} + /> + {searchTerm && ( + + setSearchTerm("")}> + + + + )} + ({ createAgentCall: vi.fn(), @@ -309,8 +310,7 @@ describe("AddAgentForm submit payload", () => { await user.type(await screen.findByLabelText("Allowed Models"), "gpt-4o,"); await user.keyboard("{Escape}"); - await user.click(screen.getByLabelText("Allowed Agents (Sub-Agents)")); - await user.click(await screen.findByTitle("Sub Agent One")); + await chooseSelectOption(user, screen.getByLabelText("Allowed Agents (Sub-Agents)"), "Sub Agent One"); await user.keyboard("{Escape}"); await user.click(screen.getByText(/Configure which models, agents, and MCP tools/)); await user.click(screen.getByRole("button", { name: /^Next/ })); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_modal.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_modal.integration.test.tsx index bc920e55abd..17a297d203d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_modal.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_modal.integration.test.tsx @@ -4,6 +4,7 @@ import React from "react"; import { beforeEach, describe, expect, it, vi } from "vitest"; import BudgetModal from "./budget_modal"; +import { chooseSelectOption } from "../../../../../tests/test-utils"; const { createMock } = vi.hoisted(() => ({ createMock: vi.fn() })); @@ -63,8 +64,7 @@ describe("BudgetModal", () => { await openOptionalSettings(user); fireEvent.change(screen.getByLabelText("Max Budget (USD)"), { target: { value: "42.567" } }); - await user.click(screen.getByRole("combobox")); - await user.click(await screen.findByText("monthly")); + await chooseSelectOption(user, screen.getByRole("combobox"), "monthly"); await create(user); @@ -80,8 +80,7 @@ describe("BudgetModal", () => { await openOptionalSettings(user); fireEvent.change(screen.getByLabelText("Max Budget (USD)"), { target: { value: "42.567" } }); - await user.click(screen.getByRole("combobox")); - await user.click(await screen.findByText("monthly")); + await chooseSelectOption(user, screen.getByRole("combobox"), "monthly"); await user.click(screen.getByText("Optional Settings")); await waitFor(() => expect(screen.queryByLabelText("Max Budget (USD)")).not.toBeInTheDocument()); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/edit_budget_modal.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/edit_budget_modal.integration.test.tsx index 3fa96b54f1d..fe0ecfc10dd 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/edit_budget_modal.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/edit_budget_modal.integration.test.tsx @@ -6,6 +6,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import type { components } from "@/lib/http/schema"; import EditBudgetModal from "./edit_budget_modal"; +import { chooseSelectOption } from "../../../../../tests/test-utils"; const { updateMock } = vi.hoisted(() => ({ updateMock: vi.fn() })); @@ -73,8 +74,7 @@ describe("EditBudgetModal", () => { await user.clear(screen.getByLabelText("Max Budget (USD)")); fireEvent.change(screen.getByLabelText("Max Budget (USD)"), { target: { value: "42.567" } }); - await user.click(screen.getByRole("combobox")); - await user.click(await screen.findByText("monthly")); + await chooseSelectOption(user, screen.getByRole("combobox"), "monthly"); await save(user); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/coordination_redis_settings/CoordinationRedisTypeSelector.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/coordination_redis_settings/CoordinationRedisTypeSelector.test.tsx index 76287e6e724..563c684237a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/coordination_redis_settings/CoordinationRedisTypeSelector.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/coordination_redis_settings/CoordinationRedisTypeSelector.test.tsx @@ -5,6 +5,7 @@ import userEvent from "@testing-library/user-event"; import { renderWithProviders } from "@/../tests/test-utils"; import CoordinationRedisTypeSelector from "./CoordinationRedisTypeSelector"; import { COORDINATION_REDIS_TYPE_DESCRIPTIONS } from "./coordinationRedisFields"; +import { chooseSelectOption } from "../../../../../../tests/test-utils"; describe("CoordinationRedisTypeSelector", () => { it("labels the control and shows the current selection", () => { @@ -36,8 +37,7 @@ describe("CoordinationRedisTypeSelector", () => { const user = userEvent.setup(); renderWithProviders(); - await user.click(screen.getByRole("combobox")); - await user.click(await screen.findByText("Cluster")); + await chooseSelectOption(user, screen.getByRole("combobox"), "Cluster"); expect(onTypeChange).toHaveBeenCalledTimes(1); expect(onTypeChange.mock.calls[0][0]).toBe("cluster"); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx index 64e03985f57..a1de608d0bb 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx @@ -97,6 +97,7 @@ import { useStopShadowEval, type ShadowEvalJob, } from "./useShadowEval"; +import { chooseSelectOption } from "../../../../../tests/test-utils"; const job = (overrides: Partial = {}): ShadowEvalJob => ({ job_id: "job-1", @@ -437,8 +438,7 @@ describe("ShadowEvalSection", () => { await user.click(within(keyList).getByText("prod-alpha")); await user.click(keyInput); await user.click(within(keyList).getByText("staging-beta")); - await user.click(screen.getByPlaceholderText("Select up to 4 auto-routers")); - await user.click(await screen.findByText("gpt-auto")); + await chooseSelectOption(user, screen.getByPlaceholderText("Select up to 4 auto-routers"), "gpt-auto"); expect(screen.getByText("Start shadow eval")).toBeDisabled(); @@ -470,8 +470,7 @@ describe("ShadowEvalSection", () => { await user.click(screen.getByPlaceholderText("Search teams by alias")); const teamList = await screen.findByTestId("paginated-multi-select-list"); await user.click(within(teamList).getByText("engineering")); - await user.click(screen.getByPlaceholderText("Select up to 4 auto-routers")); - await user.click(await screen.findByText("gpt-auto")); + await chooseSelectOption(user, screen.getByPlaceholderText("Select up to 4 auto-routers"), "gpt-auto"); await user.click(screen.getByPlaceholderText("Select a judge model")); await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ })); await user.click(screen.getByText("Start shadow eval")); @@ -502,8 +501,7 @@ describe("ShadowEvalSection", () => { await user.click(screen.getByPlaceholderText("Search keys by alias")); const keyList = await screen.findByTestId("paginated-multi-select-list"); await user.click(within(keyList).getByText("prod-alpha")); - await user.click(screen.getByPlaceholderText("Select up to 4 auto-routers")); - await user.click(await screen.findByText("gpt-auto")); + await chooseSelectOption(user, screen.getByPlaceholderText("Select up to 4 auto-routers"), "gpt-auto"); await user.click(screen.getByPlaceholderText("Select a judge model")); await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ })); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/ToolTestPanel.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/ToolTestPanel.test.tsx index 26e66c8338c..81c55e7982a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/ToolTestPanel.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/ToolTestPanel.test.tsx @@ -6,6 +6,7 @@ import { describe, expect, it, vi } from "vitest"; import { ToolTestPanel } from "./ToolTestPanel"; import { InputSchema, MCPTool } from "@/components/mcp_tools/types"; +import { chooseSelectOption } from "../../../../../tests/test-utils"; const buildTool = (schema: InputSchema | string): MCPTool => ({ name: "demo-tool", @@ -229,8 +230,7 @@ describe("ToolTestPanel argument payload", () => { const onSubmit = await submitPanel( { type: "object", properties: { active: { type: "boolean", default: false } } }, async (user) => { - await user.click(screen.getByLabelText("active")); - await user.click(await screen.findByText("True")); + await chooseSelectOption(user, screen.getByLabelText("active"), "True"); }, ); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx index 19e1e3aa8bd..efba26734ff 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx @@ -87,7 +87,6 @@ const alwaysMounted = { api_key: undefined, api_base: undefined, custom_llm_provider: "openai", - litellm_credential_name: null, model: "gpt-4o", }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/policy_test_panel.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/policy_test_panel.test.tsx index c8779be06c4..f23145a917a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/policy_test_panel.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/policy_test_panel.test.tsx @@ -5,6 +5,7 @@ import userEvent, { PointerEventsCheckLevel } from "@testing-library/user-event" import { renderWithProviders } from "@/../tests/test-utils"; import * as networking from "@/components/networking"; import PolicyTestPanel from "./policy_test_panel"; +import { chooseSelectOption } from "../../../../../tests/test-utils"; vi.mock("@/components/networking"); @@ -24,8 +25,7 @@ const setup = () => { }; const pickOption = async (user: ReturnType, label: string, option: string) => { - await user.click(screen.getByLabelText(label)); - await user.click(await screen.findByTitle(option)); + await chooseSelectOption(user, screen.getByLabelText(label), option); }; const simulate = async (user: ReturnType) => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/index.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/index.test.tsx index d6a4aaea2e1..3b0ee3a3ce2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/index.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/index.test.tsx @@ -5,6 +5,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { deletePromptCall, getPromptsList } from "@/components/networking"; import PromptsPanel from "./index"; +import { chooseSelectOption } from "../../../../../tests/test-utils"; vi.mock("@/components/networking", () => ({ getPromptsList: vi.fn(), @@ -118,8 +119,7 @@ describe("PromptsPanel toolbar", () => { expect(screen.getByText("All Environments")).toBeInTheDocument(); - await user.click(screen.getByRole("combobox")); - await user.click(await screen.findByText("Production")); + await chooseSelectOption(user, screen.getByRole("combobox"), "Production"); await waitFor(() => expect(mockGetPromptsList).toHaveBeenLastCalledWith("sk-test", "production")); }); @@ -131,12 +131,10 @@ describe("PromptsPanel toolbar", () => { renderPanel("Admin"); await screen.findByText("table-loaded"); - await user.click(screen.getByRole("combobox")); - await user.click(await screen.findByText("Production")); + await chooseSelectOption(user, screen.getByRole("combobox"), "Production"); await waitFor(() => expect(screen.getByRole("combobox")).toHaveTextContent("Production")); - await user.click(screen.getByRole("combobox")); - await user.click(await screen.findByText("All Environments")); + await chooseSelectOption(user, screen.getByRole("combobox"), "All Environments"); await waitFor(() => expect(screen.getByRole("combobox")).toHaveTextContent("All Environments")); await waitFor(() => expect(mockGetPromptsList).toHaveBeenLastCalledWith("sk-test", undefined)); diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx index 397e2ab10fa..906565750b1 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx @@ -203,12 +203,12 @@ describe("ModelHubTable", () => { }); describe("hub tabs", () => { - const renderHub = async () => { + const renderHub = async (agents: object[] = []) => { vi.mocked(networking.modelHubCall).mockResolvedValue({ data: [{ model_group: "claude-opus-4-8", providers: ["anthropic"], mode: "chat" }], }); vi.mocked(networking.getConfigFieldSetting).mockResolvedValue({ field_value: false }); - vi.mocked(networking.getAgentsList).mockResolvedValue({ agents: [] }); + vi.mocked(networking.getAgentsList).mockResolvedValue({ agents }); vi.mocked(networking.fetchMCPServers).mockResolvedValue([]); vi.mocked(networking.getUiSettings).mockResolvedValue({ values: {} }); mockUseUISettings.mockReturnValue({ data: { values: {} }, isLoading: false }); @@ -230,6 +230,37 @@ describe("ModelHubTable", () => { expect(await screen.findByPlaceholderText("Search model names...")).toHaveValue("opus"); }); + it("filters the Agent Hub table by name or description and shows the no-match state", async () => { + const { user } = await renderHub([ + { + agent_id: "a1", + agent_card_params: { name: "Billing Router", description: "routes billing questions" }, + litellm_params: { is_public: false }, + }, + { + agent_id: "a2", + agent_card_params: { name: "Support Bot", description: "handles support tickets" }, + litellm_params: { is_public: false }, + }, + ]); + const agentCount = (expected: string) => + screen.getByText((_, el) => el?.tagName === "P" && el.textContent === expected); + + await user.click(screen.getByRole("tab", { name: "Agent Hub" })); + expect(await screen.findByText("Billing Router")).toBeInTheDocument(); + + const search = screen.getByPlaceholderText("Search agent names or descriptions..."); + await user.type(search, "support tickets"); + expect(screen.queryByText("Billing Router")).not.toBeInTheDocument(); + expect(screen.getByText("Support Bot")).toBeInTheDocument(); + expect(agentCount("Showing 1 of 2 agents")).toBeInTheDocument(); + + await user.clear(search); + await user.type(search, "zzzz"); + expect(screen.getByText("No matching agents")).toBeInTheDocument(); + expect(agentCount("Showing 0 of 2 agents")).toBeInTheDocument(); + }); + it("renders the hub strip as underlined tabs rather than a segmented pill", async () => { await renderHub(); diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx index 299104b271b..475e3dcd70b 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx @@ -23,13 +23,15 @@ import { import PublicModelHub from "@/components/public_model_hub"; import { copyToClipboard } from "@/utils/dataUtils"; import { isAdminRole, isProxyAdminRole } from "@/utils/roles"; +import { filterBySearchTerm } from "@/utils/searchUtils"; import { SortingState } from "@tanstack/react-table"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Card } from "@/components/ui/card"; import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog"; +import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from "@/components/ui/input-group"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; -import { Copy, Inbox } from "lucide-react"; +import { Copy, Inbox, Search as SearchIcon, X } from "lucide-react"; import { useRouter } from "next/navigation"; import React, { useCallback, useEffect, useMemo, useState } from "react"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; @@ -80,6 +82,7 @@ const ModelHubTable: React.FC = ({ accessToken, publicPage, const [agentLoading, setAgentLoading] = useState(true); const [selectedAgent, setSelectedAgent] = useState(null); const [isAgentModalVisible, setIsAgentModalVisible] = useState(false); + const [agentSearchTerm, setAgentSearchTerm] = useState(""); // MCP Hub state const [mcpHubData, setMcpHubData] = useState(null); const [mcpLoading, setMcpLoading] = useState(true); @@ -385,6 +388,10 @@ const ModelHubTable: React.FC = ({ accessToken, publicPage, const modelColumns = useMemo(() => getModelHubTableColumns({ onModelClick: showModal }), [showModal]); const agentColumns = useMemo(() => getAgentHubTableColumns({ onAgentClick: showAgentModal }), [showAgentModal]); + const filteredAgentData = useMemo( + () => filterBySearchTerm(agentHubData ?? [], agentSearchTerm, (agent) => [agent.name, agent.description]), + [agentHubData, agentSearchTerm], + ); const mcpColumns = useMemo(() => getMCPHubTableColumns({ onServerClick: showMcpModal }), [showMcpModal]); // If this is a public page, use the dedicated PublicModelHub component @@ -505,9 +512,34 @@ const ModelHubTable: React.FC = ({ accessToken, publicPage,
)} +
+

Search Agents:

+ + + + + setAgentSearchTerm(e.target.value)} + /> + {agentSearchTerm && ( + + setAgentSearchTerm("")} + > + + + + )} + +
+ {/* Agent Table */} agent.agent_id || agent.name || String(index)} sortingMode="client" @@ -516,7 +548,14 @@ const ModelHubTable: React.FC = ({ accessToken, publicPage, isLoading={agentLoading} loadingMessage="Loading agents…" noDataMessage={ - + } size="compact" /> @@ -524,7 +563,7 @@ const ModelHubTable: React.FC = ({ accessToken, publicPage,

- Showing {agentHubData?.length || 0} agent{agentHubData?.length !== 1 ? "s" : ""} + Showing {filteredAgentData.length} of {agentHubData?.length || 0} agents

diff --git a/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.test.tsx b/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.test.tsx index 9375673ba10..d3b9a55a7d5 100644 --- a/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.test.tsx +++ b/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.test.tsx @@ -2,7 +2,7 @@ import { fireEvent, screen, waitFor, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, MockedFunction, vi } from "vitest"; -import { renderWithProviders } from "../../../tests/test-utils"; +import { chooseSelectOption, renderWithProviders } from "../../../tests/test-utils"; import { Team } from "../key_team_helpers/key_list"; import { TeamsResponse, useTeamsTable } from "@/app/(dashboard)/hooks/teams/useTeams"; import { TeamsTable } from "./TeamsTable"; @@ -243,8 +243,7 @@ describe("row actions", () => { await user.click(await screen.findByText("Edit team")); expect(onEditTeam).toHaveBeenCalledWith(expect.objectContaining({ team_id: "team-1" })); - await user.click(screen.getByTestId("team-actions-team-1")); - await user.click(await screen.findByText("Delete team")); + await chooseSelectOption(user, screen.getByTestId("team-actions-team-1"), "Delete team", "menuitem"); expect(onDeleteTeam).toHaveBeenCalledWith(expect.objectContaining({ team_id: "team-1" })); }); diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx index d09ccbf8826..b06448912d0 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx @@ -2,7 +2,7 @@ import { screen, waitFor, within, fireEvent } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import type { OnUrlUpdateFunction } from "nuqs/adapters/testing"; import { vi, it, expect, beforeEach, describe, Mock, MockedFunction } from "vitest"; -import { renderWithProviders } from "../../../tests/test-utils"; +import { chooseSelectOption, renderWithProviders } from "../../../tests/test-utils"; import { VirtualKeysTable } from "./VirtualKeysTable"; import { KeyResponse, Team } from "../key_team_helpers/key_list"; import { useKeyInfo } from "@/app/(dashboard)/hooks/keys/useKeyInfo"; @@ -327,8 +327,7 @@ it("sorts by the backend max_budget field when 'Budget descending' is chosen fro const user = userEvent.setup(); renderWithProviders(); - await user.click(screen.getByTestId("sort-trigger-spend")); - await user.click(await screen.findByText("Budget descending")); + await chooseSelectOption(user, screen.getByTestId("sort-trigger-spend"), "Budget descending", "menuitem"); await waitFor(() => { expect(mockUseKeys).toHaveBeenLastCalledWith( @@ -343,8 +342,7 @@ it("emphasizes the active field in the Spend / Budget header so the sorted colum const user = userEvent.setup(); renderWithProviders(); - await user.click(screen.getByTestId("sort-trigger-spend")); - await user.click(await screen.findByText("Budget descending")); + await chooseSelectOption(user, screen.getByTestId("sort-trigger-spend"), "Budget descending", "menuitem"); await waitFor(() => { expect(screen.getByText("Budget", { selector: "[data-sort-field='max_budget']" })).toHaveClass("font-semibold"); @@ -356,8 +354,7 @@ it("sorts by spend ascending when 'Spend ascending' is chosen from the Spend / B const user = userEvent.setup(); renderWithProviders(); - await user.click(screen.getByTestId("sort-trigger-spend")); - await user.click(await screen.findByText("Spend ascending")); + await chooseSelectOption(user, screen.getByTestId("sort-trigger-spend"), "Spend ascending", "menuitem"); await waitFor(() => { expect(mockUseKeys).toHaveBeenLastCalledWith(1, 50, expect.objectContaining({ sortBy: "spend", sortOrder: "asc" })); diff --git a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.test.tsx b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.test.tsx index 7ef09d34924..9d792480c9f 100644 --- a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.test.tsx @@ -1,6 +1,10 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { prepareModelAddRequest } from "./handle_add_model_submit"; +vi.mock("../networking", () => ({ + modelCreateCall: vi.fn(), +})); + describe("prepareModelAddRequest", () => { it("returns deployment data for the most basic form", async () => { const formValues = { @@ -73,4 +77,28 @@ describe("prepareModelAddRequest", () => { expect(deployment.litellmParamsObj.litellm_credential_name).toBe("selected-credential"); expect(deployment.litellmParamsObj.timeout).toBe(5); }); + + it("keeps litellm_credential_name from LiteLLM Params JSON when no credential is selected", async () => { + const formValues = { + model_mappings: [ + { + public_name: "Public Model", + litellm_model: "litellm/public", + }, + ], + model_name: "custom-model-name", + litellm_extra_params: JSON.stringify({ + litellm_credential_name: "from-json", + timeout: 5, + }), + litellm_credential_name: null, + }; + + const deployments = await prepareModelAddRequest({ ...formValues }, "token", null); + + expect(deployments).toHaveLength(1); + const [deployment] = deployments!; + expect(deployment.litellmParamsObj.litellm_credential_name).toBe("from-json"); + expect(deployment.litellmParamsObj.timeout).toBe(5); + }); }); diff --git a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx index bb2f78fa84e..41133958c0a 100644 --- a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx +++ b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx @@ -91,6 +91,9 @@ export const prepareModelAddRequest = async (formValues: Record, ac if (value === "") { continue; } + if (key === "litellm_credential_name" && value == null) { + continue; + } // Skip the custom_pricing and pricing_model fields as they're only used for UI control if (key === "custom_pricing" || key === "pricing_model" || key === "cache_control") { continue; @@ -124,13 +127,13 @@ export const prepareModelAddRequest = async (formValues: Record, ac if (value && value != undefined) { try { litellmExtraParams = JSON.parse(value); - if ("litellm_credential_name" in litellmExtraParams) { - delete litellmExtraParams.litellm_credential_name; - } } catch (error) { toast.fromError("Failed to parse LiteLLM Extra Params: " + error); throw new Error("Failed to parse litellm_extra_params: " + error); } + if ("litellm_credential_name" in litellmExtraParams && formValues.litellm_credential_name) { + delete litellmExtraParams.litellm_credential_name; + } for (const [key, value] of Object.entries(litellmExtraParams)) { litellmParamsObj[key] = value; } diff --git a/ui/litellm-dashboard/src/components/add_pass_through.integration.test.tsx b/ui/litellm-dashboard/src/components/add_pass_through.integration.test.tsx index df4dbff4111..daaacd7f0e2 100644 --- a/ui/litellm-dashboard/src/components/add_pass_through.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_pass_through.integration.test.tsx @@ -1,6 +1,6 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import userEvent, { PointerEventsCheckLevel } from "@testing-library/user-event"; -import { fireEvent, renderWithProviders, screen, waitFor } from "../../tests/test-utils"; +import { chooseSelectOption, fireEvent, renderWithProviders, screen, waitFor } from "../../tests/test-utils"; import AddPassThroughEndpoint from "./add_pass_through"; const createPassThroughEndpoint = vi.fn(); @@ -153,8 +153,7 @@ describe("add_pass_through submit payload", () => { await openModal(user); await fillRequiredFields(user); - await user.click(screen.getByLabelText(/HTTP Methods/)); - await user.click(await screen.findByTitle("POST")); + await chooseSelectOption(user, screen.getByLabelText(/HTTP Methods/), "POST"); await submit(user); diff --git a/ui/litellm-dashboard/src/components/common_components/KeyLifecycleSettings.test.tsx b/ui/litellm-dashboard/src/components/common_components/KeyLifecycleSettings.test.tsx index 6e627e3b45c..df21444cbad 100644 --- a/ui/litellm-dashboard/src/components/common_components/KeyLifecycleSettings.test.tsx +++ b/ui/litellm-dashboard/src/components/common_components/KeyLifecycleSettings.test.tsx @@ -2,7 +2,7 @@ import React, { useState } from "react"; import { Controller, useForm } from "react-hook-form"; import userEvent, { PointerEventsCheckLevel } from "@testing-library/user-event"; import { describe, expect, it, vi, beforeEach } from "vitest"; -import { fireEvent, renderWithProviders, screen, waitFor } from "../../../tests/test-utils"; +import { chooseSelectOption, fireEvent, renderWithProviders, screen, waitFor } from "../../../tests/test-utils"; import KeyLifecycleSettings from "./KeyLifecycleSettings"; const CREATE_PLACEHOLDER = "e.g., 30d or leave empty to never expire"; @@ -167,8 +167,7 @@ describe("KeyLifecycleSettings", () => { await user.click(screen.getByRole("switch")); expect(await screen.findByText("Rotation Interval")).toBeInTheDocument(); - await user.click(screen.getByRole("combobox")); - await user.click(await screen.findByText("90 days")); + await chooseSelectOption(user, screen.getByRole("combobox"), "90 days"); await waitFor(() => expect(screen.getAllByTitle("90 days").some(isRenderedSelection)).toBe(true)); expect(screen.getByTestId("rotation-interval-value")).toHaveTextContent("90d"); @@ -181,8 +180,7 @@ describe("KeyLifecycleSettings", () => { await user.click(screen.getByRole("switch")); expect(await screen.findByText("Rotation Interval")).toBeInTheDocument(); - await user.click(screen.getByRole("combobox")); - await user.click(await screen.findByText("Custom interval")); + await chooseSelectOption(user, screen.getByRole("combobox"), "Custom interval"); expect(await screen.findByPlaceholderText("e.g., 1s, 5m, 2h, 14d")).toBeInTheDocument(); expect(screen.getByText("Supported formats: seconds (s), minutes (m), hours (h), days (d)")).toBeInTheDocument(); @@ -196,8 +194,7 @@ describe("KeyLifecycleSettings", () => { await user.click(screen.getByRole("switch")); expect(await screen.findByText("Rotation Interval")).toBeInTheDocument(); - await user.click(screen.getByRole("combobox")); - await user.click(await screen.findByText("Custom interval")); + await chooseSelectOption(user, screen.getByRole("combobox"), "Custom interval"); const customInput = await screen.findByPlaceholderText("e.g., 1s, 5m, 2h, 14d"); fireEvent.change(customInput, { target: { value: "14d" } }); @@ -213,14 +210,12 @@ describe("KeyLifecycleSettings", () => { await user.click(screen.getByRole("switch")); expect(await screen.findByText("Rotation Interval")).toBeInTheDocument(); - await user.click(screen.getByRole("combobox")); - await user.click(await screen.findByText("Custom interval")); + await chooseSelectOption(user, screen.getByRole("combobox"), "Custom interval"); const customInput = await screen.findByPlaceholderText("e.g., 1s, 5m, 2h, 14d"); fireEvent.change(customInput, { target: { value: "14d" } }); await waitFor(() => expect(screen.getByTestId("rotation-interval-value")).toHaveTextContent("14d")); - await user.click(screen.getByRole("combobox")); - await user.click(await screen.findByText("7 days")); + await chooseSelectOption(user, screen.getByRole("combobox"), "7 days"); await waitFor(() => expect(screen.getByTestId("rotation-interval-value")).toHaveTextContent("7d")); expect(screen.queryByPlaceholderText("e.g., 1s, 5m, 2h, 14d")).not.toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/components/common_components/ModelSelector.test.tsx b/ui/litellm-dashboard/src/components/common_components/ModelSelector.test.tsx index bd7b56f1817..7a5db4667e9 100644 --- a/ui/litellm-dashboard/src/components/common_components/ModelSelector.test.tsx +++ b/ui/litellm-dashboard/src/components/common_components/ModelSelector.test.tsx @@ -2,6 +2,7 @@ import { act, fireEvent, render, screen } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { afterEach, describe, expect, it, vi } from "vitest"; import ModelSelector from "./ModelSelector"; +import { chooseSelectOption } from "../../../tests/test-utils"; vi.mock("@/components/llm_calls/fetch_models", () => ({ fetchAvailableModels: vi.fn().mockResolvedValue([]), @@ -9,8 +10,7 @@ vi.mock("@/components/llm_calls/fetch_models", () => ({ const openCustomModelInput = async () => { const user = userEvent.setup(); - await user.click(screen.getByRole("combobox")); - await user.click(await screen.findByText("Enter custom model")); + await chooseSelectOption(user, screen.getByRole("combobox"), "Enter custom model"); return screen.getByPlaceholderText("Enter custom model name"); }; diff --git a/ui/litellm-dashboard/src/components/model_filters.tsx b/ui/litellm-dashboard/src/components/model_filters.tsx index 96041905d97..0ce82d6d050 100644 --- a/ui/litellm-dashboard/src/components/model_filters.tsx +++ b/ui/litellm-dashboard/src/components/model_filters.tsx @@ -1,5 +1,6 @@ import React, { useState, useEffect, useMemo, useRef } from "react"; import { Card } from "@/components/ui/card"; +import { matchesSearchTerm } from "@/utils/searchUtils"; interface ModelGroupInfo { model_group: string; @@ -76,7 +77,7 @@ const ModelFilters: React.FC = ({ const filteredData = useMemo(() => { return ( modelHubData?.filter((model) => { - const matchesSearch = model.model_group.toLowerCase().includes(searchTerm.toLowerCase()); + const matchesSearch = matchesSearchTerm(searchTerm, [model.model_group]); const matchesProvider = selectedProvider === "" || model.providers.includes(selectedProvider); const matchesMode = selectedMode === "" || model.mode === selectedMode; diff --git a/ui/litellm-dashboard/src/components/public_model_hub.test.tsx b/ui/litellm-dashboard/src/components/public_model_hub.test.tsx index 875f89b5adc..fec46e98077 100644 --- a/ui/litellm-dashboard/src/components/public_model_hub.test.tsx +++ b/ui/litellm-dashboard/src/components/public_model_hub.test.tsx @@ -134,6 +134,27 @@ describe("PublicModelHub", () => { expect(within(gpt35Row as HTMLElement).getByText("Unknown")).toBeInTheDocument(); }); }); + it("shows no models when the search has no matches (LIT-5230 regression)", async () => { + const networkingModule = await import("./networking"); + vi.mocked(networkingModule.modelHubPublicModelsCall).mockResolvedValue([ + { model_group: "gpt-4", providers: ["openai"], mode: "chat" }, + { model_group: "claude-3", providers: ["anthropic"], mode: "chat" }, + ]); + + render(); + expect(await screen.findByText("gpt-4")).toBeInTheDocument(); + + fireEvent.change(screen.getByPlaceholderText("Search model names... (smart search enabled)"), { + target: { value: "zzzz" }, + }); + + await waitFor(() => { + expect(screen.queryByText("gpt-4")).not.toBeInTheDocument(); + expect(screen.queryByText("claude-3")).not.toBeInTheDocument(); + expect(screen.getByText("No matching models")).toBeInTheDocument(); + }); + }); + it("handles non-array response gracefully (regression test for e.filter crash)", async () => { const networkingModule = await import("./networking"); // Mock the API to return an object (like an error response) instead of an array diff --git a/ui/litellm-dashboard/src/components/public_model_hub.tsx b/ui/litellm-dashboard/src/components/public_model_hub.tsx index 171b7992325..f6364b5d9d1 100644 --- a/ui/litellm-dashboard/src/components/public_model_hub.tsx +++ b/ui/litellm-dashboard/src/components/public_model_hub.tsx @@ -47,6 +47,7 @@ import { generateCodeSnippet } from "@/components/chat_ui/CodeSnippets"; import { getEndpointType } from "@/components/chat_ui/mode_endpoint_mapping"; import { MessageType } from "@/components/chat_ui/types"; import { getProviderLogoAndName } from "./provider_info_helpers"; +import { filterBySearchTerm, rankBySearchRelevance } from "@/utils/searchUtils"; interface PublicModelHubProps { accessToken?: string | null; @@ -236,52 +237,11 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded const filteredData = useMemo(() => { if (!modelHubData || !Array.isArray(modelHubData)) return []; - let searchResults = modelHubData; - - // Apply search if there's a search term - if (searchTerm.trim()) { - const lowercaseSearch = searchTerm.toLowerCase(); - const searchWords = lowercaseSearch.split(/\s+/); - - // First, try flexible matching that handles different separators - const exactMatches = modelHubData.filter((model) => { - const modelName = model.model_group.toLowerCase(); - - // Check if it contains the exact search term - if (modelName.includes(lowercaseSearch)) { - return true; - } - - // Check if it contains all search words (handles spaces vs slashes/dashes) - return searchWords.every((word) => modelName.includes(word)); - }); - - // If we have exact matches, rank them by relevance - if (exactMatches.length > 0) { - searchResults = exactMatches.sort((a, b) => { - const aName = a.model_group.toLowerCase(); - const bName = b.model_group.toLowerCase(); - - // Calculate relevance scores - const aExactMatch = aName === lowercaseSearch ? 1000 : 0; - const bExactMatch = bName === lowercaseSearch ? 1000 : 0; - - const aStartsWith = aName.startsWith(lowercaseSearch) ? 100 : 0; - const bStartsWith = bName.startsWith(lowercaseSearch) ? 100 : 0; - - const aContainsWords = lowercaseSearch.split(/\s+/).every((word) => aName.includes(word)) ? 50 : 0; - const bContainsWords = lowercaseSearch.split(/\s+/).every((word) => bName.includes(word)) ? 50 : 0; - - const aLength = aName.length; - const bLength = bName.length; - - const aScore = aExactMatch + aStartsWith + aContainsWords + (1000 - aLength); - const bScore = bExactMatch + bStartsWith + bContainsWords + (1000 - bLength); - - return bScore - aScore; // Higher score first - }); - } - } + const searchResults = rankBySearchRelevance( + filterBySearchTerm(modelHubData, searchTerm, (model) => [model.model_group]), + searchTerm, + (model) => model.model_group, + ); // Apply other filters return searchResults.filter((model) => { @@ -310,43 +270,11 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded const filteredAgentData = useMemo(() => { if (!agentHubData || !Array.isArray(agentHubData)) return []; - let searchResults = agentHubData; - - // Apply search if there's a search term - if (agentSearchTerm.trim()) { - const lowercaseSearch = agentSearchTerm.toLowerCase(); - const searchWords = lowercaseSearch.split(/\s+/); - - searchResults = agentHubData.filter((agent) => { - const agentName = agent.name.toLowerCase(); - const agentDescription = agent.description.toLowerCase(); - - // Check if it contains the exact search term - if (agentName.includes(lowercaseSearch) || agentDescription.includes(lowercaseSearch)) { - return true; - } - - // Check if it contains all search words - return searchWords.every((word) => agentName.includes(word) || agentDescription.includes(word)); - }); - - // Sort by relevance - searchResults = searchResults.sort((a, b) => { - const aName = a.name.toLowerCase(); - const bName = b.name.toLowerCase(); - - const aExactMatch = aName === lowercaseSearch ? 1000 : 0; - const bExactMatch = bName === lowercaseSearch ? 1000 : 0; - - const aStartsWith = aName.startsWith(lowercaseSearch) ? 100 : 0; - const bStartsWith = bName.startsWith(lowercaseSearch) ? 100 : 0; - - const aScore = aExactMatch + aStartsWith + (1000 - aName.length); - const bScore = bExactMatch + bStartsWith + (1000 - bName.length); - - return bScore - aScore; - }); - } + const searchResults = rankBySearchRelevance( + filterBySearchTerm(agentHubData, agentSearchTerm, (agent) => [agent.name, agent.description]), + agentSearchTerm, + (agent) => agent.name, + ); // Apply skill filters return searchResults.filter((agent) => { @@ -361,43 +289,11 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded const filteredMcpData = useMemo(() => { if (!mcpHubData || !Array.isArray(mcpHubData)) return []; - let searchResults = mcpHubData; - - // Apply search if there's a search term - if (mcpSearchTerm.trim()) { - const lowercaseSearch = mcpSearchTerm.toLowerCase(); - const searchWords = lowercaseSearch.split(/\s+/); - - searchResults = mcpHubData.filter((server) => { - const serverName = server.server_name.toLowerCase(); - const serverDescription = (server.mcp_info?.description || "").toLowerCase(); - - // Check if it contains the exact search term - if (serverName.includes(lowercaseSearch) || serverDescription.includes(lowercaseSearch)) { - return true; - } - - // Check if it contains all search words - return searchWords.every((word) => serverName.includes(word) || serverDescription.includes(word)); - }); - - // Sort by relevance - searchResults = searchResults.sort((a, b) => { - const aName = a.server_name.toLowerCase(); - const bName = b.server_name.toLowerCase(); - - const aExactMatch = aName === lowercaseSearch ? 1000 : 0; - const bExactMatch = bName === lowercaseSearch ? 1000 : 0; - - const aStartsWith = aName.startsWith(lowercaseSearch) ? 100 : 0; - const bStartsWith = bName.startsWith(lowercaseSearch) ? 100 : 0; - - const aScore = aExactMatch + aStartsWith + (1000 - aName.length); - const bScore = bExactMatch + bStartsWith + (1000 - bName.length); - - return bScore - aScore; - }); - } + const searchResults = rankBySearchRelevance( + filterBySearchTerm(mcpHubData, mcpSearchTerm, (server) => [server.server_name, server.mcp_info?.description]), + mcpSearchTerm, + (server) => server.server_name, + ); // Apply transport filters return searchResults.filter((server) => { diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx index 7ead9bb64d4..f502ea77127 100644 --- a/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx @@ -7,6 +7,7 @@ import { describe, expect, it, vi } from "vitest"; import { DataTable } from "./DataTable"; import { DataTableMultiSortHeader, DataTableSortHeader } from "./DataTableSortHeader"; import { DataTableViewOptions } from "./DataTableViewOptions"; +import { chooseSelectOption } from "../../../../tests/test-utils"; interface Person { id: string; @@ -176,16 +177,13 @@ describe("DataTable sorting", () => { const user = userEvent.setup(); render(); - await user.click(screen.getByTestId("sort-trigger-name")); - await user.click(await screen.findByText("Ascending")); + await chooseSelectOption(user, screen.getByTestId("sort-trigger-name"), "Ascending", "menuitem"); expect(names()).toEqual(["Alice", "Bob", "Charlie"]); - await user.click(screen.getByTestId("sort-trigger-name")); - await user.click(await screen.findByText("Descending")); + await chooseSelectOption(user, screen.getByTestId("sort-trigger-name"), "Descending", "menuitem"); expect(names()).toEqual(["Charlie", "Bob", "Alice"]); - await user.click(screen.getByTestId("sort-trigger-name")); - await user.click(await screen.findByText("Reset")); + await chooseSelectOption(user, screen.getByTestId("sort-trigger-name"), "Reset", "menuitem"); expect(names()).toEqual(["Charlie", "Alice", "Bob"]); }); @@ -202,12 +200,10 @@ describe("DataTable sorting", () => { />, ); - await user.click(screen.getByTestId("sort-trigger-spend")); - await user.click(await screen.findByText("Budget descending")); + await chooseSelectOption(user, screen.getByTestId("sort-trigger-spend"), "Budget descending", "menuitem"); expect(onSortingChange).toHaveBeenLastCalledWith([{ id: "max_budget", desc: true }]); - await user.click(screen.getByTestId("sort-trigger-spend")); - await user.click(await screen.findByText("Spend ascending")); + await chooseSelectOption(user, screen.getByTestId("sort-trigger-spend"), "Spend ascending", "menuitem"); expect(onSortingChange).toHaveBeenLastCalledWith([{ id: "spend", desc: false }]); }); diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTableSortHeader.test.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTableSortHeader.test.tsx index 70403ba2efe..aeb6fa44e91 100644 --- a/ui/litellm-dashboard/src/components/shared/DataTable/DataTableSortHeader.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTableSortHeader.test.tsx @@ -13,6 +13,7 @@ import { useState } from "react"; import { describe, expect, it, vi } from "vitest"; import { DataTableSortHeader, type DataTableSortVariant } from "./DataTableSortHeader"; +import { chooseSelectOption } from "../../../../tests/test-utils"; interface Item { name: string; @@ -84,16 +85,13 @@ describe("DataTableSortHeader", () => { const user = userEvent.setup(); render(); - await user.click(screen.getByTestId("sort-trigger-name")); - await user.click(await screen.findByText("Descending")); + await chooseSelectOption(user, screen.getByTestId("sort-trigger-name"), "Descending", "menuitem"); expect(screen.getByTestId("sort-trigger-name").querySelector('[data-sort-indicator="desc"]')).not.toBeNull(); - await user.click(screen.getByTestId("sort-trigger-name")); - await user.click(await screen.findByText("Ascending")); + await chooseSelectOption(user, screen.getByTestId("sort-trigger-name"), "Ascending", "menuitem"); expect(screen.getByTestId("sort-trigger-name").querySelector('[data-sort-indicator="asc"]')).not.toBeNull(); - await user.click(screen.getByTestId("sort-trigger-name")); - await user.click(await screen.findByText("Reset")); + await chooseSelectOption(user, screen.getByTestId("sort-trigger-name"), "Reset", "menuitem"); expect(screen.getByTestId("sort-trigger-name").querySelector('[data-sort-indicator="none"]')).not.toBeNull(); }); diff --git a/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.test.tsx b/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.test.tsx index d248cf311cc..04c8d3e9012 100644 --- a/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.test.tsx @@ -5,6 +5,7 @@ import { describe, expect, it, vi } from "vitest"; import { PaginatedSearchSelect } from "./PaginatedSearchSelect"; import type { SearchSelectOption } from "./SearchSelect"; +import { chooseSelectOption } from "../../../tests/test-utils"; const OPTIONS: SearchSelectOption[] = [ { label: "alias-alpha", value: "alias-alpha" }, @@ -63,8 +64,7 @@ describe("PaginatedSearchSelect", () => { } render(); - await user.click(screen.getByRole("combobox")); - await user.click(await screen.findByText("alias-beta")); + await chooseSelectOption(user, screen.getByRole("combobox"), "alias-beta"); expect(screen.getByRole("combobox")).toHaveValue("alias-beta"); await new Promise((resolve) => setTimeout(resolve, 400)); @@ -136,8 +136,7 @@ describe("PaginatedSearchSelect", () => { const onValueChange = vi.fn(); renderSelect({ onValueChange }); - await user.click(screen.getByRole("combobox")); - await user.click(await screen.findByText("alias-beta")); + await chooseSelectOption(user, screen.getByRole("combobox"), "alias-beta"); expect(onValueChange).toHaveBeenCalledWith("alias-beta"); }); @@ -255,8 +254,7 @@ describe("PaginatedSearchSelect", () => { } render(); - await user.click(screen.getByRole("combobox")); - await user.click(await screen.findByText("Beta Team")); + await chooseSelectOption(user, screen.getByRole("combobox"), "Beta Team"); await user.click(screen.getByRole("button", { name: "refetch" })); expect(screen.getByRole("combobox")).toHaveValue("Beta Team"); diff --git a/ui/litellm-dashboard/src/components/shared/SearchSelect.test.tsx b/ui/litellm-dashboard/src/components/shared/SearchSelect.test.tsx index c0b9bc7d8ae..62a510268dd 100644 --- a/ui/litellm-dashboard/src/components/shared/SearchSelect.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/SearchSelect.test.tsx @@ -3,6 +3,7 @@ import userEvent from "@testing-library/user-event"; import { describe, expect, it, vi } from "vitest"; import { SearchSelect } from "./SearchSelect"; +import { chooseSelectOption } from "../../../tests/test-utils"; const OPTIONS = [ { label: "Acme Prod", value: "team-1" }, @@ -71,8 +72,7 @@ describe("SearchSelect", () => { const onValueChange = vi.fn(); const user = userEvent.setup(); render(); - await user.click(screen.getByRole("combobox")); - await user.click(await screen.findByText("Growth")); + await chooseSelectOption(user, screen.getByRole("combobox"), "Growth"); expect(onValueChange).toHaveBeenCalledWith("team-2"); }); diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx index e0ec66e1ae2..5a35c7ae16b 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx @@ -3,7 +3,7 @@ import userEvent from "@testing-library/user-event"; import { useState } from "react"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import { renderWithProviders, testQueryClient } from "../../../tests/test-utils"; +import { chooseSelectOption, renderWithProviders, testQueryClient } from "../../../tests/test-utils"; import { ERROR_CODE_OPTIONS } from "./constants"; import { LOG_FILTER_IDS } from "./log_filter_logic"; import { RequestLogsFilters } from "./RequestLogsFilters"; @@ -125,8 +125,7 @@ describe("RequestLogsFilters", () => { const user = userEvent.setup(); const { set } = renderFilters(); - await user.click(await screen.findByPlaceholderText("Search an internal user")); - await user.click(await screen.findByText("alice@example.com")); + await chooseSelectOption(user, await screen.findByPlaceholderText("Search an internal user"), "alice@example.com"); expect(set).toHaveBeenCalledWith(LOG_FILTER_IDS.USER_ID, "alice@example.com"); }); diff --git a/ui/litellm-dashboard/src/utils/searchUtils.test.ts b/ui/litellm-dashboard/src/utils/searchUtils.test.ts new file mode 100644 index 00000000000..4935e8ab15d --- /dev/null +++ b/ui/litellm-dashboard/src/utils/searchUtils.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest"; + +import { filterBySearchTerm, matchesSearchTerm, rankBySearchRelevance } from "./searchUtils"; + +describe("matchesSearchTerm", () => { + it("matches everything on an empty or whitespace-only term", () => { + expect(matchesSearchTerm("", ["anything"])).toBe(true); + expect(matchesSearchTerm(" ", ["anything"])).toBe(true); + }); + + it("matches a substring of any field, case-insensitively", () => { + expect(matchesSearchTerm("BILL", ["Billing Router", "routes invoices"])).toBe(true); + expect(matchesSearchTerm("invoice", ["Billing Router", "routes invoices"])).toBe(true); + }); + + it("matches when every word appears in some field", () => { + expect(matchesSearchTerm("router invoices", ["Billing Router", "routes invoices"])).toBe(true); + expect(matchesSearchTerm("router refunds", ["Billing Router", "routes invoices"])).toBe(false); + }); + + it("returns false when nothing matches", () => { + expect(matchesSearchTerm("zzzz", ["Billing Router", "routes invoices"])).toBe(false); + }); + + it("ignores null and undefined fields", () => { + expect(matchesSearchTerm("billing", [null, undefined, "Billing Router"])).toBe(true); + expect(matchesSearchTerm("billing", [null, undefined])).toBe(false); + }); +}); + +describe("filterBySearchTerm", () => { + const agents = [ + { name: "Billing Router", description: "routes invoices" }, + { name: "Support Bot", description: "handles tickets" }, + ]; + + it("keeps only items whose fields match", () => { + expect(filterBySearchTerm(agents, "tickets", (a) => [a.name, a.description])).toEqual([agents[1]]); + }); + + it("returns an empty list when nothing matches", () => { + expect(filterBySearchTerm(agents, "zzzz", (a) => [a.name, a.description])).toEqual([]); + }); + + it("returns all items for an empty term", () => { + expect(filterBySearchTerm(agents, "", (a) => [a.name, a.description])).toEqual(agents); + }); +}); + +describe("rankBySearchRelevance", () => { + it("orders exact match, then prefix match, then shorter names", () => { + const items = [{ name: "gpt-4o-mini-transcribe" }, { name: "gpt-4o" }, { name: "chatgpt-4o-latest" }]; + expect(rankBySearchRelevance(items, "gpt-4o", (m) => m.name).map((m) => m.name)).toEqual([ + "gpt-4o", + "gpt-4o-mini-transcribe", + "chatgpt-4o-latest", + ]); + }); + + it("keeps the original order for an empty term", () => { + const items = [{ name: "b" }, { name: "a" }]; + expect(rankBySearchRelevance(items, "", (m) => m.name)).toEqual(items); + }); +}); diff --git a/ui/litellm-dashboard/src/utils/searchUtils.ts b/ui/litellm-dashboard/src/utils/searchUtils.ts new file mode 100644 index 00000000000..b256a5db0c4 --- /dev/null +++ b/ui/litellm-dashboard/src/utils/searchUtils.ts @@ -0,0 +1,32 @@ +type SearchField = string | null | undefined; + +const normalizeTerm = (term: string): string => term.trim().toLowerCase(); + +export function matchesSearchTerm(term: string, fields: ReadonlyArray): boolean { + const needle = normalizeTerm(term); + if (needle === "") return true; + + const haystacks = fields.filter((field): field is string => typeof field === "string").map((f) => f.toLowerCase()); + if (haystacks.some((haystack) => haystack.includes(needle))) return true; + + return needle.split(/\s+/).every((word) => haystacks.some((haystack) => haystack.includes(word))); +} + +export function filterBySearchTerm( + items: ReadonlyArray, + term: string, + fields: (item: T) => ReadonlyArray, +): T[] { + return items.filter((item) => matchesSearchTerm(term, fields(item))); +} + +export function rankBySearchRelevance(items: ReadonlyArray, term: string, name: (item: T) => string): T[] { + const needle = normalizeTerm(term); + if (needle === "") return [...items]; + + const score = (item: T): number => { + const candidate = name(item).toLowerCase(); + return (candidate === needle ? 1000 : 0) + (candidate.startsWith(needle) ? 100 : 0) + (1000 - candidate.length); + }; + return [...items].sort((a, b) => score(b) - score(a)); +} diff --git a/ui/litellm-dashboard/tests/test-utils.tsx b/ui/litellm-dashboard/tests/test-utils.tsx index 162b8a3df7b..553726faff0 100644 --- a/ui/litellm-dashboard/tests/test-utils.tsx +++ b/ui/litellm-dashboard/tests/test-utils.tsx @@ -44,20 +44,14 @@ const pointerBlocked = (element: HTMLElement): boolean => { return false; }; -/** - * Opens a Base UI Select and picks an option by its accessible name. - * - * The option is in the DOM one render before the popup finishes entering, and until then its - * positioner still carries `pointer-events: none`, which user-event refuses to click. Waiting on - * the option text alone is a race that React 19's flush timing loses. - */ export const chooseSelectOption = async ( user: Pick, "click">, trigger: HTMLElement, optionName: string | RegExp, + role: "option" | "menuitem" | "menuitemradio" = "option", ) => { await user.click(trigger); - const option = await screen.findByRole("option", { name: optionName }); + const option = await screen.findByRole(role, { name: optionName }); await waitFor(() => expect(pointerBlocked(option)).toBe(false)); await user.click(option); };