From ee7203281b5dceb3158f057299ce7e56bfaba761 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2026 12:39:57 +0000 Subject: [PATCH 01/29] fix(ptu): take the router as an argument instead of the proxy module global The rollup read litellm.proxy.proxy_server.llm_router out of sys.modules, so a run priced and swept whatever deployments anything else in the process had left on that module. Under xdist the shard's module-to-worker assignment varies per run, which made three rollup tests fail or pass on the same commit depending on ordering. Callers now hand the router in, and the proxy's scheduled job passes its own. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/proxy_server.py | 1 + .../spend_tracking/ptu_flat_cost_rollup.py | 40 ++-- .../test_ptu_flat_cost_rollup.py | 206 +++++++++--------- tests/test_litellm/proxy/test_proxy_server.py | 29 +++ 4 files changed, 147 insertions(+), 129 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 9ee62f94647..2e57d3e0708 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -9089,6 +9089,7 @@ class ProxyStartupEvent: prisma_client, pod_lock_manager=proxy_logging_obj.db_spend_update_writer.pod_lock_manager, alert=_alert_ptu_rollup_failure, + router=llm_router, ) scheduler.add_job( diff --git a/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py b/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py index f1f7248c064..a4eac992d89 100644 --- a/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py +++ b/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py @@ -14,7 +14,6 @@ and share the existing unique constraint. import asyncio import json -import sys from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass from datetime import date, datetime, time, timedelta, timezone @@ -327,16 +326,6 @@ class _LoadedDeployments: config_sourced: bool -def _running_router() -> object | None: - """The proxy's router, or None outside a running proxy. - - Read out of ``sys.modules`` rather than imported, so a rollup driven from a test or a - script does not pull the whole proxy server in behind it. - """ - proxy_server: Final = sys.modules.get("litellm.proxy.proxy_server") - return getattr(proxy_server, "llm_router", None) if proxy_server is not None else None - - def _config_deployments(router: object | None, *, owned_by_db: frozenset[str]) -> tuple[_PTUDeployment, ...]: """Deployments the router holds that no ``LiteLLM_ProxyModelTable`` row owns. @@ -357,15 +346,17 @@ def _config_deployments(router: object | None, *, owned_by_db: frozenset[str]) - ) -async def _load_ptu_models(prisma_client: "PrismaClient") -> _LoadedDeployments: +async def _load_ptu_models(prisma_client: "PrismaClient", *, router: object | None) -> _LoadedDeployments: """Every deployment carrying valid manual PTU config, and every id the scan saw. Reserved capacity is billed by the provider whichever file declared it, so a deployment the proxy only knows from config.yaml accrues alongside the stored ones. + The router is handed in rather than read off the proxy module, so a run prices exactly + the deployments its caller declares and nothing a co-resident process left behind. """ rows: Final = await prisma_client.db.litellm_proxymodeltable.find_many() db_ids: Final = frozenset(model_id for row in rows if (model_id := str(getattr(row, "model_id", "") or ""))) - config_records: Final = _config_deployments(_running_router(), owned_by_db=db_ids) + config_records: Final = _config_deployments(router, owned_by_db=db_ids) models: Final = tuple( parsed for parsed in (_parse_ptu_model(row) for row in (*rows, *config_records)) if parsed is not None ) @@ -382,6 +373,7 @@ async def run_ptu_flat_cost_rollup( prisma_client: "PrismaClient", target_date: date | None = None, may_prune: bool = True, + router: object | None = None, ) -> RollupResult: """Rollup one UTC day of flat PTU cost across all PTU-configured model deployments. @@ -406,7 +398,7 @@ async def run_ptu_flat_cost_rollup( date_str: Final = day.isoformat() run_started: Final = datetime.now(timezone.utc) - loaded: Final = await _load_ptu_models(prisma_client) + loaded: Final = await _load_ptu_models(prisma_client, router=router) ptu_models: Final = loaded.models charges: Final = _aggregate_charges(ptu_models, day) @@ -527,6 +519,7 @@ async def _existing_sentinel_keys( async def run_ptu_flat_cost_backfill( prisma_client: "PrismaClient", today: date | None = None, + router: object | None = None, ) -> BackfillResult: """Price the elapsed days of every PTU window that carry no sentinel row yet. @@ -546,7 +539,7 @@ async def run_ptu_flat_cost_backfill( verbose_proxy_logger.warning("PTU backfill: prisma_client is None, skipping") return BackfillResult(start=end, end=end, days_scanned=0, rows_written=0) - ptu_models: Final = (await _load_ptu_models(prisma_client)).models + ptu_models: Final = (await _load_ptu_models(prisma_client, router=router)).models days: Final = _backfill_window(ptu_models, end) if not days: @@ -591,6 +584,7 @@ async def run_scheduled_ptu_rollup( pod_lock_manager: "PodLockManager | None" = None, target_date: date | None = None, alert: Callable[[str], Awaitable[None]] | None = None, + router: object | None = None, ) -> RollupResult | None: """Run the daily rollup under a cross-pod lock so only one proxy reconciles a day. @@ -615,7 +609,7 @@ async def run_scheduled_ptu_rollup( return None if pod_lock_manager is None or pod_lock_manager.redis_cache is None: - return await _run_and_alert(prisma_client, target_date=target_date, alert=alert, may_prune=False) + return await _run_and_alert(prisma_client, target_date=target_date, alert=alert, may_prune=False, router=router) if not await pod_lock_manager.acquire_lock(cronjob_id=PTU_ROLLUP_JOB_ID, ttl=PTU_ROLLUP_LOCK_TTL_SECONDS): if await _lock_is_held(pod_lock_manager): @@ -629,10 +623,10 @@ async def run_scheduled_ptu_rollup( "PTU rollup: could not take the rollup lock and no other pod holds it, " "running unguarded rather than skipping the day" ) - return await _run_and_alert(prisma_client, target_date=target_date, alert=alert, may_prune=False) + return await _run_and_alert(prisma_client, target_date=target_date, alert=alert, may_prune=False, router=router) try: - return await _run_and_alert(prisma_client, target_date=target_date, alert=alert, may_prune=True) + return await _run_and_alert(prisma_client, target_date=target_date, alert=alert, may_prune=True, router=router) finally: await pod_lock_manager.release_lock(cronjob_id=PTU_ROLLUP_JOB_ID) @@ -657,6 +651,7 @@ async def _run_and_alert( target_date: date | None, alert: "Callable[[str], Awaitable[None]] | None", may_prune: bool = True, + router: object | None = None, ) -> RollupResult: """Reconcile the day, catch up any days left unpriced, and alert on charges that did not land. @@ -669,7 +664,9 @@ async def _run_and_alert( explicit date means reconcile exactly that day, so it stays a single-day operation. Its failure is contained: the day's own result is returned either way. """ - result: Final = await run_ptu_flat_cost_rollup(prisma_client, target_date=target_date, may_prune=may_prune) + result: Final = await run_ptu_flat_cost_rollup( + prisma_client, target_date=target_date, may_prune=may_prune, router=router + ) if result.rows_failed: await _deliver_alert( alert, @@ -686,7 +683,7 @@ async def _run_and_alert( "by the provider with nothing attributing it here. Extend the window, or retire the deployment.", ) if target_date is None: - await _backfill_and_alert(prisma_client, alert=alert) + await _backfill_and_alert(prisma_client, alert=alert, router=router) return result @@ -694,6 +691,7 @@ async def _backfill_and_alert( prisma_client: "PrismaClient", *, alert: "Callable[[str], Awaitable[None]] | None", + router: object | None = None, ) -> None: """Catch up unpriced PTU days, alerting on charges that did not land. @@ -701,7 +699,7 @@ async def _backfill_and_alert( caller whatever the catch-up pass does. """ try: - backfill: Final = await run_ptu_flat_cost_backfill(prisma_client) + backfill: Final = await run_ptu_flat_cost_backfill(prisma_client, router=router) except Exception as exc: # noqa: BLE001 # the catch-up pass must not fail the day's rollup verbose_proxy_logger.error("PTU backfill: catch-up pass failed, the day's rollup still stands: %s", exc) return diff --git a/tests/test_litellm/proxy/spend_tracking/test_ptu_flat_cost_rollup.py b/tests/test_litellm/proxy/spend_tracking/test_ptu_flat_cost_rollup.py index e039455607d..17a487ddd4b 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_ptu_flat_cost_rollup.py +++ b/tests/test_litellm/proxy/spend_tracking/test_ptu_flat_cost_rollup.py @@ -1767,16 +1767,20 @@ async def test_the_prune_cutoff_allows_for_clock_skew_between_hosts(): @pytest.mark.asyncio -async def test_a_run_pricing_config_cannot_prune_a_row_it_did_not_scan(monkeypatch): +async def test_a_run_pricing_config_cannot_prune_a_row_it_did_not_scan(): """Staleness alone stops being evidence once two hosts hold different configuration: a row this run never considered belongs to a deployment another host is pricing from its own file, and sweeping it drops that charge.""" table = _FakeSentinelTable() table.seed("t", DAY, "dep-elsewhere", 480.0, updated_at=datetime(2020, 1, 1, tzinfo=timezone.utc)) entry = _router_entry(model_id="cfg-here", model_info=dict(_VALID_PTU)) - monkeypatch.setattr(ptu_rollup, "_running_router", lambda: _router_holding(entry)) - await run_scheduled_ptu_rollup(_prisma_for([], table), pod_lock_manager=_pod_lock(acquired=True), target_date=DAY) + await run_scheduled_ptu_rollup( + _prisma_for([], table), + pod_lock_manager=_pod_lock(acquired=True), + target_date=DAY, + router=_router_holding(entry), + ) assert ("t", DAY.isoformat(), PTU_SENTINEL_API_KEY, "dep-elsewhere") in table.rows assert ("t", DAY.isoformat(), PTU_SENTINEL_API_KEY, "cfg-here") in table.rows @@ -1784,7 +1788,7 @@ async def test_a_run_pricing_config_cannot_prune_a_row_it_did_not_scan(monkeypat @pytest.mark.asyncio -async def test_a_deployment_deleted_from_the_table_keeps_the_day_it_was_charged(monkeypatch): +async def test_a_deployment_deleted_from_the_table_keeps_the_day_it_was_charged(): """The accepted cost of bounding the prune, driven through the sequence that produces it: charge the day while the deployment exists, remove it, run the day again. Nothing scans it now, so nothing may judge its row, and the amount it was billed stands.""" @@ -1793,18 +1797,19 @@ async def test_a_deployment_deleted_from_the_table_keeps_the_day_it_was_charged( live_row = _model_row(model_id="dep-live", model_info=ptu) doomed_row = _model_row(model_id="dep-doomed", model_info=ptu) charged_key = ("t", DAY.isoformat(), PTU_SENTINEL_API_KEY, "dep-doomed") - monkeypatch.setattr( - ptu_rollup, "_running_router", lambda: _router_holding(_router_entry(model_id="cfg", model_info=dict(ptu))) - ) + router = _router_holding(_router_entry(model_id="cfg", model_info=dict(ptu))) await run_scheduled_ptu_rollup( - _prisma_for([live_row, doomed_row], table), pod_lock_manager=_pod_lock(acquired=True), target_date=DAY + _prisma_for([live_row, doomed_row], table), + pod_lock_manager=_pod_lock(acquired=True), + target_date=DAY, + router=router, ) billed = table.rows[charged_key]["ptu_flat_cost"] table.rows[charged_key]["updated_at"] = datetime(2020, 1, 1, tzinfo=timezone.utc) await run_scheduled_ptu_rollup( - _prisma_for([live_row], table), pod_lock_manager=_pod_lock(acquired=True), target_date=DAY + _prisma_for([live_row], table), pod_lock_manager=_pod_lock(acquired=True), target_date=DAY, router=router ) assert table.rows[charged_key]["ptu_flat_cost"] == billed @@ -1842,7 +1847,7 @@ async def test_every_deployment_that_prices_is_inside_the_set_that_bounds_the_pr table, ) - loaded = await ptu_rollup._load_ptu_models(prisma) + loaded = await ptu_rollup._load_ptu_models(prisma, router=None) assert {model.model_id for model in loaded.models} <= loaded.scanned_ids assert loaded.scanned_ids == {"dep-a", "dep-b", "dep-unpriced"} @@ -1858,7 +1863,7 @@ async def test_a_priced_deployment_is_in_the_bound_even_with_an_id_the_scan_skip _FakeSentinelTable(), ) - loaded = await ptu_rollup._load_ptu_models(prisma) + loaded = await ptu_rollup._load_ptu_models(prisma, router=None) assert {model.model_id for model in loaded.models} <= loaded.scanned_ids @@ -1872,13 +1877,13 @@ async def test_the_prune_splits_the_id_set_across_statements(monkeypatch): table = _FakeSentinelTable() ptu = {"ptu_count": 5, "cost_per_ptu_per_hour": 2.0, "team_id": "t"} deployments = [_model_row(model_id=f"dep-{n}", model_info=ptu) for n in range(4)] - monkeypatch.setattr( - ptu_rollup, "_running_router", lambda: _router_holding(_router_entry(model_id="dep-4", model_info=dict(ptu))) - ) table.seed("t", DAY, "dep-3", 480.0, updated_at=datetime(2020, 1, 1, tzinfo=timezone.utc)) await run_scheduled_ptu_rollup( - _prisma_for(deployments, table), pod_lock_manager=_pod_lock(acquired=True), target_date=DAY + _prisma_for(deployments, table), + pod_lock_manager=_pod_lock(acquired=True), + target_date=DAY, + router=_router_holding(_router_entry(model_id="dep-4", model_info=dict(ptu))), ) chunks = [call["model"]["in"] for call in table.delete_many_calls] @@ -1911,140 +1916,124 @@ def _router_holding(*entries): @pytest.mark.asyncio -async def test_a_config_declared_deployment_is_priced(monkeypatch): +async def test_a_config_declared_deployment_is_priced(): """The whole point. A PTU deployment the proxy only knows from config.yaml is not in LiteLLM_ProxyModelTable, so a DB-only scan bills the provider's reservation to nobody.""" entry = _router_entry(model_id="cfg-1", model_name="gpt-4o-ptu", model_info=dict(_VALID_PTU)) - monkeypatch.setattr(ptu_rollup, "_running_router", lambda: _router_holding(entry)) - loaded = await ptu_rollup._load_ptu_models(_prisma_for([], _FakeSentinelTable())) + loaded = await ptu_rollup._load_ptu_models(_prisma_for([], _FakeSentinelTable()), router=_router_holding(entry)) assert [(m.model_id, m.model_name, m.team_id) for m in loaded.models] == [("cfg-1", "gpt-4o-ptu", "t")] assert "cfg-1" in loaded.scanned_ids @pytest.mark.asyncio -async def test_a_database_backed_router_entry_is_not_counted_twice(monkeypatch): +async def test_a_database_backed_router_entry_is_not_counted_twice(): """Every deployment loaded from the table is also in the router, flagged db_model. Pricing both copies would write two charges for one reservation.""" row = _model_row(model_id="db-1", model_info=dict(_VALID_PTU)) mirrored = _router_entry(model_id="db-1", model_info={**_VALID_PTU, "db_model": True}) - monkeypatch.setattr(ptu_rollup, "_running_router", lambda: _router_holding(mirrored)) - - loaded = await ptu_rollup._load_ptu_models(_prisma_for([row], _FakeSentinelTable())) - - assert [m.model_id for m in loaded.models] == ["db-1"] - - -@pytest.mark.asyncio -async def test_a_router_entry_sharing_an_id_with_the_table_is_priced_once(monkeypatch): - """db_model is data the router carries rather than something this module controls, so the - id anti-join is what actually maps onto the failure: two charges under one id.""" - row = _model_row(model_id="both-1", model_info=dict(_VALID_PTU)) - unflagged = _router_entry(model_id="both-1", model_info=dict(_VALID_PTU)) - monkeypatch.setattr(ptu_rollup, "_running_router", lambda: _router_holding(unflagged)) - - loaded = await ptu_rollup._load_ptu_models(_prisma_for([row], _FakeSentinelTable())) - - assert [m.model_id for m in loaded.models] == ["both-1"] - - -@pytest.mark.asyncio -async def test_a_client_credential_clone_is_not_priced(monkeypatch): - """Supplying an api_key on a request mints a clone of the deployment under a fresh id, - carrying the source's PTU config. Pricing it bills one reservation per distinct caller key.""" - source = _router_entry(model_id="cfg-1", model_info=dict(_VALID_PTU)) - clone = _router_entry(model_id="cfg-1-clone", model_info={**_VALID_PTU, "original_model_id": "cfg-1"}) - monkeypatch.setattr(ptu_rollup, "_running_router", lambda: _router_holding(source, clone)) - - loaded = await ptu_rollup._load_ptu_models(_prisma_for([], _FakeSentinelTable())) - - assert [m.model_id for m in loaded.models] == ["cfg-1"] - - -@pytest.mark.asyncio -async def test_a_config_deployment_without_ptu_config_is_scanned_but_not_priced(monkeypatch): - """It has to stay in the scanned set or its leftover sentinel rows become unprunable.""" - entry = _router_entry(model_id="cfg-plain", model_info={"team_id": "t"}) - monkeypatch.setattr(ptu_rollup, "_running_router", lambda: _router_holding(entry)) - - loaded = await ptu_rollup._load_ptu_models(_prisma_for([], _FakeSentinelTable())) - - assert loaded.models == () - assert "cfg-plain" in loaded.scanned_ids - - -@pytest.mark.asyncio -async def test_no_router_in_the_process_prices_the_database_alone(monkeypatch): - """The rollup is importable and callable outside a running proxy.""" - monkeypatch.setattr(ptu_rollup, "_running_router", lambda: None) loaded = await ptu_rollup._load_ptu_models( - _prisma_for([_model_row(model_id="db-1", model_info=dict(_VALID_PTU))], _FakeSentinelTable()) + _prisma_for([row], _FakeSentinelTable()), router=_router_holding(mirrored) ) assert [m.model_id for m in loaded.models] == ["db-1"] @pytest.mark.asyncio -async def test_a_config_deployment_is_charged_end_to_end(monkeypatch): +async def test_a_router_entry_sharing_an_id_with_the_table_is_priced_once(): + """db_model is data the router carries rather than something this module controls, so the + id anti-join is what actually maps onto the failure: two charges under one id.""" + row = _model_row(model_id="both-1", model_info=dict(_VALID_PTU)) + unflagged = _router_entry(model_id="both-1", model_info=dict(_VALID_PTU)) + + loaded = await ptu_rollup._load_ptu_models( + _prisma_for([row], _FakeSentinelTable()), router=_router_holding(unflagged) + ) + + assert [m.model_id for m in loaded.models] == ["both-1"] + + +@pytest.mark.asyncio +async def test_a_client_credential_clone_is_not_priced(): + """Supplying an api_key on a request mints a clone of the deployment under a fresh id, + carrying the source's PTU config. Pricing it bills one reservation per distinct caller key.""" + source = _router_entry(model_id="cfg-1", model_info=dict(_VALID_PTU)) + clone = _router_entry(model_id="cfg-1-clone", model_info={**_VALID_PTU, "original_model_id": "cfg-1"}) + + loaded = await ptu_rollup._load_ptu_models( + _prisma_for([], _FakeSentinelTable()), router=_router_holding(source, clone) + ) + + assert [m.model_id for m in loaded.models] == ["cfg-1"] + + +@pytest.mark.asyncio +async def test_a_config_deployment_without_ptu_config_is_scanned_but_not_priced(): + """It has to stay in the scanned set or its leftover sentinel rows become unprunable.""" + entry = _router_entry(model_id="cfg-plain", model_info={"team_id": "t"}) + + loaded = await ptu_rollup._load_ptu_models(_prisma_for([], _FakeSentinelTable()), router=_router_holding(entry)) + + assert loaded.models == () + assert "cfg-plain" in loaded.scanned_ids + + +@pytest.mark.asyncio +async def test_no_router_in_the_process_prices_the_database_alone(): + """The rollup is importable and callable outside a running proxy.""" + loaded = await ptu_rollup._load_ptu_models( + _prisma_for([_model_row(model_id="db-1", model_info=dict(_VALID_PTU))], _FakeSentinelTable()), router=None + ) + + assert [m.model_id for m in loaded.models] == ["db-1"] + + +@pytest.mark.asyncio +async def test_a_config_deployment_is_charged_end_to_end(): """Through the scheduled entry point, so the charge lands in a sentinel row rather than stopping at the loader.""" table = _FakeSentinelTable() entry = _router_entry(model_id="cfg-1", model_name="gpt-4o-ptu", model_info=dict(_VALID_PTU)) - monkeypatch.setattr(ptu_rollup, "_running_router", lambda: _router_holding(entry)) - await run_scheduled_ptu_rollup(_prisma_for([], table), pod_lock_manager=_pod_lock(acquired=True), target_date=DAY) + await run_scheduled_ptu_rollup( + _prisma_for([], table), + pod_lock_manager=_pod_lock(acquired=True), + target_date=DAY, + router=_router_holding(entry), + ) assert ("t", DAY.isoformat(), PTU_SENTINEL_API_KEY, "cfg-1") in table.rows @pytest.mark.asyncio -async def test_a_stale_database_backed_router_entry_is_not_treated_as_config(monkeypatch): +async def test_a_stale_database_backed_router_entry_is_not_treated_as_config(): """The reconcile can leave a deployment on the router after its row is gone. The id anti-join cannot see that one, so the flag is what keeps it from being priced as though config.yaml had declared it.""" stale = _router_entry(model_id="db-gone", model_info={**_VALID_PTU, "db_model": True}) - monkeypatch.setattr(ptu_rollup, "_running_router", lambda: _router_holding(stale)) - loaded = await ptu_rollup._load_ptu_models(_prisma_for([], _FakeSentinelTable())) + loaded = await ptu_rollup._load_ptu_models(_prisma_for([], _FakeSentinelTable()), router=_router_holding(stale)) assert loaded.models == () -def test_the_router_lookup_reads_the_proxys_own_global(): - """Every other config test replaces this helper, so without one test driving the real - body a typo in the module path or the attribute name leaves the whole feature dead in - production with the suite still green.""" - import sys - import types as _types +@pytest.mark.asyncio +async def test_a_router_left_on_the_proxy_module_is_not_scanned(monkeypatch): + """A run scans the router its caller hands it and nothing else. Reading the proxy module's + global instead made every run depend on whatever else in the process had set one, which + is what a caller passing no router is asking not to happen.""" + import litellm.proxy.proxy_server as proxy_server - assert ptu_rollup._running_router() is None or "litellm.proxy.proxy_server" in sys.modules + ambient = _router_holding(_router_entry(model_id="ambient-1", model_info=dict(_VALID_PTU))) + monkeypatch.setattr(proxy_server, "llm_router", ambient, raising=False) - sentinel = object() - stub = _types.SimpleNamespace(llm_router=sentinel) - real = sys.modules.get("litellm.proxy.proxy_server") - sys.modules["litellm.proxy.proxy_server"] = stub - try: - assert ptu_rollup._running_router() is sentinel - del stub.llm_router - assert ptu_rollup._running_router() is None - finally: - if real is None: - del sys.modules["litellm.proxy.proxy_server"] - else: - sys.modules["litellm.proxy.proxy_server"] = real + loaded = await ptu_rollup._load_ptu_models(_prisma_for([], _FakeSentinelTable()), router=None) - -def test_the_router_lookup_returns_none_outside_a_proxy(): - import sys - - real = sys.modules.pop("litellm.proxy.proxy_server", None) - try: - assert ptu_rollup._running_router() is None - finally: - if real is not None: - sys.modules["litellm.proxy.proxy_server"] = real + assert loaded.models == () + assert loaded.scanned_ids == frozenset() + assert loaded.config_sourced is False @pytest.mark.parametrize("chunk", [None, ("dep-a", "dep-b")], ids=["unbounded", "bounded"]) @@ -2063,7 +2052,7 @@ def test_the_prune_filter_is_a_plain_dict(chunk): @pytest.mark.asyncio -async def test_the_catch_up_pass_reaches_a_config_declared_deployment(monkeypatch): +async def test_the_catch_up_pass_reaches_a_config_declared_deployment(): """The catch-up shares the loader, so config deployments join it without being wired in. That is what prices the elapsed days of a reservation declared before today.""" table = _FakeSentinelTable() @@ -2073,9 +2062,10 @@ async def test_the_catch_up_pass_reaches_a_config_declared_deployment(monkeypatc model_id="cfg-back", model_info={"ptu_count": 100, "cost_per_ptu_per_hour": 0.02, "team_id": "t", "ptu_effective_from": started}, ) - monkeypatch.setattr(ptu_rollup, "_running_router", lambda: _router_holding(entry)) - await run_scheduled_ptu_rollup(_prisma_for([], table), pod_lock_manager=_pod_lock(acquired=True)) + await run_scheduled_ptu_rollup( + _prisma_for([], table), pod_lock_manager=_pod_lock(acquired=True), router=_router_holding(entry) + ) charged = sorted(day for (_, day, _, model) in table.rows if model == "cfg-back") yesterday = (now.date() - timedelta(days=1)).isoformat() diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 83e9095c8ec..71fb184eb4b 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -11048,6 +11048,35 @@ async def test_ptu_rollup_job_registered_at_startup(monkeypatch): assert scheduler.get_job(PTU_ROLLUP_JOB_ID) is not None +@pytest.mark.asyncio +async def test_ptu_rollup_job_hands_the_rollup_the_proxys_router(monkeypatch): + """The rollup prices PTU deployments declared in config.yaml, which only the router + knows about. It takes the router as an argument, so nothing but this call site puts the + proxy's own router in front of it: without it that half of the feature is dead.""" + monkeypatch.delenv("STORE_MODEL_IN_DB", raising=False) + from litellm.proxy.spend_tracking import ptu_flat_cost_rollup + from litellm.proxy.spend_tracking.ptu_feature_flag import PTU_COST_ATTRIBUTION_ENV_VAR + from litellm.proxy.spend_tracking.ptu_flat_cost_rollup import PTU_ROLLUP_JOB_ID + + monkeypatch.setenv(PTU_COST_ATTRIBUTION_ENV_VAR, "true") + calls = [] + monkeypatch.setattr( + ptu_flat_cost_rollup, + "run_scheduled_ptu_rollup", + AsyncMock(side_effect=lambda *args, **kwargs: calls.append(kwargs)), + ) + + scheduler = await _run_scheduled_background_jobs() + + import litellm.proxy.proxy_server as ps + + router = MagicMock() + monkeypatch.setattr(ps, "llm_router", router) + await scheduler.get_job(PTU_ROLLUP_JOB_ID).func() + + assert [call["router"] for call in calls] == [router] + + @pytest.mark.asyncio async def test_ptu_rollup_job_not_registered_without_opt_in(monkeypatch): """Without LITELLM_ENABLE_PTU_COST_ATTRIBUTION the rollup never runs, so no sentinel row From 0b938e37f46d1149db2e99198b1036d207aeb49a Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 19 Aug 2026 09:52:33 +0000 Subject: [PATCH 02/29] test: invalidate memoized model-cost lookups between unit tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/test_litellm/conftest.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_litellm/conftest.py b/tests/test_litellm/conftest.py index 1fe73b552da..62c95cb100b 100644 --- a/tests/test_litellm/conftest.py +++ b/tests/test_litellm/conftest.py @@ -375,6 +375,9 @@ def isolate_litellm_state(): litellm.in_memory_llm_clients_cache.flush_cache() image_handling_module.in_memory_cache.flush_cache() _reset_module_level_aws_auth_caches() + # litellm.get_model_info() memoizes ModelInfo built from litellm.model_cost, so a + # test that rebinds the cost map leaves later tests pricing against the old map. + litellm_utils_module._invalidate_model_cost_lowercase_map() # Clear all callback lists to prevent cross-test contamination if hasattr(litellm, "callbacks"): @@ -418,6 +421,7 @@ def isolate_litellm_state(): litellm_utils_module._runtime_registered_model_cost.clear() litellm_utils_module._runtime_registered_model_cost.update(original_runtime_registered_model_cost) + litellm_utils_module._invalidate_model_cost_lowercase_map() for _router in tuple(litellm_router_module._live_routers): litellm_router_module._live_routers.discard(_router) From e4a72c587d8dfb372185fd9f6ec9dd8cf2ead111 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 20 Aug 2026 09:49:03 +0000 Subject: [PATCH 03/29] fix(ci): retry transient PyPI license lookups Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/code_coverage_tests/check_licenses.py | 66 ++++++++++++++----- tests/test_litellm/test_check_licenses.py | 71 +++++++++++++++++++++ 2 files changed, 120 insertions(+), 17 deletions(-) diff --git a/tests/code_coverage_tests/check_licenses.py b/tests/code_coverage_tests/check_licenses.py index 389e534b1ff..67d1d91a6f7 100644 --- a/tests/code_coverage_tests/check_licenses.py +++ b/tests/code_coverage_tests/check_licenses.py @@ -5,8 +5,9 @@ import json from pathlib import Path import re import sys +import time import tomllib -from typing import Dict, List, Optional, Set, Tuple +from typing import Callable, Dict, Final, List, Optional, Set, Tuple from packaging.requirements import Requirement import requests @@ -37,6 +38,8 @@ DEFAULT_TRANSITIVE_PIN_PACKAGES = ( # of the identifier, not an operator. _SPDX_OPERATOR_SPLIT = re.compile(r"\s+(?:OR|AND)\s+") _SPDX_WITH_SUFFIX = re.compile(r"\s+WITH\s+.*", re.DOTALL) +_PYPI_FETCH_ATTEMPTS: Final[int] = 3 +_PYPI_FETCH_BACKOFF_SECONDS: Final[float] = 0.5 @dataclass @@ -50,7 +53,10 @@ class PackageLicense: class LicenseChecker: def __init__( - self, config_file: Path = Path("./tests/code_coverage_tests/liccheck.ini") + self, + config_file: Path = Path("./tests/code_coverage_tests/liccheck.ini"), + http_get: Optional[Callable[..., requests.Response]] = None, + sleep: Optional[Callable[[float], None]] = None, ): if not config_file.exists(): print(f"Error: Config file {config_file} not found") @@ -79,6 +85,8 @@ class LicenseChecker: # Track package results self.package_results: List[PackageLicense] = [] + self._http_get = http_get + self._sleep = sleep @staticmethod def _normalize_package_name(package_name: str) -> str: @@ -123,21 +131,45 @@ class LicenseChecker: last resort derives the license from the ``License :: OSI Approved :: ...`` trove classifiers. """ - try: - url = f"https://pypi.org/pypi/{package_name}/{version}/json" - response = requests.get(url, timeout=10) - response.raise_for_status() - info = response.json().get("info", {}) or {} - return ( - info.get("license_expression") - or info.get("license") - or self._license_from_classifiers(info.get("classifiers") or []) - ) - except Exception as e: - print( - f"Warning: Failed to fetch license for {package_name} {version}: {str(e)}" - ) - return None + url = f"https://pypi.org/pypi/{package_name}/{version}/json" + http_get = self._http_get if self._http_get is not None else requests.get + sleep = self._sleep if self._sleep is not None else time.sleep + + for attempt in range(_PYPI_FETCH_ATTEMPTS): + try: + response = http_get(url, timeout=10) + response.raise_for_status() + info = response.json().get("info", {}) or {} + return ( + info.get("license_expression") + or info.get("license") + or self._license_from_classifiers(info.get("classifiers") or []) + ) + except requests.HTTPError as error: + status_code = error.response.status_code if error.response is not None else None + if status_code != 429 and (status_code is None or status_code < 500): + print( + f"Warning: Failed to fetch license for {package_name} {version}: {str(error)}" + ) + return None + if attempt == _PYPI_FETCH_ATTEMPTS - 1: + print( + f"Warning: Failed to fetch license for {package_name} {version}: {str(error)}" + ) + return None + sleep(_PYPI_FETCH_BACKOFF_SECONDS) + except (requests.ConnectionError, requests.Timeout) as error: + if attempt == _PYPI_FETCH_ATTEMPTS - 1: + print( + f"Warning: Failed to fetch license for {package_name} {version}: {str(error)}" + ) + return None + sleep(_PYPI_FETCH_BACKOFF_SECONDS) + except Exception as error: + print( + f"Warning: Failed to fetch license for {package_name} {version}: {str(error)}" + ) + return None @staticmethod def _license_from_classifiers(classifiers: List[str]) -> Optional[str]: diff --git a/tests/test_litellm/test_check_licenses.py b/tests/test_litellm/test_check_licenses.py index 4d72f185a25..1218e44fade 100644 --- a/tests/test_litellm/test_check_licenses.py +++ b/tests/test_litellm/test_check_licenses.py @@ -12,6 +12,8 @@ import os import sys from pathlib import Path +import requests + _CODE_COVERAGE_DIR = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "code_coverage_tests" ) @@ -122,6 +124,75 @@ def test_get_license_returns_none_on_request_failure(monkeypatch): assert checker.get_package_license_from_pypi("pkg", "1.0.0") is None +def test_get_license_retries_connection_error_then_resolves_license(): + responses = iter( + ( + requests.ConnectionError("connection reset"), + requests.ConnectionError("connection reset"), + _FakeResponse({"info": {"license_expression": "MIT"}}), + ) + ) + calls = [] + sleeps = [] + + def _fake_get(url, timeout=None): + calls.append((url, timeout)) + response = next(responses) + if isinstance(response, Exception): + raise response + return response + + checker = check_licenses.LicenseChecker( + config_file=_LICCHECK_INI, + http_get=_fake_get, + sleep=sleeps.append, + ) + + assert checker.get_package_license_from_pypi("pkg", "1.0.0") == "MIT" + assert len(calls) == 3 + assert len(sleeps) == 2 + + +def test_get_license_does_not_retry_not_found_http_error(): + response = requests.Response() + response.status_code = 404 + calls = [] + sleeps = [] + + def _fake_get(url, timeout=None): + calls.append((url, timeout)) + raise requests.HTTPError("not found", response=response) + + checker = check_licenses.LicenseChecker( + config_file=_LICCHECK_INI, + http_get=_fake_get, + sleep=sleeps.append, + ) + + assert checker.get_package_license_from_pypi("pkg", "1.0.0") is None + assert len(calls) == 1 + assert sleeps == [] + + +def test_get_license_returns_none_after_connection_retry_limit(): + calls = [] + sleeps = [] + + def _fake_get(url, timeout=None): + calls.append((url, timeout)) + raise requests.ConnectionError("connection reset") + + checker = check_licenses.LicenseChecker( + config_file=_LICCHECK_INI, + http_get=_fake_get, + sleep=sleeps.append, + ) + + assert checker.get_package_license_from_pypi("pkg", "1.0.0") is None + assert len(calls) == 3 + assert len(sleeps) == 2 + + # -------------------------------------------------------------------------- # is_license_acceptable: SPDX identifiers and compound expressions # -------------------------------------------------------------------------- From 134b6252e0012e92ac59c2f335af354b941aba4e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 20 Aug 2026 09:54:38 +0000 Subject: [PATCH 04/29] refactor(ci): simplify PyPI license retries Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/code_coverage_tests/check_licenses.py | 42 ++++++++++----------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/tests/code_coverage_tests/check_licenses.py b/tests/code_coverage_tests/check_licenses.py index 67d1d91a6f7..158e25180e1 100644 --- a/tests/code_coverage_tests/check_licenses.py +++ b/tests/code_coverage_tests/check_licenses.py @@ -7,7 +7,7 @@ import re import sys import time import tomllib -from typing import Callable, Dict, Final, List, Optional, Set, Tuple +from typing import Callable, Dict, Final, List, Optional, Protocol, Set, Tuple from packaging.requirements import Requirement import requests @@ -42,6 +42,11 @@ _PYPI_FETCH_ATTEMPTS: Final[int] = 3 _PYPI_FETCH_BACKOFF_SECONDS: Final[float] = 0.5 +class _HttpGet(Protocol): + def __call__(self, url: str, *, timeout: float) -> requests.Response: + ... + + @dataclass class PackageLicense: name: str @@ -55,7 +60,7 @@ class LicenseChecker: def __init__( self, config_file: Path = Path("./tests/code_coverage_tests/liccheck.ini"), - http_get: Optional[Callable[..., requests.Response]] = None, + http_get: Optional[_HttpGet] = None, sleep: Optional[Callable[[float], None]] = None, ): if not config_file.exists(): @@ -145,31 +150,24 @@ class LicenseChecker: or info.get("license") or self._license_from_classifiers(info.get("classifiers") or []) ) - except requests.HTTPError as error: - status_code = error.response.status_code if error.response is not None else None - if status_code != 429 and (status_code is None or status_code < 500): - print( - f"Warning: Failed to fetch license for {package_name} {version}: {str(error)}" - ) - return None - if attempt == _PYPI_FETCH_ATTEMPTS - 1: - print( - f"Warning: Failed to fetch license for {package_name} {version}: {str(error)}" - ) - return None - sleep(_PYPI_FETCH_BACKOFF_SECONDS) - except (requests.ConnectionError, requests.Timeout) as error: - if attempt == _PYPI_FETCH_ATTEMPTS - 1: - print( - f"Warning: Failed to fetch license for {package_name} {version}: {str(error)}" - ) - return None - sleep(_PYPI_FETCH_BACKOFF_SECONDS) except Exception as error: + if self._is_retryable_pypi_error(error) and attempt < _PYPI_FETCH_ATTEMPTS - 1: + sleep(_PYPI_FETCH_BACKOFF_SECONDS) + continue print( f"Warning: Failed to fetch license for {package_name} {version}: {str(error)}" ) return None + return None + + @staticmethod + def _is_retryable_pypi_error(error: Exception) -> bool: + if isinstance(error, (requests.ConnectionError, requests.Timeout)): + return True + if not isinstance(error, requests.HTTPError) or error.response is None: + return False + status_code = error.response.status_code + return status_code == 429 or status_code >= 500 @staticmethod def _license_from_classifiers(classifiers: List[str]) -> Optional[str]: From 8deade4f345bcd983c721f0862fcd8ad30dfebda Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 23 Aug 2026 10:02:41 +0000 Subject: [PATCH 05/29] test(ptu): drop the assertion on the flag removed upstream Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/spend_tracking/test_ptu_flat_cost_rollup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_litellm/proxy/spend_tracking/test_ptu_flat_cost_rollup.py b/tests/test_litellm/proxy/spend_tracking/test_ptu_flat_cost_rollup.py index 76fa41c83be..8f25cffecf5 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_ptu_flat_cost_rollup.py +++ b/tests/test_litellm/proxy/spend_tracking/test_ptu_flat_cost_rollup.py @@ -2034,7 +2034,6 @@ async def test_a_router_left_on_the_proxy_module_is_not_scanned(monkeypatch): assert loaded.models == () assert loaded.scanned_ids == frozenset() - assert loaded.config_sourced is False def test_the_prune_filter_is_a_plain_dict(): From 54ea379c91ab4c60e45c4b671fdd76003fea6188 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 25 Aug 2026 10:03:57 +0000 Subject: [PATCH 06/29] fix(tests): drain the logging worker queue between MCP tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/mcp_tests/conftest.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/mcp_tests/conftest.py b/tests/mcp_tests/conftest.py index d1dc3ec7216..e46c03b5498 100644 --- a/tests/mcp_tests/conftest.py +++ b/tests/mcp_tests/conftest.py @@ -42,6 +42,22 @@ def setup_and_teardown(): asyncio.set_event_loop(None) # Remove the reference to the loop +@pytest.fixture(scope="function", autouse=True) +async def drain_logging_worker(): + """ + The logging queue is bound to the running loop, so anything left queued when a test's loop + goes away is carried onto the next test's loop and fires against its callbacks. + """ + from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + + yield + + try: + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.clear_queue(), timeout=10) + except asyncio.TimeoutError: + pass + + def pytest_collection_modifyitems(config, items): # Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests custom_logger_tests = [ From ab160fb9537dee34183e6a5cb730791b7e1791a0 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 25 Aug 2026 13:26:48 +0000 Subject: [PATCH 07/29] fix(model_prices): sync gpt-5.6-sol bedrock rates, add gpt-5.6-cyber, fix claude 3 1h cache writes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 80 +++++++++++++------ model_prices_and_context_window.json | 80 +++++++++++++------ ..._cross_region_inference_profile_mapping.py | 24 +++--- ...bedrock_mantle_responses_transformation.py | 13 ++- 4 files changed, 131 insertions(+), 66 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 9f953e11df1..73e16715cf0 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -12282,7 +12282,7 @@ }, "claude-3-haiku-20240307": { "cache_creation_input_token_cost": 3e-07, - "cache_creation_input_token_cost_above_1hr": 6e-06, + "cache_creation_input_token_cost_above_1hr": 5e-07, "cache_read_input_token_cost": 3e-08, "deprecation_date": "2026-04-20", "input_cost_per_token": 2.5e-07, @@ -12301,7 +12301,7 @@ }, "claude-3-opus-20240229": { "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 6e-06, + "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, "deprecation_date": "2026-01-05", "input_cost_per_token": 1.5e-05, @@ -49007,14 +49007,14 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-5.6-sol": { - "input_cost_per_token": 5.5e-06, - "input_cost_per_token_above_272k_tokens": 1.1e-05, - "cache_creation_input_token_cost": 6.875e-06, - "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, - "cache_read_input_token_cost": 5.5e-07, - "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, - "output_cost_per_token": 3.3e-05, - "output_cost_per_token_above_272k_tokens": 4.95e-05, + "input_cost_per_token": 4.4e-06, + "input_cost_per_token_above_272k_tokens": 8.8e-06, + "cache_creation_input_token_cost": 5.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.1e-05, + "cache_read_input_token_cost": 4.4e-07, + "cache_read_input_token_cost_above_272k_tokens": 8.8e-07, + "output_cost_per_token": 2.2e-05, + "output_cost_per_token_above_272k_tokens": 3.3e-05, "litellm_provider": "bedrock_mantle", "max_input_tokens": 1000000, "max_output_tokens": 128000, @@ -49070,6 +49070,34 @@ "supports_tool_choice": true, "supports_vision": true }, + "bedrock_mantle/openai.gpt-5.6-cyber": { + "input_cost_per_token": 1.375e-05, + "cache_creation_input_token_cost": 1.71875e-05, + "cache_read_input_token_cost": 1.375e-06, + "output_cost_per_token": 8.25e-05, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "bedrock_mantle/openai.gpt-5.6-luna": { "input_cost_per_token": 2.2e-07, "input_cost_per_token_above_272k_tokens": 4.4e-07, @@ -49103,14 +49131,14 @@ "supports_vision": true }, "us.openai.gpt-5.6-sol": { - "input_cost_per_token": 5.5e-06, - "input_cost_per_token_above_272k_tokens": 1.1e-05, - "cache_creation_input_token_cost": 6.875e-06, - "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, - "cache_read_input_token_cost": 5.5e-07, - "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, - "output_cost_per_token": 3.3e-05, - "output_cost_per_token_above_272k_tokens": 4.95e-05, + "input_cost_per_token": 4.4e-06, + "input_cost_per_token_above_272k_tokens": 8.8e-06, + "cache_creation_input_token_cost": 5.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.1e-05, + "cache_read_input_token_cost": 4.4e-07, + "cache_read_input_token_cost_above_272k_tokens": 8.8e-07, + "output_cost_per_token": 2.2e-05, + "output_cost_per_token_above_272k_tokens": 3.3e-05, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, @@ -49128,14 +49156,14 @@ "supports_vision": true }, "global.openai.gpt-5.6-sol": { - "input_cost_per_token": 5e-06, - "input_cost_per_token_above_272k_tokens": 1e-05, - "cache_creation_input_token_cost": 6.25e-06, - "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05, - "cache_read_input_token_cost": 5e-07, - "cache_read_input_token_cost_above_272k_tokens": 1e-06, - "output_cost_per_token": 3e-05, - "output_cost_per_token_above_272k_tokens": 4.5e-05, + "input_cost_per_token": 4e-06, + "input_cost_per_token_above_272k_tokens": 8e-06, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1e-05, + "cache_read_input_token_cost": 4e-07, + "cache_read_input_token_cost_above_272k_tokens": 8e-07, + "output_cost_per_token": 2e-05, + "output_cost_per_token_above_272k_tokens": 3e-05, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 9f953e11df1..73e16715cf0 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -12282,7 +12282,7 @@ }, "claude-3-haiku-20240307": { "cache_creation_input_token_cost": 3e-07, - "cache_creation_input_token_cost_above_1hr": 6e-06, + "cache_creation_input_token_cost_above_1hr": 5e-07, "cache_read_input_token_cost": 3e-08, "deprecation_date": "2026-04-20", "input_cost_per_token": 2.5e-07, @@ -12301,7 +12301,7 @@ }, "claude-3-opus-20240229": { "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 6e-06, + "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, "deprecation_date": "2026-01-05", "input_cost_per_token": 1.5e-05, @@ -49007,14 +49007,14 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-5.6-sol": { - "input_cost_per_token": 5.5e-06, - "input_cost_per_token_above_272k_tokens": 1.1e-05, - "cache_creation_input_token_cost": 6.875e-06, - "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, - "cache_read_input_token_cost": 5.5e-07, - "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, - "output_cost_per_token": 3.3e-05, - "output_cost_per_token_above_272k_tokens": 4.95e-05, + "input_cost_per_token": 4.4e-06, + "input_cost_per_token_above_272k_tokens": 8.8e-06, + "cache_creation_input_token_cost": 5.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.1e-05, + "cache_read_input_token_cost": 4.4e-07, + "cache_read_input_token_cost_above_272k_tokens": 8.8e-07, + "output_cost_per_token": 2.2e-05, + "output_cost_per_token_above_272k_tokens": 3.3e-05, "litellm_provider": "bedrock_mantle", "max_input_tokens": 1000000, "max_output_tokens": 128000, @@ -49070,6 +49070,34 @@ "supports_tool_choice": true, "supports_vision": true }, + "bedrock_mantle/openai.gpt-5.6-cyber": { + "input_cost_per_token": 1.375e-05, + "cache_creation_input_token_cost": 1.71875e-05, + "cache_read_input_token_cost": 1.375e-06, + "output_cost_per_token": 8.25e-05, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "bedrock_mantle/openai.gpt-5.6-luna": { "input_cost_per_token": 2.2e-07, "input_cost_per_token_above_272k_tokens": 4.4e-07, @@ -49103,14 +49131,14 @@ "supports_vision": true }, "us.openai.gpt-5.6-sol": { - "input_cost_per_token": 5.5e-06, - "input_cost_per_token_above_272k_tokens": 1.1e-05, - "cache_creation_input_token_cost": 6.875e-06, - "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, - "cache_read_input_token_cost": 5.5e-07, - "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, - "output_cost_per_token": 3.3e-05, - "output_cost_per_token_above_272k_tokens": 4.95e-05, + "input_cost_per_token": 4.4e-06, + "input_cost_per_token_above_272k_tokens": 8.8e-06, + "cache_creation_input_token_cost": 5.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.1e-05, + "cache_read_input_token_cost": 4.4e-07, + "cache_read_input_token_cost_above_272k_tokens": 8.8e-07, + "output_cost_per_token": 2.2e-05, + "output_cost_per_token_above_272k_tokens": 3.3e-05, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, @@ -49128,14 +49156,14 @@ "supports_vision": true }, "global.openai.gpt-5.6-sol": { - "input_cost_per_token": 5e-06, - "input_cost_per_token_above_272k_tokens": 1e-05, - "cache_creation_input_token_cost": 6.25e-06, - "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05, - "cache_read_input_token_cost": 5e-07, - "cache_read_input_token_cost_above_272k_tokens": 1e-06, - "output_cost_per_token": 3e-05, - "output_cost_per_token_above_272k_tokens": 4.5e-05, + "input_cost_per_token": 4e-06, + "input_cost_per_token_above_272k_tokens": 8e-06, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1e-05, + "cache_read_input_token_cost": 4e-07, + "cache_read_input_token_cost_above_272k_tokens": 8e-07, + "output_cost_per_token": 2e-05, + "output_cost_per_token_above_272k_tokens": 3e-05, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, diff --git a/tests/test_litellm/llms/bedrock/test_cross_region_inference_profile_mapping.py b/tests/test_litellm/llms/bedrock/test_cross_region_inference_profile_mapping.py index dbd31c7e81b..694ed109025 100644 --- a/tests/test_litellm/llms/bedrock/test_cross_region_inference_profile_mapping.py +++ b/tests/test_litellm/llms/bedrock/test_cross_region_inference_profile_mapping.py @@ -59,17 +59,17 @@ class GptProfile(NamedTuple): GPT_5_6_PROFILES = [ GptProfile( model_id="us.openai.gpt-5.6-sol", - input_cost=5.5e-06, input_cost_above_272k=1.1e-05, - cache_write=6.875e-06, cache_write_above_272k=1.375e-05, - cache_read=5.5e-07, cache_read_above_272k=1.1e-06, - output_cost=3.3e-05, output_cost_above_272k=4.95e-05, + input_cost=4.4e-06, input_cost_above_272k=8.8e-06, + cache_write=5.5e-06, cache_write_above_272k=1.1e-05, + cache_read=4.4e-07, cache_read_above_272k=8.8e-07, + output_cost=2.2e-05, output_cost_above_272k=3.3e-05, ), GptProfile( model_id="global.openai.gpt-5.6-sol", - input_cost=5e-06, input_cost_above_272k=1e-05, - cache_write=6.25e-06, cache_write_above_272k=1.25e-05, - cache_read=5e-07, cache_read_above_272k=1e-06, - output_cost=3e-05, output_cost_above_272k=4.5e-05, + input_cost=4e-06, input_cost_above_272k=8e-06, + cache_write=5e-06, cache_write_above_272k=1e-05, + cache_read=4e-07, cache_read_above_272k=8e-07, + output_cost=2e-05, output_cost_above_272k=3e-05, ), GptProfile( model_id="us.openai.gpt-5.6-terra", @@ -221,7 +221,7 @@ def test_bedrock_gpt_5_6_above_272k_tier_applies_to_cost(local_model_cost_map): custom_llm_provider="bedrock", ) - assert cost == pytest.approx((300000 * 1.1e-05) + (1000 * 4.95e-05), rel=1e-9) + assert cost == pytest.approx((300000 * 8.8e-06) + (1000 * 3.3e-05), rel=1e-9) def test_bedrock_gpt_5_6_bills_cache_read_tokens(local_model_cost_map): @@ -241,10 +241,10 @@ def test_bedrock_gpt_5_6_bills_cache_read_tokens(local_model_cost_map): custom_llm_provider="bedrock", ) - expected = (2 * 5.5e-06) + (15609 * 5.5e-07) + (5 * 3.3e-05) + expected = (2 * 4.4e-06) + (15609 * 4.4e-07) + (5 * 2.2e-05) assert cost == pytest.approx(expected, rel=1e-9) # Without cache_read_input_token_cost the cached prefix bills at zero. - assert cost > (15611 * 5.5e-06) * 0.1 + assert cost > (15611 * 4.4e-06) * 0.1 def test_bedrock_gpt_5_6_bills_cache_write_tokens(local_model_cost_map): @@ -263,7 +263,7 @@ def test_bedrock_gpt_5_6_bills_cache_write_tokens(local_model_cost_map): custom_llm_provider="bedrock", ) - expected = (2 * 5.5e-06) + (15609 * 6.875e-06) + (5 * 3.3e-05) + expected = (2 * 4.4e-06) + (15609 * 5.5e-06) + (5 * 2.2e-05) assert cost == pytest.approx(expected, rel=1e-9) diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py index 9e05d48a18f..94c2d6ff6b3 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py @@ -1506,10 +1506,19 @@ class TestBedrockMantleResponsesPricing: assert info["cache_read_input_token_cost"] == pytest.approx(2.75e-07) assert info["max_input_tokens"] == 272000 + def test_gpt_5_6_cyber_pricing_and_mode(self, local_cost_map): + info = litellm.get_model_info("bedrock_mantle/openai.gpt-5.6-cyber") + assert info["mode"] == "responses" + assert info["input_cost_per_token"] == pytest.approx(1.375e-05) + assert info["cache_creation_input_token_cost"] == pytest.approx(1.71875e-05) + assert info["cache_read_input_token_cost"] == pytest.approx(1.375e-06) + assert info["output_cost_per_token"] == pytest.approx(8.25e-05) + assert info["max_input_tokens"] == 272000 + @pytest.mark.parametrize( "model, input_cost, cache_creation_cost, cache_read_cost, output_cost", [ - ("openai.gpt-5.6-sol", 5.5e-06, 6.875e-06, 5.5e-07, 3.3e-05), + ("openai.gpt-5.6-sol", 4.4e-06, 5.5e-06, 4.4e-07, 2.2e-05), ("openai.gpt-5.6-terra", 2.2e-06, 2.75e-06, 2.2e-07, 1.32e-05), ("openai.gpt-5.6-luna", 2.2e-07, 2.75e-07, 2.2e-08, 1.32e-06), ], @@ -1532,7 +1541,7 @@ class TestBedrockMantleResponsesPricing: @pytest.mark.parametrize( "model, input_cost, output_cost", [ - ("openai.gpt-5.6-sol", 5.5e-06, 3.3e-05), + ("openai.gpt-5.6-sol", 4.4e-06, 2.2e-05), ("openai.gpt-5.6-terra", 2.2e-06, 1.32e-05), ("openai.gpt-5.6-luna", 2.2e-07, 1.32e-06), ], From 310591f63c283633199cbc0d3249f3d911d0220a Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:05:07 +0000 Subject: [PATCH 08/29] test(model_prices): pin claude 3 1h cache write rates to 2x base input Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...test_anthropic_sonnet_1hr_cache_pricing.py | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/test_litellm/test_anthropic_sonnet_1hr_cache_pricing.py b/tests/test_litellm/test_anthropic_sonnet_1hr_cache_pricing.py index f534b431508..11fcdf31dfc 100644 --- a/tests/test_litellm/test_anthropic_sonnet_1hr_cache_pricing.py +++ b/tests/test_litellm/test_anthropic_sonnet_1hr_cache_pricing.py @@ -87,3 +87,56 @@ def test_anthropic_sonnet_1hr_cache_write_pricing( ), f"{model_key}: long-context 1hr/5min ratio is {ratio_lc}, expected 1.6" else: assert "cache_creation_input_token_cost_above_1hr_above_200k_tokens" not in info + + +CLAUDE_3_EXPECTED = [ + ("claude-3-haiku-20240307", 5e-07), + ("claude-3-opus-20240229", 3e-05), +] + + +@pytest.mark.parametrize("model_key, expected_1hr", CLAUDE_3_EXPECTED) +def test_claude_3_1hr_cache_write_pricing(model_data, model_key, expected_1hr): + """Haiku 3 and Opus 3 both carried Sonnet's 6e-06 1hr rate, overbilling Haiku 3 + 1-hour cache writes 12x and underbilling Opus 3 5x.""" + info = model_data[model_key] + + assert info["cache_creation_input_token_cost_above_1hr"] == expected_1hr + + +@pytest.mark.parametrize("model_key, expected_1hr", CLAUDE_3_EXPECTED) +def test_backup_matches_main_for_claude_3_1hr_cache_write(model_key, expected_1hr): + json_path = os.path.join( + os.path.dirname(__file__), + "../../litellm/model_prices_and_context_window_backup.json", + ) + with open(json_path) as f: + backup = json.load(f) + + assert ( + backup[model_key]["cache_creation_input_token_cost_above_1hr"] == expected_1hr + ) + + +def test_first_party_anthropic_1hr_cache_writes_are_2x_base_input(model_data): + """Anthropic charges 1-hour cache writes at 2x base input for every first-party + model, so any entry that drifts off that multiple is a copy-paste error.""" + offenders = tuple( + ( + model_key, + info["input_cost_per_token"], + info["cache_creation_input_token_cost_above_1hr"], + ) + for model_key, info in model_data.items() + if isinstance(info, dict) + and info.get("litellm_provider") == "anthropic" + and info.get("input_cost_per_token") + and info.get("cache_creation_input_token_cost_above_1hr") + and abs( + info["cache_creation_input_token_cost_above_1hr"] + - 2 * info["input_cost_per_token"] + ) + > 1e-12 + ) + + assert offenders == (), f"1hr cache write is not 2x base input for: {offenders}" From fb15851f535dbb2bf68b85a0b73e3b4bf1065319 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 25 Aug 2026 19:16:14 +0000 Subject: [PATCH 09/29] fix(model_prices): verified Novita, DeepInfra, W&B, Gemini cache-read and Fireworks registry fixes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 2340 ++++++++++++++++- model_prices_and_context_window.json | 2340 ++++++++++++++++- 2 files changed, 4406 insertions(+), 274 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 87b4b08ea62..e133f31cdac 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -16023,12 +16023,13 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 8e-08, - "output_cost_per_token": 9e-08, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 4e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/NousResearch/Hermes-3-Llama-3.1-405B": { "max_tokens": 131072, @@ -16045,11 +16046,12 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 3e-07, - "output_cost_per_token": 3e-07, + "input_cost_per_token": 7e-07, + "output_cost_per_token": 7e-07, "litellm_provider": "deepinfra", "mode": "chat", - "supports_tool_choice": false + "supports_tool_choice": false, + "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/QwQ-32B": { "max_tokens": 131072, @@ -16066,12 +16068,13 @@ "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, - "input_cost_per_token": 1.2e-07, - "output_cost_per_token": 3.9e-07, + "input_cost_per_token": 3.6e-07, + "output_cost_per_token": 4e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen2.5-7B-Instruct": { "max_tokens": 32768, @@ -16099,12 +16102,13 @@ "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, - "input_cost_per_token": 6e-08, + "input_cost_per_token": 1.2e-07, "output_cost_per_token": 2.4e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-235B-A22B": { "max_tokens": 40960, @@ -16122,11 +16126,12 @@ "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 9e-08, - "output_cost_per_token": 6e-07, + "output_cost_per_token": 5.5e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-235B-A22B-Thinking-2507": { "max_tokens": 262144, @@ -16143,23 +16148,25 @@ "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, - "input_cost_per_token": 8e-08, - "output_cost_per_token": 2.9e-07, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 5e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-32B": { "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, - "input_cost_per_token": 1e-07, + "input_cost_per_token": 8e-08, "output_cost_per_token": 2.8e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-Coder-480B-A35B-Instruct": { "max_tokens": 262144, @@ -16176,23 +16183,27 @@ "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, - "input_cost_per_token": 2.9e-07, - "output_cost_per_token": 1.2e-06, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1e-06, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "cache_read_input_token_cost": 1e-07, + "supports_prompt_caching": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-Next-80B-A3B-Instruct": { "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, - "input_cost_per_token": 1.4e-07, - "output_cost_per_token": 1.4e-06, + "input_cost_per_token": 9e-08, + "output_cost_per_token": 1.1e-06, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-Next-80B-A3B-Thinking": { "max_tokens": 262144, @@ -16219,11 +16230,12 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 6.5e-07, - "output_cost_per_token": 7.5e-07, + "input_cost_per_token": 8.5e-07, + "output_cost_per_token": 8.5e-07, "litellm_provider": "deepinfra", "mode": "chat", - "supports_tool_choice": false + "supports_tool_choice": false, + "source": "https://deepinfra.com/pricing" }, "deepinfra/Sao10K/L3.3-70B-Euryale-v2.3": { "max_tokens": 131072, @@ -16349,36 +16361,41 @@ "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, - "input_cost_per_token": 3.8e-07, + "input_cost_per_token": 3.2e-07, "output_cost_per_token": 8.9e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V3-0324": { "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, - "input_cost_per_token": 2.5e-07, - "output_cost_per_token": 8.8e-07, + "input_cost_per_token": 2.4e-07, + "output_cost_per_token": 9e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "cache_read_input_token_cost": 1.35e-07, + "supports_prompt_caching": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V3.1": { "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, - "input_cost_per_token": 2.7e-07, - "output_cost_per_token": 1e-06, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 9.5e-07, "cache_read_input_token_cost": 2.16e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, "supports_reasoning": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V3.1-Terminus": { "max_tokens": 163840, @@ -16432,33 +16449,36 @@ "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 5e-08, - "output_cost_per_token": 1e-07, + "output_cost_per_token": 1.5e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-3-27b-it": { "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 9e-08, + "input_cost_per_token": 8e-08, "output_cost_per_token": 1.6e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-3-4b-it": { "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 4e-08, - "output_cost_per_token": 8e-08, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 1e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Llama-3.2-11B-Vision-Instruct": { "max_tokens": 131072, @@ -16496,34 +16516,37 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 1.3e-07, - "output_cost_per_token": 3.9e-07, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3.2e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_function_calling": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8": { "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "input_cost_per_token": 1.5e-07, - "output_cost_per_token": 6e-07, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 8e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "max_tokens": 327680, "max_input_tokens": 327680, "max_output_tokens": 327680, - "input_cost_per_token": 8e-08, + "input_cost_per_token": 1e-07, "output_cost_per_token": 3e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Llama-Guard-3-8B": { "max_tokens": 131072, @@ -16571,12 +16594,13 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 1e-07, - "output_cost_per_token": 2.8e-07, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 4e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Meta-Llama-3.1-8B-Instruct": { "max_tokens": 131072, @@ -16594,11 +16618,12 @@ "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-08, - "output_cost_per_token": 3e-08, + "output_cost_per_token": 4e-08, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/microsoft/WizardLM-2-8x22B": { "max_tokens": 65536, @@ -16625,12 +16650,13 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 2e-08, - "output_cost_per_token": 4e-08, + "input_cost_per_token": 1.9e-08, + "output_cost_per_token": 3e-08, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/mistralai/Mistral-Small-24B-Instruct-2501": { "max_tokens": 32768, @@ -16712,14 +16738,16 @@ }, "deepinfra/nvidia/NVIDIA-Nemotron-3.5-Lightning": { "max_input_tokens": 262144, - "input_cost_per_token": 5e-08, + "input_cost_per_token": 8e-08, "output_cost_per_token": 2e-07, "litellm_provider": "deepinfra", "mode": "chat", - "source": "https://deepinfra.com/nvidia/NVIDIA-Nemotron-3.5-Lightning", + "source": "https://deepinfra.com/pricing", "supports_tool_choice": true, "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 4e-08, + "supports_prompt_caching": true }, "deepinfra/nvidia/NVIDIA-Nemotron-Nano-9B-v2": { "max_tokens": 131072, @@ -16736,23 +16764,25 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 5e-08, - "output_cost_per_token": 4.5e-07, + "input_cost_per_token": 3.7e-08, + "output_cost_per_token": 1.7e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/openai/gpt-oss-20b": { "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 4e-08, - "output_cost_per_token": 1.5e-07, + "input_cost_per_token": 3e-08, + "output_cost_per_token": 1.4e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-4.5": { "max_tokens": 131072, @@ -20237,7 +20267,7 @@ "supports_image_size": false }, "gemini-2.5-flash-preview-09-2025": { - "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost": 3e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, "litellm_provider": "vertex_ai-language-models", @@ -20247,7 +20277,7 @@ "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, - "source": "https://developers.googleblog.com/en/continuing-to-bring-you-our-latest-models-with-an-improved-gemini-2-5-flash-and-flash-lite-release/", + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -20373,7 +20403,7 @@ }, "gemini-2.5-flash-lite-preview-06-17": { "deprecation_date": "2025-11-18", - "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost": 1e-08, "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-language-models", @@ -20383,7 +20413,7 @@ "mode": "chat", "output_cost_per_reasoning_token": 4e-07, "output_cost_per_token": 4e-07, - "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -21953,7 +21983,7 @@ "supports_image_size": false }, "gemini/gemini-2.5-flash-preview-09-2025": { - "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost": 3e-08, "deprecation_date": "2026-02-17", "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, @@ -21965,7 +21995,7 @@ "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, "rpm": 15, - "source": "https://developers.googleblog.com/en/continuing-to-bring-you-our-latest-models-with-an-improved-gemini-2-5-flash-and-flash-lite-release/", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -22001,7 +22031,7 @@ "supports_image_size": false }, "gemini/gemini-flash-latest": { - "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost": 3e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, "litellm_provider": "gemini", @@ -22012,7 +22042,7 @@ "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, "rpm": 15, - "source": "https://developers.googleblog.com/en/continuing-to-bring-you-our-latest-models-with-an-improved-gemini-2-5-flash-and-flash-lite-release/", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -22047,7 +22077,7 @@ } }, "gemini/gemini-flash-lite-latest": { - "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost": 1e-08, "input_cost_per_audio_token": 3e-07, "input_cost_per_token": 1e-07, "litellm_provider": "gemini", @@ -22058,7 +22088,7 @@ "output_cost_per_reasoning_token": 4e-07, "output_cost_per_token": 4e-07, "rpm": 15, - "source": "https://developers.googleblog.com/en/continuing-to-bring-you-our-latest-models-with-an-improved-gemini-2-5-flash-and-flash-lite-release/", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -22094,7 +22124,7 @@ }, "gemini/gemini-2.5-flash-lite-preview-06-17": { "deprecation_date": "2025-11-18", - "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost": 1e-08, "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 1e-07, "litellm_provider": "gemini", @@ -22105,7 +22135,7 @@ "output_cost_per_reasoning_token": 4e-07, "output_cost_per_token": 4e-07, "rpm": 15, - "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-lite", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -42511,19 +42541,21 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 0.015, - "output_cost_per_token": 0.06, + "input_cost_per_token": 3e-08, + "output_cost_per_token": 1.7e-07, "litellm_provider": "wandb", - "mode": "chat" + "mode": "chat", + "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/openai/gpt-oss-20b": { "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 0.005, - "output_cost_per_token": 0.02, + "input_cost_per_token": 3e-08, + "output_cost_per_token": 1.3e-07, "litellm_provider": "wandb", - "mode": "chat" + "mode": "chat", + "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/zai-org/GLM-4.5": { "max_tokens": 131072, @@ -42547,10 +42579,11 @@ "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, - "input_cost_per_token": 0.1, - "output_cost_per_token": 0.15, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 1.5e-06, "litellm_provider": "wandb", - "mode": "chat" + "mode": "chat", + "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3-235B-A22B-Thinking-2507": { "max_tokens": 262144, @@ -42602,19 +42635,21 @@ "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, - "input_cost_per_token": 0.022, - "output_cost_per_token": 0.022, + "input_cost_per_token": 2.2e-07, + "output_cost_per_token": 2.2e-07, "litellm_provider": "wandb", - "mode": "chat" + "mode": "chat", + "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/deepseek-ai/DeepSeek-V3.1": { "max_tokens": 128000, - "max_input_tokens": 128000, + "max_input_tokens": 161000, "max_output_tokens": 128000, - "input_cost_per_token": 0.055, - "output_cost_per_token": 0.165, + "input_cost_per_token": 5.5e-07, + "output_cost_per_token": 1.65e-06, "litellm_provider": "wandb", - "mode": "chat" + "mode": "chat", + "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/deepseek-ai/DeepSeek-R1-0528": { "max_tokens": 161000, @@ -42638,10 +42673,11 @@ "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, - "input_cost_per_token": 0.071, - "output_cost_per_token": 0.071, + "input_cost_per_token": 7.1e-07, + "output_cost_per_token": 7.1e-07, "litellm_provider": "wandb", - "mode": "chat" + "mode": "chat", + "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "max_tokens": 64000, @@ -46608,8 +46644,8 @@ "novita/xiaomimimo/mimo-v2-flash": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1e-07, - "output_cost_per_token": 3e-07, + "input_cost_per_token": 1.1e-07, + "output_cost_per_token": 3.3e-07, "max_input_tokens": 262144, "max_output_tokens": 32000, "max_tokens": 32000, @@ -46618,8 +46654,8 @@ "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, - "cache_read_input_token_cost": 2e-08, - "input_cost_per_token_cache_hit": 2e-08, + "cache_read_input_token_cost": 2.4e-08, + "input_cost_per_token_cache_hit": 2.4e-08, "supports_reasoning": true }, "novita/zai-org/autoglm-phone-9b-multilingual": { @@ -46639,14 +46675,16 @@ "input_cost_per_token": 6e-07, "output_cost_per_token": 2.5e-06, "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 100352, + "max_tokens": 100352, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 1.5e-07, + "supports_prompt_caching": true }, "novita/minimax/minimax-m2": { "litellm_provider": "novita", @@ -46662,7 +46700,8 @@ "supports_system_messages": true, "cache_read_input_token_cost": 3e-08, "input_cost_per_token_cache_hit": 3e-08, - "supports_reasoning": true + "supports_reasoning": true, + "supports_response_schema": true }, "novita/paddlepaddle/paddleocr-vl": { "litellm_provider": "novita", @@ -46700,7 +46739,9 @@ "max_tokens": 32768, "supports_vision": true, "supports_system_messages": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_function_calling": true, + "supports_tool_choice": true }, "novita/zai-org/glm-4.6v": { "litellm_provider": "novita", @@ -46765,7 +46806,8 @@ "supports_parallel_function_calling": true, "supports_tool_choice": true, "supports_system_messages": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_reasoning": true }, "novita/qwen/qwen3-next-80b-a3b-thinking": { "litellm_provider": "novita", @@ -46877,8 +46919,8 @@ "input_cost_per_token": 6e-07, "output_cost_per_token": 2.5e-06, "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 100352, + "max_tokens": 100352, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -46888,8 +46930,8 @@ "novita/qwen/qwen3-coder-480b-a35b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.3e-06, + "input_cost_per_token": 3.8e-07, + "output_cost_per_token": 1.55e-06, "max_input_tokens": 262144, "max_output_tokens": 65536, "max_tokens": 65536, @@ -46935,8 +46977,8 @@ "input_cost_per_token": 5.7e-07, "output_cost_per_token": 2.3e-06, "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 100352, + "max_tokens": 100352, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -46949,8 +46991,8 @@ "input_cost_per_token": 2.7e-07, "output_cost_per_token": 1.12e-06, "max_input_tokens": 163840, - "max_output_tokens": 163840, - "max_tokens": 163840, + "max_output_tokens": 65536, + "max_tokens": 65536, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -46997,7 +47039,8 @@ "max_input_tokens": 16384, "max_output_tokens": 16384, "max_tokens": 16384, - "supports_system_messages": true + "supports_system_messages": true, + "supports_response_schema": true }, "novita/google/gemma-3-12b-it": { "litellm_provider": "novita", @@ -47076,13 +47119,14 @@ "mode": "chat", "input_cost_per_token": 1.35e-07, "output_cost_per_token": 4e-07, - "max_input_tokens": 131072, - "max_output_tokens": 120000, - "max_tokens": 120000, + "max_input_tokens": 12288, + "max_output_tokens": 12288, + "max_tokens": 12288, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, - "supports_system_messages": true + "supports_system_messages": true, + "supports_response_schema": true }, "novita/qwen/qwen-2.5-72b-instruct": { "litellm_provider": "novita", @@ -47122,7 +47166,8 @@ "supports_parallel_function_calling": true, "supports_tool_choice": true, "supports_system_messages": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_response_schema": true }, "novita/deepseek/deepseek-r1-0528": { "litellm_provider": "novita", @@ -47162,7 +47207,8 @@ "max_input_tokens": 8192, "max_output_tokens": 8192, "max_tokens": 8192, - "supports_system_messages": true + "supports_system_messages": true, + "supports_response_schema": true }, "novita/microsoft/wizardlm-2-8x22b": { "litellm_provider": "novita", @@ -47172,7 +47218,8 @@ "max_input_tokens": 65535, "max_output_tokens": 8000, "max_tokens": 8000, - "supports_system_messages": true + "supports_system_messages": true, + "supports_response_schema": true }, "novita/deepseek/deepseek-r1-0528-qwen3-8b": { "litellm_provider": "novita", @@ -47219,7 +47266,8 @@ "max_output_tokens": 20000, "max_tokens": 20000, "supports_system_messages": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_response_schema": true }, "novita/meta-llama/llama-4-maverick-17b-128e-instruct-fp8": { "litellm_provider": "novita", @@ -47230,7 +47278,8 @@ "max_output_tokens": 8192, "max_tokens": 8192, "supports_vision": true, - "supports_system_messages": true + "supports_system_messages": true, + "supports_response_schema": true }, "novita/meta-llama/llama-4-scout-17b-16e-instruct": { "litellm_provider": "novita", @@ -47366,7 +47415,9 @@ "max_output_tokens": 20000, "max_tokens": 20000, "supports_system_messages": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_function_calling": true, + "supports_tool_choice": true }, "novita/google/gemma-3-27b-it": { "litellm_provider": "novita", @@ -47404,7 +47455,8 @@ "supports_parallel_function_calling": true, "supports_tool_choice": true, "supports_system_messages": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_response_schema": true }, "novita/Sao10K/L3-8B-Stheno-v3.2": { "litellm_provider": "novita", @@ -47472,7 +47524,9 @@ "supports_parallel_function_calling": true, "supports_tool_choice": true, "supports_system_messages": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 2.5e-08, + "supports_prompt_caching": true }, "novita/qwen/qwen3-vl-30b-a3b-instruct": { "litellm_provider": "novita", @@ -47593,10 +47647,12 @@ "input_cost_per_token": 3e-08, "output_cost_per_token": 3e-08, "max_input_tokens": 128000, - "max_output_tokens": 20000, - "max_tokens": 20000, + "max_output_tokens": 8192, + "max_tokens": 8192, "supports_system_messages": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_function_calling": true, + "supports_tool_choice": true }, "novita/qwen/qwen2.5-7b-instruct": { "litellm_provider": "novita", @@ -47604,8 +47660,8 @@ "input_cost_per_token": 7e-08, "output_cost_per_token": 7e-08, "max_input_tokens": 32000, - "max_output_tokens": 32000, - "max_tokens": 32000, + "max_output_tokens": 8192, + "max_tokens": 8192, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -50838,14 +50894,14 @@ "supports_embedding_image_input": true }, "fireworks_ai/accounts/fireworks/models/deepseek-v4-flash-0731": { - "cache_read_input_token_cost": 2.8e-08, - "input_cost_per_token": 1.4e-07, + "cache_read_input_token_cost": 7e-09, + "input_cost_per_token": 2.2e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 2.8e-07, + "output_cost_per_token": 6.6e-07, "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, "supports_reasoning": true, @@ -51152,5 +51208,2015 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "novita/zai-org/glm-5.3": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "novita", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://api.novita.ai/v3/openai/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/deepseek/deepseek-v4-pro-0813": { + "cache_read_input_token_cost": 1.3200000000000002e-07, + "input_cost_per_token": 1.32e-06, + "litellm_provider": "novita", + "max_input_tokens": 1048576, + "max_output_tokens": 393216, + "max_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 3.96e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/moonshotai/kimi-k3": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "novita", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/tencent/hy3": { + "cache_read_input_token_cost": 3.5e-08, + "input_cost_per_token": 1.4e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 5.8e-07, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/zai-org/glm-5.2": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "novita", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/moonshotai/kimi-k2.7-code": { + "cache_read_input_token_cost": 1.9e-07, + "input_cost_per_token": 9.499999999999999e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/deepseek/deepseek-v4-flash-vision-exp": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 4.4e-07, + "litellm_provider": "novita", + "max_input_tokens": 1048576, + "max_output_tokens": 393216, + "max_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 1.32e-06, + "source": "https://api.novita.ai/v3/openai/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/deepseek/deepseek-v4-flash-0731": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 4.4e-07, + "litellm_provider": "novita", + "max_input_tokens": 1048576, + "max_output_tokens": 393216, + "max_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 1.32e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/mindai/macaron-v1-venti": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 1.5e-06, + "litellm_provider": "novita", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4.5e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/minimax/minimax-m3": { + "cache_read_input_token_cost": 6e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "novita", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/deepseek/deepseek-v4-flash": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 1.4e-07, + "litellm_provider": "novita", + "max_input_tokens": 1048576, + "max_output_tokens": 393216, + "max_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 2.8e-07, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/deepseek/deepseek-v4-pro": { + "cache_read_input_token_cost": 1.35e-07, + "input_cost_per_token": 1.6000000000000001e-06, + "litellm_provider": "novita", + "max_input_tokens": 1048576, + "max_output_tokens": 393216, + "max_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 3.2000000000000003e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/inclusionai/ling-3.0-flash-fast": { + "cache_read_input_token_cost": 1.2e-08, + "input_cost_per_token": 6e-08, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1.8e-07, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/qwen/qwen3.8-max": { + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "novita", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 6e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/inclusionai/ling-3.0-flash": { + "cache_read_input_token_cost": 1.2e-08, + "input_cost_per_token": 6e-08, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1.8e-07, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/mindai/macaron-v1-tall": { + "cache_read_input_token_cost": 8e-08, + "input_cost_per_token": 4.5000000000000003e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 2.6e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/stepfun/step-3.7-flash": { + "cache_read_input_token_cost": 4e-08, + "input_cost_per_token": 2.0000000000000002e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 1.15e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/nvidia/nemotron-3-nano-30b-a3b": { + "input_cost_per_token": 5.0000000000000004e-08, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 2.0000000000000002e-07, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/baidu/cobuddy": { + "cache_read_input_token_cost": 7e-08, + "input_cost_per_token": 2.8e-07, + "litellm_provider": "novita", + "max_input_tokens": 131072, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.13e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/xiaomimimo/mimo-v2.5": { + "cache_read_input_token_cost": 3.4e-09, + "input_cost_per_token": 1.6800000000000002e-07, + "litellm_provider": "novita", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 3.3600000000000004e-07, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/qwen/qwen3.7-max": { + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "novita", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3.75e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/xiaomimimo/mimo-v2.5-pro": { + "cache_read_input_token_cost": 4.3e-09, + "input_cost_per_token": 5.22e-07, + "litellm_provider": "novita", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.044e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/qwen/qwen3.6-27b": { + "input_cost_per_token": 6e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3.6000000000000003e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/moonshotai/kimi-k2.6": { + "cache_read_input_token_cost": 1.6e-07, + "input_cost_per_token": 8.000000000000001e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.4e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/zai-org/glm-5.1": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.38e-06, + "litellm_provider": "novita", + "max_input_tokens": 204800, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/minimax/minimax-m2.7-highspeed": { + "cache_read_input_token_cost": 6e-08, + "input_cost_per_token": 6e-07, + "litellm_provider": "novita", + "max_input_tokens": 204800, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.4e-06, + "source": "https://api.novita.ai/v3/openai/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/zai-org/glm-5v-turbo": { + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 1.2e-06, + "litellm_provider": "novita", + "max_input_tokens": 204800, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://api.novita.ai/v3/openai/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/google/gemma-4-26b-a4b-it": { + "input_cost_per_token": 1.3e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4.0000000000000003e-07, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/google/gemma-4-31b-it": { + "input_cost_per_token": 1.4e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4.0000000000000003e-07, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/zai-org/glm-5-turbo": { + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 1.2e-06, + "litellm_provider": "novita", + "max_input_tokens": 202800, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://api.novita.ai/v3/openai/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/minimax/minimax-m2.7": { + "cache_read_input_token_cost": 6e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "novita", + "max_input_tokens": 204800, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/minimax/minimax-m2.5-highspeed": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 6e-07, + "litellm_provider": "novita", + "max_input_tokens": 204800, + "max_output_tokens": 131100, + "max_tokens": 131100, + "mode": "chat", + "output_cost_per_token": 2.4e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/qwen/qwen3.5-27b": { + "input_cost_per_token": 3e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 2.4e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/qwen/qwen3.5-122b-a10b": { + "input_cost_per_token": 4.0000000000000003e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3.2000000000000003e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/qwen/qwen3.5-35b-a3b": { + "input_cost_per_token": 2.5e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 2e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/qwen/qwen3.5-397b-a17b": { + "input_cost_per_token": 6e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3.6000000000000003e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/minimax/minimax-m2.5": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "novita", + "max_input_tokens": 204800, + "max_output_tokens": 131100, + "max_tokens": 131100, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/zai-org/glm-5": { + "cache_read_input_token_cost": 2.0000000000000002e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "novita", + "max_input_tokens": 202800, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 3.2000000000000003e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/qwen/qwen3-coder-next": { + "input_cost_per_token": 2.0000000000000002e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/deepseek/deepseek-ocr-2": { + "input_cost_per_token": 3e-08, + "litellm_provider": "novita", + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 3e-08, + "source": "https://novita.ai/pricing", + "supports_vision": true + }, + "novita/moonshotai/kimi-k2.5": { + "cache_read_input_token_cost": 1.0000000000000001e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/zai-org/glm-4.7-h": { + "cache_read_input_token_cost": 1.1e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "novita", + "max_input_tokens": 204800, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.2e-06, + "source": "https://api.novita.ai/v3/openai/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/zai-org/glm-4.7-flash": { + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 7e-08, + "litellm_provider": "novita", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.0000000000000003e-07, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/qwen/qwen3.6-35b-a3b": { + "input_cost_per_token": 2.48e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.4850000000000002e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/deepseek/deepseek_v3": { + "input_cost_per_token": 8.900000000000001e-07, + "litellm_provider": "novita", + "max_input_tokens": 64000, + "max_output_tokens": 16000, + "max_tokens": 16000, + "mode": "chat", + "output_cost_per_token": 8.900000000000001e-07, + "source": "https://api.novita.ai/v3/openai/models", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/deepseek/deepseek-r1": { + "input_cost_per_token": 4e-06, + "litellm_provider": "novita", + "max_input_tokens": 64000, + "max_output_tokens": 16000, + "max_tokens": 16000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://api.novita.ai/v3/openai/models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/deepseek/deepseek-v3/community": { + "input_cost_per_token": 8.900000000000001e-07, + "litellm_provider": "novita", + "max_input_tokens": 64000, + "max_output_tokens": 8000, + "max_tokens": 8000, + "mode": "chat", + "output_cost_per_token": 8.900000000000001e-07, + "source": "https://api.novita.ai/v3/openai/models", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/deepseek/deepseek-r1/community": { + "input_cost_per_token": 4e-06, + "litellm_provider": "novita", + "max_input_tokens": 64000, + "max_output_tokens": 8000, + "max_tokens": 8000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://api.novita.ai/v3/openai/models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/thudm/glm-4-32b-0414": { + "input_cost_per_token": 5.5e-07, + "litellm_provider": "novita", + "max_input_tokens": 32000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 1.66e-06, + "source": "https://api.novita.ai/v3/openai/models", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/meta-llama/llama-3.2-1b-instruct": { + "input_cost_per_token": 2e-08, + "litellm_provider": "novita", + "max_input_tokens": 131000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 2e-08, + "source": "https://api.novita.ai/v3/openai/models", + "supports_response_schema": true, + "supports_vision": false + }, + "wandb/deepseek-ai/DeepSeek-V4-Flash": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 2.8e-07, + "cache_read_input_token_cost": 7e-08, + "supports_prompt_caching": true, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": false, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/deepseek-ai/DeepSeek-V4-Flash-0731": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 1.3e-07, + "output_cost_per_token": 2.8e-07, + "cache_read_input_token_cost": 7e-08, + "supports_prompt_caching": true, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": false, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/deepseek-ai/DeepSeek-V4-Pro": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "input_cost_per_token": 1.15e-06, + "output_cost_per_token": 2.55e-06, + "cache_read_input_token_cost": 2e-07, + "supports_prompt_caching": true, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": false, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/google/gemma-4-31B-it": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3.4e-07, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": true, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/ibm-granite/granite-4.1-8b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 1e-07, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": false, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/JetBrains/Mellum2-12B-A2.5B-Instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 1e-07, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": false, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/meta-llama/Llama-3.1-70B-Instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "input_cost_per_token": 8e-07, + "output_cost_per_token": 8e-07, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": false, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/MiniMaxAI/MiniMax-M3": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 2.3e-07, + "output_cost_per_token": 9.6e-07, + "cache_read_input_token_cost": 5e-08, + "supports_prompt_caching": true, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": true, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/moonshotai/Kimi-K2.7-Code": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 7.1e-07, + "output_cost_per_token": 3.5e-06, + "cache_read_input_token_cost": 1.5e-07, + "supports_prompt_caching": true, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": true, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/moonshotai/Kimi-K2.6": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 6.5e-07, + "output_cost_per_token": 3.41e-06, + "cache_read_input_token_cost": 1.5e-07, + "supports_prompt_caching": true, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": true, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 2.5e-07, + "cache_read_input_token_cost": 5e-08, + "supports_prompt_caching": true, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": false, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 7.5e-07, + "output_cost_per_token": 2.75e-06, + "cache_read_input_token_cost": 1.5e-07, + "supports_prompt_caching": true, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": false, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/OpenPipe/Qwen3-14B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 2.2e-07, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": false, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/Qwen/Qwen3.8-27B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 3e-06, + "cache_read_input_token_cost": 1.5e-07, + "supports_prompt_caching": true, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": true, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/Qwen/Qwen3.6-35B-A3B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.25e-06, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": true, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/Qwen/Qwen3.6-27B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 6e-07, + "output_cost_per_token": 3.6e-06, + "cache_read_input_token_cost": 1.2e-07, + "supports_prompt_caching": true, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": true, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/Qwen/Qwen3.5-35B-A3B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.25e-06, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": true, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/Qwen/Qwen3-30B-A3B-Instruct-2507": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": false, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/zai-org/GLM-5.2": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 7.6e-07, + "output_cost_per_token": 2.42e-06, + "cache_read_input_token_cost": 1.4e-07, + "supports_prompt_caching": true, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": false, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "deepinfra/openai/gpt-oss-120b-Turbo": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/MiniMaxAI/MiniMax-M2.7": { + "max_tokens": 196608, + "max_input_tokens": 196608, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1e-06, + "cache_read_input_token_cost": 5e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3.8-27B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 3e-06, + "cache_read_input_token_cost": 4e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/google/gemma-4-31B-it-Ultra": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "input_cost_per_token": 2.7e-07, + "output_cost_per_token": 7.6e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/moonshotai/Kimi-K2.5": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 4.5e-07, + "output_cost_per_token": 2.25e-06, + "cache_read_input_token_cost": 7e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/zai-org/GLM-4.7-Flash": { + "max_tokens": 202752, + "max_input_tokens": 202752, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 1e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/zai-org/GLM-4.6": { + "max_tokens": 202752, + "max_input_tokens": 202752, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 1e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/anthropic/claude-opus-4-8": { + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/anthropic/claude-sonnet-4-6": { + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/google/gemini-3.5-flash": { + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 9e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/XiaomiMiMo/MiMo-V2.5": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 8e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3-Max": { + "max_tokens": 256000, + "max_input_tokens": 256000, + "input_cost_per_token": 1.2e-06, + "output_cost_per_token": 6e-06, + "cache_read_input_token_cost": 2.4e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/google/gemma-4-31B-it-turbo": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 9e-08, + "output_cost_per_token": 3.4e-07, + "cache_read_input_token_cost": 5e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/thinkingmachines/Inkling-Small": { + "max_tokens": 524288, + "max_input_tokens": 524288, + "input_cost_per_token": 4.5e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 1e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/meta-models/Muse-Glimmer-30B": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 4e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3-Max-Thinking": { + "max_tokens": 256000, + "max_input_tokens": 256000, + "input_cost_per_token": 1.2e-06, + "output_cost_per_token": 6e-06, + "cache_read_input_token_cost": 2.4e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3-VL-235B-A22B-Instruct": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 8.8e-07, + "cache_read_input_token_cost": 1.1e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3-VL-30B-A3B-Instruct": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3.5-27B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 2.6e-07, + "output_cost_per_token": 2.6e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3.6-35B-A3B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 9.5e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/nvidia/Nemotron-Content-Safety-3.5": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/anthropic/claude-opus-5": { + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/thinkingmachines/Inkling": { + "max_tokens": 524288, + "max_input_tokens": 524288, + "input_cost_per_token": 9.5e-07, + "output_cost_per_token": 4.05e-06, + "cache_read_input_token_cost": 1.6e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/moonshotai/Kimi-K2.6": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 7.5e-07, + "output_cost_per_token": 3.5e-06, + "cache_read_input_token_cost": 1.5e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/deepseek-ai/DeepSeek-V4-Pro-0813": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "input_cost_per_token": 1.3e-06, + "output_cost_per_token": 2.6e-06, + "cache_read_input_token_cost": 1e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3.7-Max": { + "max_tokens": 256000, + "max_input_tokens": 256000, + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 7.5e-06, + "cache_read_input_token_cost": 5e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/ByteDance/Seed-2.0-mini": { + "max_tokens": 256000, + "max_input_tokens": 256000, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 2e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3.8-2.4T-A95B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + "cache_read_input_token_cost": 2e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/MiniMaxAI/MiniMax-M3": { + "max_tokens": 524288, + "max_input_tokens": 524288, + "input_cost_per_token": 2.8e-07, + "output_cost_per_token": 1.1e-06, + "cache_read_input_token_cost": 5.6e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/google/gemini-3.1-flash-lite": { + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.5e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/google/gemini-3.7-flash": { + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "input_cost_per_token": 7.5e-07, + "output_cost_per_token": 3.75e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/inclusionAI/Ling-3.0-flash": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 1.8e-07, + "cache_read_input_token_cost": 1.2e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/stepfun-ai/Step-3.7-Flash": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 1.15e-06, + "cache_read_input_token_cost": 4e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3.5-35B-A3B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 1e-06, + "cache_read_input_token_cost": 5e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/ByteDance/Seed-1.8": { + "max_tokens": 256000, + "max_input_tokens": 256000, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 5e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/tencent/Hy3": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 5.8e-07, + "cache_read_input_token_cost": 3.5e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/ByteDance/Seed-2.0-code": { + "max_tokens": 256000, + "max_input_tokens": 256000, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 3e-06, + "cache_read_input_token_cost": 1e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/ByteDance/Seed-2.0-pro": { + "max_tokens": 256000, + "max_input_tokens": 256000, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 3e-06, + "cache_read_input_token_cost": 1e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/zai-org/GLM-5": { + "max_tokens": 202752, + "max_input_tokens": 202752, + "input_cost_per_token": 6e-07, + "output_cost_per_token": 2.08e-06, + "cache_read_input_token_cost": 1.2e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/nvidia/Nemotron-3-Nano-30B-A3B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 2e-07, + "cache_read_input_token_cost": 2.5e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/moonshotai/Kimi-K2.7-Code": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 6.8e-07, + "output_cost_per_token": 3.4e-06, + "cache_read_input_token_cost": 1.36e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/anthropic/claude-sonnet-5": { + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3.5-397B-A17B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 4.5e-07, + "output_cost_per_token": 3e-06, + "cache_read_input_token_cost": 2.2e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/deepseek-ai/DeepSeek-V4-Flash-0731": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "input_cost_per_token": 8e-08, + "output_cost_per_token": 1.8e-07, + "cache_read_input_token_cost": 1.6e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/google/gemma-4-E4B-it": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "input_cost_per_token": 2e-08, + "output_cost_per_token": 1e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/deepseek-ai/DeepSeek-V3.2": { + "max_tokens": 163840, + "max_input_tokens": 163840, + "input_cost_per_token": 2.6e-07, + "output_cost_per_token": 3.8e-07, + "cache_read_input_token_cost": 1.3e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3.8-Max": { + "max_tokens": 256000, + "max_input_tokens": 256000, + "input_cost_per_token": 1.65e-06, + "output_cost_per_token": 4.951e-06, + "cache_read_input_token_cost": 2.06e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/anthropic/claude-fable-5": { + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 5e-05, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 2.2e-06, + "cache_read_input_token_cost": 1e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3.5-122B-A10B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 2.9e-07, + "output_cost_per_token": 2.4e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/zai-org/GLM-5.1": { + "max_tokens": 202752, + "max_input_tokens": 202752, + "input_cost_per_token": 1.05e-06, + "output_cost_per_token": 3.5e-06, + "cache_read_input_token_cost": 2.05e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/deepseek-ai/DeepSeek-V4-Pro": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "input_cost_per_token": 1.3e-06, + "output_cost_per_token": 2.6e-06, + "cache_read_input_token_cost": 1e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 8.5e-08, + "output_cost_per_token": 4e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/zai-org/GLM-5.2": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "input_cost_per_token": 7.5e-07, + "output_cost_per_token": 2.4e-06, + "cache_read_input_token_cost": 1.4e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/moonshotai/Kimi-K3": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "input_cost_per_token": 2.85e-06, + "output_cost_per_token": 1.425e-05, + "cache_read_input_token_cost": 2.85e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/anthropic/claude-opus-4-7": { + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3.6-27B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 3.2e-07, + "output_cost_per_token": 3.2e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/google/gemma-4-26B-A4B-it": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 3.4e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/google/gemini-3.1-pro": { + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1.2e-05, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/XiaomiMiMo/MiMo-V2.5-Pro": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 3e-06, + "cache_read_input_token_cost": 2e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/anthropic/claude-haiku-4-5": { + "max_tokens": 200000, + "max_input_tokens": 200000, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 5e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/deepseek-ai/DeepSeek-V4-Flash": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "input_cost_per_token": 9e-08, + "output_cost_per_token": 1.8e-07, + "cache_read_input_token_cost": 1.8e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/openai/gpt-oss-120b-Ultra": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 9.5e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3.5-9B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 1.5e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/MiniMaxAI/MiniMax-M2.7-Turbo": { + "max_tokens": 196608, + "max_input_tokens": 196608, + "input_cost_per_token": 3.8e-07, + "output_cost_per_token": 1.7e-06, + "cache_read_input_token_cost": 7e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/zai-org/GLM-4.7": { + "max_tokens": 202752, + "max_input_tokens": 202752, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 1.75e-06, + "cache_read_input_token_cost": 8e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/google/gemma-4-31B-it": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 1.3e-07, + "output_cost_per_token": 3.8e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 87b4b08ea62..e133f31cdac 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -16023,12 +16023,13 @@ "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, - "input_cost_per_token": 8e-08, - "output_cost_per_token": 9e-08, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 4e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/NousResearch/Hermes-3-Llama-3.1-405B": { "max_tokens": 131072, @@ -16045,11 +16046,12 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 3e-07, - "output_cost_per_token": 3e-07, + "input_cost_per_token": 7e-07, + "output_cost_per_token": 7e-07, "litellm_provider": "deepinfra", "mode": "chat", - "supports_tool_choice": false + "supports_tool_choice": false, + "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/QwQ-32B": { "max_tokens": 131072, @@ -16066,12 +16068,13 @@ "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, - "input_cost_per_token": 1.2e-07, - "output_cost_per_token": 3.9e-07, + "input_cost_per_token": 3.6e-07, + "output_cost_per_token": 4e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen2.5-7B-Instruct": { "max_tokens": 32768, @@ -16099,12 +16102,13 @@ "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, - "input_cost_per_token": 6e-08, + "input_cost_per_token": 1.2e-07, "output_cost_per_token": 2.4e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-235B-A22B": { "max_tokens": 40960, @@ -16122,11 +16126,12 @@ "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 9e-08, - "output_cost_per_token": 6e-07, + "output_cost_per_token": 5.5e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-235B-A22B-Thinking-2507": { "max_tokens": 262144, @@ -16143,23 +16148,25 @@ "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, - "input_cost_per_token": 8e-08, - "output_cost_per_token": 2.9e-07, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 5e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-32B": { "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, - "input_cost_per_token": 1e-07, + "input_cost_per_token": 8e-08, "output_cost_per_token": 2.8e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-Coder-480B-A35B-Instruct": { "max_tokens": 262144, @@ -16176,23 +16183,27 @@ "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, - "input_cost_per_token": 2.9e-07, - "output_cost_per_token": 1.2e-06, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1e-06, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "cache_read_input_token_cost": 1e-07, + "supports_prompt_caching": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-Next-80B-A3B-Instruct": { "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, - "input_cost_per_token": 1.4e-07, - "output_cost_per_token": 1.4e-06, + "input_cost_per_token": 9e-08, + "output_cost_per_token": 1.1e-06, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-Next-80B-A3B-Thinking": { "max_tokens": 262144, @@ -16219,11 +16230,12 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 6.5e-07, - "output_cost_per_token": 7.5e-07, + "input_cost_per_token": 8.5e-07, + "output_cost_per_token": 8.5e-07, "litellm_provider": "deepinfra", "mode": "chat", - "supports_tool_choice": false + "supports_tool_choice": false, + "source": "https://deepinfra.com/pricing" }, "deepinfra/Sao10K/L3.3-70B-Euryale-v2.3": { "max_tokens": 131072, @@ -16349,36 +16361,41 @@ "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, - "input_cost_per_token": 3.8e-07, + "input_cost_per_token": 3.2e-07, "output_cost_per_token": 8.9e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V3-0324": { "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, - "input_cost_per_token": 2.5e-07, - "output_cost_per_token": 8.8e-07, + "input_cost_per_token": 2.4e-07, + "output_cost_per_token": 9e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "cache_read_input_token_cost": 1.35e-07, + "supports_prompt_caching": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V3.1": { "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, - "input_cost_per_token": 2.7e-07, - "output_cost_per_token": 1e-06, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 9.5e-07, "cache_read_input_token_cost": 2.16e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, "supports_reasoning": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V3.1-Terminus": { "max_tokens": 163840, @@ -16432,33 +16449,36 @@ "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 5e-08, - "output_cost_per_token": 1e-07, + "output_cost_per_token": 1.5e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-3-27b-it": { "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 9e-08, + "input_cost_per_token": 8e-08, "output_cost_per_token": 1.6e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-3-4b-it": { "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 4e-08, - "output_cost_per_token": 8e-08, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 1e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Llama-3.2-11B-Vision-Instruct": { "max_tokens": 131072, @@ -16496,34 +16516,37 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 1.3e-07, - "output_cost_per_token": 3.9e-07, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3.2e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_function_calling": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8": { "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "input_cost_per_token": 1.5e-07, - "output_cost_per_token": 6e-07, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 8e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "max_tokens": 327680, "max_input_tokens": 327680, "max_output_tokens": 327680, - "input_cost_per_token": 8e-08, + "input_cost_per_token": 1e-07, "output_cost_per_token": 3e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Llama-Guard-3-8B": { "max_tokens": 131072, @@ -16571,12 +16594,13 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 1e-07, - "output_cost_per_token": 2.8e-07, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 4e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Meta-Llama-3.1-8B-Instruct": { "max_tokens": 131072, @@ -16594,11 +16618,12 @@ "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-08, - "output_cost_per_token": 3e-08, + "output_cost_per_token": 4e-08, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/microsoft/WizardLM-2-8x22B": { "max_tokens": 65536, @@ -16625,12 +16650,13 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 2e-08, - "output_cost_per_token": 4e-08, + "input_cost_per_token": 1.9e-08, + "output_cost_per_token": 3e-08, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/mistralai/Mistral-Small-24B-Instruct-2501": { "max_tokens": 32768, @@ -16712,14 +16738,16 @@ }, "deepinfra/nvidia/NVIDIA-Nemotron-3.5-Lightning": { "max_input_tokens": 262144, - "input_cost_per_token": 5e-08, + "input_cost_per_token": 8e-08, "output_cost_per_token": 2e-07, "litellm_provider": "deepinfra", "mode": "chat", - "source": "https://deepinfra.com/nvidia/NVIDIA-Nemotron-3.5-Lightning", + "source": "https://deepinfra.com/pricing", "supports_tool_choice": true, "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 4e-08, + "supports_prompt_caching": true }, "deepinfra/nvidia/NVIDIA-Nemotron-Nano-9B-v2": { "max_tokens": 131072, @@ -16736,23 +16764,25 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 5e-08, - "output_cost_per_token": 4.5e-07, + "input_cost_per_token": 3.7e-08, + "output_cost_per_token": 1.7e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/openai/gpt-oss-20b": { "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 4e-08, - "output_cost_per_token": 1.5e-07, + "input_cost_per_token": 3e-08, + "output_cost_per_token": 1.4e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-4.5": { "max_tokens": 131072, @@ -20237,7 +20267,7 @@ "supports_image_size": false }, "gemini-2.5-flash-preview-09-2025": { - "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost": 3e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, "litellm_provider": "vertex_ai-language-models", @@ -20247,7 +20277,7 @@ "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, - "source": "https://developers.googleblog.com/en/continuing-to-bring-you-our-latest-models-with-an-improved-gemini-2-5-flash-and-flash-lite-release/", + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -20373,7 +20403,7 @@ }, "gemini-2.5-flash-lite-preview-06-17": { "deprecation_date": "2025-11-18", - "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost": 1e-08, "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-language-models", @@ -20383,7 +20413,7 @@ "mode": "chat", "output_cost_per_reasoning_token": 4e-07, "output_cost_per_token": 4e-07, - "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -21953,7 +21983,7 @@ "supports_image_size": false }, "gemini/gemini-2.5-flash-preview-09-2025": { - "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost": 3e-08, "deprecation_date": "2026-02-17", "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, @@ -21965,7 +21995,7 @@ "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, "rpm": 15, - "source": "https://developers.googleblog.com/en/continuing-to-bring-you-our-latest-models-with-an-improved-gemini-2-5-flash-and-flash-lite-release/", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -22001,7 +22031,7 @@ "supports_image_size": false }, "gemini/gemini-flash-latest": { - "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost": 3e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, "litellm_provider": "gemini", @@ -22012,7 +22042,7 @@ "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, "rpm": 15, - "source": "https://developers.googleblog.com/en/continuing-to-bring-you-our-latest-models-with-an-improved-gemini-2-5-flash-and-flash-lite-release/", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -22047,7 +22077,7 @@ } }, "gemini/gemini-flash-lite-latest": { - "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost": 1e-08, "input_cost_per_audio_token": 3e-07, "input_cost_per_token": 1e-07, "litellm_provider": "gemini", @@ -22058,7 +22088,7 @@ "output_cost_per_reasoning_token": 4e-07, "output_cost_per_token": 4e-07, "rpm": 15, - "source": "https://developers.googleblog.com/en/continuing-to-bring-you-our-latest-models-with-an-improved-gemini-2-5-flash-and-flash-lite-release/", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -22094,7 +22124,7 @@ }, "gemini/gemini-2.5-flash-lite-preview-06-17": { "deprecation_date": "2025-11-18", - "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost": 1e-08, "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 1e-07, "litellm_provider": "gemini", @@ -22105,7 +22135,7 @@ "output_cost_per_reasoning_token": 4e-07, "output_cost_per_token": 4e-07, "rpm": 15, - "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-lite", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -42511,19 +42541,21 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 0.015, - "output_cost_per_token": 0.06, + "input_cost_per_token": 3e-08, + "output_cost_per_token": 1.7e-07, "litellm_provider": "wandb", - "mode": "chat" + "mode": "chat", + "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/openai/gpt-oss-20b": { "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 0.005, - "output_cost_per_token": 0.02, + "input_cost_per_token": 3e-08, + "output_cost_per_token": 1.3e-07, "litellm_provider": "wandb", - "mode": "chat" + "mode": "chat", + "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/zai-org/GLM-4.5": { "max_tokens": 131072, @@ -42547,10 +42579,11 @@ "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, - "input_cost_per_token": 0.1, - "output_cost_per_token": 0.15, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 1.5e-06, "litellm_provider": "wandb", - "mode": "chat" + "mode": "chat", + "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3-235B-A22B-Thinking-2507": { "max_tokens": 262144, @@ -42602,19 +42635,21 @@ "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, - "input_cost_per_token": 0.022, - "output_cost_per_token": 0.022, + "input_cost_per_token": 2.2e-07, + "output_cost_per_token": 2.2e-07, "litellm_provider": "wandb", - "mode": "chat" + "mode": "chat", + "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/deepseek-ai/DeepSeek-V3.1": { "max_tokens": 128000, - "max_input_tokens": 128000, + "max_input_tokens": 161000, "max_output_tokens": 128000, - "input_cost_per_token": 0.055, - "output_cost_per_token": 0.165, + "input_cost_per_token": 5.5e-07, + "output_cost_per_token": 1.65e-06, "litellm_provider": "wandb", - "mode": "chat" + "mode": "chat", + "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/deepseek-ai/DeepSeek-R1-0528": { "max_tokens": 161000, @@ -42638,10 +42673,11 @@ "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, - "input_cost_per_token": 0.071, - "output_cost_per_token": 0.071, + "input_cost_per_token": 7.1e-07, + "output_cost_per_token": 7.1e-07, "litellm_provider": "wandb", - "mode": "chat" + "mode": "chat", + "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "max_tokens": 64000, @@ -46608,8 +46644,8 @@ "novita/xiaomimimo/mimo-v2-flash": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1e-07, - "output_cost_per_token": 3e-07, + "input_cost_per_token": 1.1e-07, + "output_cost_per_token": 3.3e-07, "max_input_tokens": 262144, "max_output_tokens": 32000, "max_tokens": 32000, @@ -46618,8 +46654,8 @@ "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, - "cache_read_input_token_cost": 2e-08, - "input_cost_per_token_cache_hit": 2e-08, + "cache_read_input_token_cost": 2.4e-08, + "input_cost_per_token_cache_hit": 2.4e-08, "supports_reasoning": true }, "novita/zai-org/autoglm-phone-9b-multilingual": { @@ -46639,14 +46675,16 @@ "input_cost_per_token": 6e-07, "output_cost_per_token": 2.5e-06, "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 100352, + "max_tokens": 100352, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 1.5e-07, + "supports_prompt_caching": true }, "novita/minimax/minimax-m2": { "litellm_provider": "novita", @@ -46662,7 +46700,8 @@ "supports_system_messages": true, "cache_read_input_token_cost": 3e-08, "input_cost_per_token_cache_hit": 3e-08, - "supports_reasoning": true + "supports_reasoning": true, + "supports_response_schema": true }, "novita/paddlepaddle/paddleocr-vl": { "litellm_provider": "novita", @@ -46700,7 +46739,9 @@ "max_tokens": 32768, "supports_vision": true, "supports_system_messages": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_function_calling": true, + "supports_tool_choice": true }, "novita/zai-org/glm-4.6v": { "litellm_provider": "novita", @@ -46765,7 +46806,8 @@ "supports_parallel_function_calling": true, "supports_tool_choice": true, "supports_system_messages": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_reasoning": true }, "novita/qwen/qwen3-next-80b-a3b-thinking": { "litellm_provider": "novita", @@ -46877,8 +46919,8 @@ "input_cost_per_token": 6e-07, "output_cost_per_token": 2.5e-06, "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 100352, + "max_tokens": 100352, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -46888,8 +46930,8 @@ "novita/qwen/qwen3-coder-480b-a35b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.3e-06, + "input_cost_per_token": 3.8e-07, + "output_cost_per_token": 1.55e-06, "max_input_tokens": 262144, "max_output_tokens": 65536, "max_tokens": 65536, @@ -46935,8 +46977,8 @@ "input_cost_per_token": 5.7e-07, "output_cost_per_token": 2.3e-06, "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 100352, + "max_tokens": 100352, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -46949,8 +46991,8 @@ "input_cost_per_token": 2.7e-07, "output_cost_per_token": 1.12e-06, "max_input_tokens": 163840, - "max_output_tokens": 163840, - "max_tokens": 163840, + "max_output_tokens": 65536, + "max_tokens": 65536, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -46997,7 +47039,8 @@ "max_input_tokens": 16384, "max_output_tokens": 16384, "max_tokens": 16384, - "supports_system_messages": true + "supports_system_messages": true, + "supports_response_schema": true }, "novita/google/gemma-3-12b-it": { "litellm_provider": "novita", @@ -47076,13 +47119,14 @@ "mode": "chat", "input_cost_per_token": 1.35e-07, "output_cost_per_token": 4e-07, - "max_input_tokens": 131072, - "max_output_tokens": 120000, - "max_tokens": 120000, + "max_input_tokens": 12288, + "max_output_tokens": 12288, + "max_tokens": 12288, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, - "supports_system_messages": true + "supports_system_messages": true, + "supports_response_schema": true }, "novita/qwen/qwen-2.5-72b-instruct": { "litellm_provider": "novita", @@ -47122,7 +47166,8 @@ "supports_parallel_function_calling": true, "supports_tool_choice": true, "supports_system_messages": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_response_schema": true }, "novita/deepseek/deepseek-r1-0528": { "litellm_provider": "novita", @@ -47162,7 +47207,8 @@ "max_input_tokens": 8192, "max_output_tokens": 8192, "max_tokens": 8192, - "supports_system_messages": true + "supports_system_messages": true, + "supports_response_schema": true }, "novita/microsoft/wizardlm-2-8x22b": { "litellm_provider": "novita", @@ -47172,7 +47218,8 @@ "max_input_tokens": 65535, "max_output_tokens": 8000, "max_tokens": 8000, - "supports_system_messages": true + "supports_system_messages": true, + "supports_response_schema": true }, "novita/deepseek/deepseek-r1-0528-qwen3-8b": { "litellm_provider": "novita", @@ -47219,7 +47266,8 @@ "max_output_tokens": 20000, "max_tokens": 20000, "supports_system_messages": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_response_schema": true }, "novita/meta-llama/llama-4-maverick-17b-128e-instruct-fp8": { "litellm_provider": "novita", @@ -47230,7 +47278,8 @@ "max_output_tokens": 8192, "max_tokens": 8192, "supports_vision": true, - "supports_system_messages": true + "supports_system_messages": true, + "supports_response_schema": true }, "novita/meta-llama/llama-4-scout-17b-16e-instruct": { "litellm_provider": "novita", @@ -47366,7 +47415,9 @@ "max_output_tokens": 20000, "max_tokens": 20000, "supports_system_messages": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_function_calling": true, + "supports_tool_choice": true }, "novita/google/gemma-3-27b-it": { "litellm_provider": "novita", @@ -47404,7 +47455,8 @@ "supports_parallel_function_calling": true, "supports_tool_choice": true, "supports_system_messages": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_response_schema": true }, "novita/Sao10K/L3-8B-Stheno-v3.2": { "litellm_provider": "novita", @@ -47472,7 +47524,9 @@ "supports_parallel_function_calling": true, "supports_tool_choice": true, "supports_system_messages": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 2.5e-08, + "supports_prompt_caching": true }, "novita/qwen/qwen3-vl-30b-a3b-instruct": { "litellm_provider": "novita", @@ -47593,10 +47647,12 @@ "input_cost_per_token": 3e-08, "output_cost_per_token": 3e-08, "max_input_tokens": 128000, - "max_output_tokens": 20000, - "max_tokens": 20000, + "max_output_tokens": 8192, + "max_tokens": 8192, "supports_system_messages": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_function_calling": true, + "supports_tool_choice": true }, "novita/qwen/qwen2.5-7b-instruct": { "litellm_provider": "novita", @@ -47604,8 +47660,8 @@ "input_cost_per_token": 7e-08, "output_cost_per_token": 7e-08, "max_input_tokens": 32000, - "max_output_tokens": 32000, - "max_tokens": 32000, + "max_output_tokens": 8192, + "max_tokens": 8192, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -50838,14 +50894,14 @@ "supports_embedding_image_input": true }, "fireworks_ai/accounts/fireworks/models/deepseek-v4-flash-0731": { - "cache_read_input_token_cost": 2.8e-08, - "input_cost_per_token": 1.4e-07, + "cache_read_input_token_cost": 7e-09, + "input_cost_per_token": 2.2e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 2.8e-07, + "output_cost_per_token": 6.6e-07, "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, "supports_reasoning": true, @@ -51152,5 +51208,2015 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "novita/zai-org/glm-5.3": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "novita", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://api.novita.ai/v3/openai/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/deepseek/deepseek-v4-pro-0813": { + "cache_read_input_token_cost": 1.3200000000000002e-07, + "input_cost_per_token": 1.32e-06, + "litellm_provider": "novita", + "max_input_tokens": 1048576, + "max_output_tokens": 393216, + "max_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 3.96e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/moonshotai/kimi-k3": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "novita", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/tencent/hy3": { + "cache_read_input_token_cost": 3.5e-08, + "input_cost_per_token": 1.4e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 5.8e-07, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/zai-org/glm-5.2": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "novita", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/moonshotai/kimi-k2.7-code": { + "cache_read_input_token_cost": 1.9e-07, + "input_cost_per_token": 9.499999999999999e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/deepseek/deepseek-v4-flash-vision-exp": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 4.4e-07, + "litellm_provider": "novita", + "max_input_tokens": 1048576, + "max_output_tokens": 393216, + "max_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 1.32e-06, + "source": "https://api.novita.ai/v3/openai/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/deepseek/deepseek-v4-flash-0731": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 4.4e-07, + "litellm_provider": "novita", + "max_input_tokens": 1048576, + "max_output_tokens": 393216, + "max_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 1.32e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/mindai/macaron-v1-venti": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 1.5e-06, + "litellm_provider": "novita", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4.5e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/minimax/minimax-m3": { + "cache_read_input_token_cost": 6e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "novita", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/deepseek/deepseek-v4-flash": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 1.4e-07, + "litellm_provider": "novita", + "max_input_tokens": 1048576, + "max_output_tokens": 393216, + "max_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 2.8e-07, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/deepseek/deepseek-v4-pro": { + "cache_read_input_token_cost": 1.35e-07, + "input_cost_per_token": 1.6000000000000001e-06, + "litellm_provider": "novita", + "max_input_tokens": 1048576, + "max_output_tokens": 393216, + "max_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 3.2000000000000003e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/inclusionai/ling-3.0-flash-fast": { + "cache_read_input_token_cost": 1.2e-08, + "input_cost_per_token": 6e-08, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1.8e-07, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/qwen/qwen3.8-max": { + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "novita", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 6e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/inclusionai/ling-3.0-flash": { + "cache_read_input_token_cost": 1.2e-08, + "input_cost_per_token": 6e-08, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1.8e-07, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/mindai/macaron-v1-tall": { + "cache_read_input_token_cost": 8e-08, + "input_cost_per_token": 4.5000000000000003e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 2.6e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/stepfun/step-3.7-flash": { + "cache_read_input_token_cost": 4e-08, + "input_cost_per_token": 2.0000000000000002e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 1.15e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/nvidia/nemotron-3-nano-30b-a3b": { + "input_cost_per_token": 5.0000000000000004e-08, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 2.0000000000000002e-07, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/baidu/cobuddy": { + "cache_read_input_token_cost": 7e-08, + "input_cost_per_token": 2.8e-07, + "litellm_provider": "novita", + "max_input_tokens": 131072, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.13e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/xiaomimimo/mimo-v2.5": { + "cache_read_input_token_cost": 3.4e-09, + "input_cost_per_token": 1.6800000000000002e-07, + "litellm_provider": "novita", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 3.3600000000000004e-07, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/qwen/qwen3.7-max": { + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "novita", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3.75e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/xiaomimimo/mimo-v2.5-pro": { + "cache_read_input_token_cost": 4.3e-09, + "input_cost_per_token": 5.22e-07, + "litellm_provider": "novita", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.044e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/qwen/qwen3.6-27b": { + "input_cost_per_token": 6e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3.6000000000000003e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/moonshotai/kimi-k2.6": { + "cache_read_input_token_cost": 1.6e-07, + "input_cost_per_token": 8.000000000000001e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.4e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/zai-org/glm-5.1": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.38e-06, + "litellm_provider": "novita", + "max_input_tokens": 204800, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/minimax/minimax-m2.7-highspeed": { + "cache_read_input_token_cost": 6e-08, + "input_cost_per_token": 6e-07, + "litellm_provider": "novita", + "max_input_tokens": 204800, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.4e-06, + "source": "https://api.novita.ai/v3/openai/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/zai-org/glm-5v-turbo": { + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 1.2e-06, + "litellm_provider": "novita", + "max_input_tokens": 204800, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://api.novita.ai/v3/openai/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/google/gemma-4-26b-a4b-it": { + "input_cost_per_token": 1.3e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4.0000000000000003e-07, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/google/gemma-4-31b-it": { + "input_cost_per_token": 1.4e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4.0000000000000003e-07, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/zai-org/glm-5-turbo": { + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 1.2e-06, + "litellm_provider": "novita", + "max_input_tokens": 202800, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://api.novita.ai/v3/openai/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/minimax/minimax-m2.7": { + "cache_read_input_token_cost": 6e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "novita", + "max_input_tokens": 204800, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/minimax/minimax-m2.5-highspeed": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 6e-07, + "litellm_provider": "novita", + "max_input_tokens": 204800, + "max_output_tokens": 131100, + "max_tokens": 131100, + "mode": "chat", + "output_cost_per_token": 2.4e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/qwen/qwen3.5-27b": { + "input_cost_per_token": 3e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 2.4e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/qwen/qwen3.5-122b-a10b": { + "input_cost_per_token": 4.0000000000000003e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3.2000000000000003e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/qwen/qwen3.5-35b-a3b": { + "input_cost_per_token": 2.5e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 2e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/qwen/qwen3.5-397b-a17b": { + "input_cost_per_token": 6e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3.6000000000000003e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/minimax/minimax-m2.5": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "novita", + "max_input_tokens": 204800, + "max_output_tokens": 131100, + "max_tokens": 131100, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/zai-org/glm-5": { + "cache_read_input_token_cost": 2.0000000000000002e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "novita", + "max_input_tokens": 202800, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 3.2000000000000003e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/qwen/qwen3-coder-next": { + "input_cost_per_token": 2.0000000000000002e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/deepseek/deepseek-ocr-2": { + "input_cost_per_token": 3e-08, + "litellm_provider": "novita", + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 3e-08, + "source": "https://novita.ai/pricing", + "supports_vision": true + }, + "novita/moonshotai/kimi-k2.5": { + "cache_read_input_token_cost": 1.0000000000000001e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/zai-org/glm-4.7-h": { + "cache_read_input_token_cost": 1.1e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "novita", + "max_input_tokens": 204800, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.2e-06, + "source": "https://api.novita.ai/v3/openai/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/zai-org/glm-4.7-flash": { + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 7e-08, + "litellm_provider": "novita", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.0000000000000003e-07, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/qwen/qwen3.6-35b-a3b": { + "input_cost_per_token": 2.48e-07, + "litellm_provider": "novita", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.4850000000000002e-06, + "source": "https://novita.ai/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "novita/deepseek/deepseek_v3": { + "input_cost_per_token": 8.900000000000001e-07, + "litellm_provider": "novita", + "max_input_tokens": 64000, + "max_output_tokens": 16000, + "max_tokens": 16000, + "mode": "chat", + "output_cost_per_token": 8.900000000000001e-07, + "source": "https://api.novita.ai/v3/openai/models", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/deepseek/deepseek-r1": { + "input_cost_per_token": 4e-06, + "litellm_provider": "novita", + "max_input_tokens": 64000, + "max_output_tokens": 16000, + "max_tokens": 16000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://api.novita.ai/v3/openai/models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/deepseek/deepseek-v3/community": { + "input_cost_per_token": 8.900000000000001e-07, + "litellm_provider": "novita", + "max_input_tokens": 64000, + "max_output_tokens": 8000, + "max_tokens": 8000, + "mode": "chat", + "output_cost_per_token": 8.900000000000001e-07, + "source": "https://api.novita.ai/v3/openai/models", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/deepseek/deepseek-r1/community": { + "input_cost_per_token": 4e-06, + "litellm_provider": "novita", + "max_input_tokens": 64000, + "max_output_tokens": 8000, + "max_tokens": 8000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://api.novita.ai/v3/openai/models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/thudm/glm-4-32b-0414": { + "input_cost_per_token": 5.5e-07, + "litellm_provider": "novita", + "max_input_tokens": 32000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 1.66e-06, + "source": "https://api.novita.ai/v3/openai/models", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "novita/meta-llama/llama-3.2-1b-instruct": { + "input_cost_per_token": 2e-08, + "litellm_provider": "novita", + "max_input_tokens": 131000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 2e-08, + "source": "https://api.novita.ai/v3/openai/models", + "supports_response_schema": true, + "supports_vision": false + }, + "wandb/deepseek-ai/DeepSeek-V4-Flash": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 2.8e-07, + "cache_read_input_token_cost": 7e-08, + "supports_prompt_caching": true, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": false, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/deepseek-ai/DeepSeek-V4-Flash-0731": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 1.3e-07, + "output_cost_per_token": 2.8e-07, + "cache_read_input_token_cost": 7e-08, + "supports_prompt_caching": true, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": false, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/deepseek-ai/DeepSeek-V4-Pro": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "input_cost_per_token": 1.15e-06, + "output_cost_per_token": 2.55e-06, + "cache_read_input_token_cost": 2e-07, + "supports_prompt_caching": true, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": false, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/google/gemma-4-31B-it": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3.4e-07, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": true, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/ibm-granite/granite-4.1-8b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 1e-07, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": false, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/JetBrains/Mellum2-12B-A2.5B-Instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 1e-07, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": false, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/meta-llama/Llama-3.1-70B-Instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "input_cost_per_token": 8e-07, + "output_cost_per_token": 8e-07, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": false, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/MiniMaxAI/MiniMax-M3": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 2.3e-07, + "output_cost_per_token": 9.6e-07, + "cache_read_input_token_cost": 5e-08, + "supports_prompt_caching": true, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": true, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/moonshotai/Kimi-K2.7-Code": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 7.1e-07, + "output_cost_per_token": 3.5e-06, + "cache_read_input_token_cost": 1.5e-07, + "supports_prompt_caching": true, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": true, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/moonshotai/Kimi-K2.6": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 6.5e-07, + "output_cost_per_token": 3.41e-06, + "cache_read_input_token_cost": 1.5e-07, + "supports_prompt_caching": true, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": true, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 2.5e-07, + "cache_read_input_token_cost": 5e-08, + "supports_prompt_caching": true, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": false, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 7.5e-07, + "output_cost_per_token": 2.75e-06, + "cache_read_input_token_cost": 1.5e-07, + "supports_prompt_caching": true, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": false, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/OpenPipe/Qwen3-14B-Instruct": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 2.2e-07, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": false, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/Qwen/Qwen3.8-27B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 3e-06, + "cache_read_input_token_cost": 1.5e-07, + "supports_prompt_caching": true, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": true, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/Qwen/Qwen3.6-35B-A3B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.25e-06, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": true, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/Qwen/Qwen3.6-27B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 6e-07, + "output_cost_per_token": 3.6e-06, + "cache_read_input_token_cost": 1.2e-07, + "supports_prompt_caching": true, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": true, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/Qwen/Qwen3.5-35B-A3B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.25e-06, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": true, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/Qwen/Qwen3-30B-A3B-Instruct-2507": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3e-07, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": false, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/zai-org/GLM-5.2": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 7.6e-07, + "output_cost_per_token": 2.42e-06, + "cache_read_input_token_cost": 1.4e-07, + "supports_prompt_caching": true, + "litellm_provider": "wandb", + "mode": "chat", + "supports_vision": false, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "deepinfra/openai/gpt-oss-120b-Turbo": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/MiniMaxAI/MiniMax-M2.7": { + "max_tokens": 196608, + "max_input_tokens": 196608, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1e-06, + "cache_read_input_token_cost": 5e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3.8-27B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 3e-06, + "cache_read_input_token_cost": 4e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/google/gemma-4-31B-it-Ultra": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "input_cost_per_token": 2.7e-07, + "output_cost_per_token": 7.6e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/moonshotai/Kimi-K2.5": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 4.5e-07, + "output_cost_per_token": 2.25e-06, + "cache_read_input_token_cost": 7e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/zai-org/GLM-4.7-Flash": { + "max_tokens": 202752, + "max_input_tokens": 202752, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 1e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/zai-org/GLM-4.6": { + "max_tokens": 202752, + "max_input_tokens": 202752, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 1e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/anthropic/claude-opus-4-8": { + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/anthropic/claude-sonnet-4-6": { + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/google/gemini-3.5-flash": { + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 9e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/XiaomiMiMo/MiMo-V2.5": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 8e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3-Max": { + "max_tokens": 256000, + "max_input_tokens": 256000, + "input_cost_per_token": 1.2e-06, + "output_cost_per_token": 6e-06, + "cache_read_input_token_cost": 2.4e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/google/gemma-4-31B-it-turbo": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 9e-08, + "output_cost_per_token": 3.4e-07, + "cache_read_input_token_cost": 5e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/thinkingmachines/Inkling-Small": { + "max_tokens": 524288, + "max_input_tokens": 524288, + "input_cost_per_token": 4.5e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 1e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/meta-models/Muse-Glimmer-30B": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 4e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3-Max-Thinking": { + "max_tokens": 256000, + "max_input_tokens": 256000, + "input_cost_per_token": 1.2e-06, + "output_cost_per_token": 6e-06, + "cache_read_input_token_cost": 2.4e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3-VL-235B-A22B-Instruct": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 8.8e-07, + "cache_read_input_token_cost": 1.1e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3-VL-30B-A3B-Instruct": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3.5-27B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 2.6e-07, + "output_cost_per_token": 2.6e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3.6-35B-A3B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 9.5e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/nvidia/Nemotron-Content-Safety-3.5": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/anthropic/claude-opus-5": { + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/thinkingmachines/Inkling": { + "max_tokens": 524288, + "max_input_tokens": 524288, + "input_cost_per_token": 9.5e-07, + "output_cost_per_token": 4.05e-06, + "cache_read_input_token_cost": 1.6e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/moonshotai/Kimi-K2.6": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 7.5e-07, + "output_cost_per_token": 3.5e-06, + "cache_read_input_token_cost": 1.5e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/deepseek-ai/DeepSeek-V4-Pro-0813": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "input_cost_per_token": 1.3e-06, + "output_cost_per_token": 2.6e-06, + "cache_read_input_token_cost": 1e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3.7-Max": { + "max_tokens": 256000, + "max_input_tokens": 256000, + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 7.5e-06, + "cache_read_input_token_cost": 5e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/ByteDance/Seed-2.0-mini": { + "max_tokens": 256000, + "max_input_tokens": 256000, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 2e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3.8-2.4T-A95B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + "cache_read_input_token_cost": 2e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/MiniMaxAI/MiniMax-M3": { + "max_tokens": 524288, + "max_input_tokens": 524288, + "input_cost_per_token": 2.8e-07, + "output_cost_per_token": 1.1e-06, + "cache_read_input_token_cost": 5.6e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/google/gemini-3.1-flash-lite": { + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.5e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/google/gemini-3.7-flash": { + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "input_cost_per_token": 7.5e-07, + "output_cost_per_token": 3.75e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/inclusionAI/Ling-3.0-flash": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 1.8e-07, + "cache_read_input_token_cost": 1.2e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/stepfun-ai/Step-3.7-Flash": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 1.15e-06, + "cache_read_input_token_cost": 4e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3.5-35B-A3B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 1e-06, + "cache_read_input_token_cost": 5e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/ByteDance/Seed-1.8": { + "max_tokens": 256000, + "max_input_tokens": 256000, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 5e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/tencent/Hy3": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 5.8e-07, + "cache_read_input_token_cost": 3.5e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/ByteDance/Seed-2.0-code": { + "max_tokens": 256000, + "max_input_tokens": 256000, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 3e-06, + "cache_read_input_token_cost": 1e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/ByteDance/Seed-2.0-pro": { + "max_tokens": 256000, + "max_input_tokens": 256000, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 3e-06, + "cache_read_input_token_cost": 1e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/zai-org/GLM-5": { + "max_tokens": 202752, + "max_input_tokens": 202752, + "input_cost_per_token": 6e-07, + "output_cost_per_token": 2.08e-06, + "cache_read_input_token_cost": 1.2e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/nvidia/Nemotron-3-Nano-30B-A3B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 2e-07, + "cache_read_input_token_cost": 2.5e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/moonshotai/Kimi-K2.7-Code": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 6.8e-07, + "output_cost_per_token": 3.4e-06, + "cache_read_input_token_cost": 1.36e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/anthropic/claude-sonnet-5": { + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3.5-397B-A17B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 4.5e-07, + "output_cost_per_token": 3e-06, + "cache_read_input_token_cost": 2.2e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/deepseek-ai/DeepSeek-V4-Flash-0731": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "input_cost_per_token": 8e-08, + "output_cost_per_token": 1.8e-07, + "cache_read_input_token_cost": 1.6e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/google/gemma-4-E4B-it": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "input_cost_per_token": 2e-08, + "output_cost_per_token": 1e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/deepseek-ai/DeepSeek-V3.2": { + "max_tokens": 163840, + "max_input_tokens": 163840, + "input_cost_per_token": 2.6e-07, + "output_cost_per_token": 3.8e-07, + "cache_read_input_token_cost": 1.3e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3.8-Max": { + "max_tokens": 256000, + "max_input_tokens": 256000, + "input_cost_per_token": 1.65e-06, + "output_cost_per_token": 4.951e-06, + "cache_read_input_token_cost": 2.06e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/anthropic/claude-fable-5": { + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 5e-05, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 2.2e-06, + "cache_read_input_token_cost": 1e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3.5-122B-A10B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 2.9e-07, + "output_cost_per_token": 2.4e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/zai-org/GLM-5.1": { + "max_tokens": 202752, + "max_input_tokens": 202752, + "input_cost_per_token": 1.05e-06, + "output_cost_per_token": 3.5e-06, + "cache_read_input_token_cost": 2.05e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/deepseek-ai/DeepSeek-V4-Pro": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "input_cost_per_token": 1.3e-06, + "output_cost_per_token": 2.6e-06, + "cache_read_input_token_cost": 1e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 8.5e-08, + "output_cost_per_token": 4e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/zai-org/GLM-5.2": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "input_cost_per_token": 7.5e-07, + "output_cost_per_token": 2.4e-06, + "cache_read_input_token_cost": 1.4e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/moonshotai/Kimi-K3": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "input_cost_per_token": 2.85e-06, + "output_cost_per_token": 1.425e-05, + "cache_read_input_token_cost": 2.85e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/anthropic/claude-opus-4-7": { + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3.6-27B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 3.2e-07, + "output_cost_per_token": 3.2e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/google/gemma-4-26B-A4B-it": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 3.4e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/google/gemini-3.1-pro": { + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1.2e-05, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/XiaomiMiMo/MiMo-V2.5-Pro": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 3e-06, + "cache_read_input_token_cost": 2e-07, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/anthropic/claude-haiku-4-5": { + "max_tokens": 200000, + "max_input_tokens": 200000, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 5e-06, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/deepseek-ai/DeepSeek-V4-Flash": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "input_cost_per_token": 9e-08, + "output_cost_per_token": 1.8e-07, + "cache_read_input_token_cost": 1.8e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/openai/gpt-oss-120b-Ultra": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 9.5e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/Qwen/Qwen3.5-9B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 1.5e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/MiniMaxAI/MiniMax-M2.7-Turbo": { + "max_tokens": 196608, + "max_input_tokens": 196608, + "input_cost_per_token": 3.8e-07, + "output_cost_per_token": 1.7e-06, + "cache_read_input_token_cost": 7e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/zai-org/GLM-4.7": { + "max_tokens": 202752, + "max_input_tokens": 202752, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 1.75e-06, + "cache_read_input_token_cost": 8e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/pricing" + }, + "deepinfra/google/gemma-4-31B-it": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "input_cost_per_token": 1.3e-07, + "output_cost_per_token": 3.8e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://deepinfra.com/pricing" } } From d266326a42af668fa73c562b44d0322667265fc3 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:56:53 +0000 Subject: [PATCH 10/29] fix(model_prices): azure gpt-4.1-nano retirement date, together deprecations, novita gpt-oss-120b vision flag, fireworks deepseek-v4-pro-0813 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 24 +++++- model_prices_and_context_window.json | 24 +++++- .../test_fireworks_serverless_model_costs.py | 86 +++++++++++++++++++ 3 files changed, 128 insertions(+), 6 deletions(-) create mode 100644 tests/test_litellm/test_fireworks_serverless_model_costs.py diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 267b43a2add..00c840c8df1 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4681,7 +4681,7 @@ "supports_web_search": false }, "azure/gpt-4.1-nano": { - "deprecation_date": "2026-10-14", + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 1e-07, "input_cost_per_token_batches": 5e-08, @@ -4714,7 +4714,7 @@ "supports_vision": true }, "azure/gpt-4.1-nano-2025-04-14": { - "deprecation_date": "2026-10-14", + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 1e-07, "input_cost_per_token_batches": 5e-08, @@ -18577,6 +18577,22 @@ "supports_tool_choice": true, "supports_vision": false }, + "fireworks_ai/accounts/fireworks/models/deepseek-v4-pro-0813": { + "cache_read_input_token_cost": 4.4e-08, + "input_cost_per_token": 1.32e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 3.96e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, "fireworks_ai/accounts/fireworks/models/firefunction-v2": { "input_cost_per_token": 9e-07, "litellm_provider": "fireworks_ai", @@ -38475,6 +38491,7 @@ "supports_tool_choice": true }, "together_ai/google/gemma-3n-E4B-it": { + "deprecation_date": "2026-08-25", "input_cost_per_token": 6e-08, "litellm_provider": "together_ai", "max_input_tokens": 32768, @@ -38510,6 +38527,7 @@ "source": "https://docs.together.ai/docs/serverless-models" }, "together_ai/meta-llama/Llama-Guard-4-12B": { + "deprecation_date": "2026-08-25", "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, @@ -47311,7 +47329,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, - "supports_vision": true, + "supports_vision": false, "supports_system_messages": true, "supports_response_schema": true, "supports_reasoning": true diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 267b43a2add..00c840c8df1 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -4681,7 +4681,7 @@ "supports_web_search": false }, "azure/gpt-4.1-nano": { - "deprecation_date": "2026-10-14", + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 1e-07, "input_cost_per_token_batches": 5e-08, @@ -4714,7 +4714,7 @@ "supports_vision": true }, "azure/gpt-4.1-nano-2025-04-14": { - "deprecation_date": "2026-10-14", + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 1e-07, "input_cost_per_token_batches": 5e-08, @@ -18577,6 +18577,22 @@ "supports_tool_choice": true, "supports_vision": false }, + "fireworks_ai/accounts/fireworks/models/deepseek-v4-pro-0813": { + "cache_read_input_token_cost": 4.4e-08, + "input_cost_per_token": 1.32e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 3.96e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, "fireworks_ai/accounts/fireworks/models/firefunction-v2": { "input_cost_per_token": 9e-07, "litellm_provider": "fireworks_ai", @@ -38475,6 +38491,7 @@ "supports_tool_choice": true }, "together_ai/google/gemma-3n-E4B-it": { + "deprecation_date": "2026-08-25", "input_cost_per_token": 6e-08, "litellm_provider": "together_ai", "max_input_tokens": 32768, @@ -38510,6 +38527,7 @@ "source": "https://docs.together.ai/docs/serverless-models" }, "together_ai/meta-llama/Llama-Guard-4-12B": { + "deprecation_date": "2026-08-25", "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, @@ -47311,7 +47329,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, - "supports_vision": true, + "supports_vision": false, "supports_system_messages": true, "supports_response_schema": true, "supports_reasoning": true diff --git a/tests/test_litellm/test_fireworks_serverless_model_costs.py b/tests/test_litellm/test_fireworks_serverless_model_costs.py new file mode 100644 index 00000000000..0458af0da0e --- /dev/null +++ b/tests/test_litellm/test_fireworks_serverless_model_costs.py @@ -0,0 +1,86 @@ +""" +Validate the Fireworks AI Serverless entry added for #37274 exists in +`model_prices_and_context_window.json` and that the bare Fireworks model ID +resolves through `get_model_info`. + +Pricing as published at https://docs.fireworks.ai/serverless/pricing +(USD per 1M tokens, uncached input / cached input / output): + + accounts/fireworks/models/deepseek-v4-pro-0813 -> $1.32 / $0.044 / $3.96 +""" + +import json +import os + +import pytest + +import litellm +from litellm.utils import get_model_info + + +@pytest.fixture(scope="module", autouse=True) +def _local_model_cost_map(): + """ + Point litellm at the bundled cost map for the duration of this module + only. ``mp.undo()`` restores both the environment variable and + ``litellm.model_cost`` so nothing leaks into later tests. + """ + mp = pytest.MonkeyPatch() + mp.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + mp.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + get_model_info.cache_clear() + yield + mp.undo() + get_model_info.cache_clear() + + +NEW_ENTRIES = { + "fireworks_ai/accounts/fireworks/models/deepseek-v4-pro-0813": { + "input_cost_per_token": 1.32e-06, + "cache_read_input_token_cost": 4.4e-08, + "output_cost_per_token": 3.96e-06, + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + }, +} + + +@pytest.fixture(scope="module") +def model_data(): + json_path = os.path.join( + os.path.dirname(__file__), "../../model_prices_and_context_window.json" + ) + with open(json_path) as f: + return json.load(f) + + +def test_fireworks_serverless_entries_exist(model_data): + """The new prefixed entry carries the pricing and metadata from #37274.""" + for key, expected in NEW_ENTRIES.items(): + assert key in model_data, f"{key} is missing from model_prices_and_context_window.json" + entry = model_data[key] + for field, value in expected.items(): + assert entry[field] == pytest.approx(value), f"{key}.{field}" + assert entry["litellm_provider"] == "fireworks_ai" + assert entry["mode"] == "chat" + assert entry["supports_function_calling"] is True + assert entry["supports_vision"] is False + + +def test_bare_fireworks_ids_resolve_through_prefixed_entries(): + """Bare IDs from #37274 resolve via the provider-prefix lookup path.""" + for bare_id, prefixed_key in [ + ( + "accounts/fireworks/models/deepseek-v4-pro-0813", + "fireworks_ai/accounts/fireworks/models/deepseek-v4-pro-0813", + ), + ]: + info = get_model_info(model=bare_id, custom_llm_provider="fireworks_ai") + expected = NEW_ENTRIES[prefixed_key] + assert info.get("key") == prefixed_key + assert info["litellm_provider"] == "fireworks_ai" + assert info["input_cost_per_token"] == pytest.approx(expected["input_cost_per_token"]) + assert info["cache_read_input_token_cost"] == pytest.approx(expected["cache_read_input_token_cost"]) + assert info["output_cost_per_token"] == pytest.approx(expected["output_cost_per_token"]) + assert info["max_input_tokens"] == expected["max_input_tokens"] + assert info["max_output_tokens"] == expected["max_output_tokens"] From d3ede97189e6c65157da5628a195e26972d0ec95 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:57:26 +0000 Subject: [PATCH 11/29] fix(model_prices): keep novita gpt-oss-120b vision flag per provider catalog Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 2 +- model_prices_and_context_window.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 00c840c8df1..1ca0e748d38 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -47329,7 +47329,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, - "supports_vision": false, + "supports_vision": true, "supports_system_messages": true, "supports_response_schema": true, "supports_reasoning": true diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 00c840c8df1..1ca0e748d38 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -47329,7 +47329,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, - "supports_vision": false, + "supports_vision": true, "supports_system_messages": true, "supports_response_schema": true, "supports_reasoning": true From 07c9812739b06d8d972f78fb141fedc834040b57 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 26 Aug 2026 17:18:51 +0000 Subject: [PATCH 12/29] fix(model_prices): carry anthropic behavior flags on deepinfra claude entries, move retired together models to deprecated list Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 84 +++++++++++-------- model_prices_and_context_window.json | 84 +++++++++++-------- .../test_together_ai_model_metadata.py | 4 +- 3 files changed, 102 insertions(+), 70 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1ca0e748d38..5f565b84d38 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -52689,18 +52689,21 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/anthropic/claude-opus-4-8": { - "max_tokens": 1000000, - "max_input_tokens": 1000000, "input_cost_per_token": 5e-06, - "output_cost_per_token": 2.5e-05, "litellm_provider": "deepinfra", + "max_input_tokens": 1000000, + "max_tokens": 1000000, "mode": "chat", + "output_cost_per_token": 2.5e-05, + "prompt_cache_min_tokens": 1024, + "source": "https://deepinfra.com/pricing", + "supports_adaptive_thinking": true, "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, "supports_reasoning": true, - "supports_vision": true, - "source": "https://deepinfra.com/pricing" + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true }, "deepinfra/anthropic/claude-sonnet-4-6": { "max_tokens": 1000000, @@ -52890,18 +52893,21 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/anthropic/claude-opus-5": { - "max_tokens": 1000000, - "max_input_tokens": 1000000, "input_cost_per_token": 5e-06, - "output_cost_per_token": 2.5e-05, "litellm_provider": "deepinfra", + "max_input_tokens": 1000000, + "max_tokens": 1000000, "mode": "chat", + "output_cost_per_token": 2.5e-05, + "prompt_cache_min_tokens": 512, + "source": "https://deepinfra.com/pricing", + "supports_adaptive_thinking": true, "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, "supports_reasoning": true, - "supports_vision": true, - "source": "https://deepinfra.com/pricing" + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true }, "deepinfra/thinkingmachines/Inkling": { "max_tokens": 524288, @@ -53199,18 +53205,21 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/anthropic/claude-sonnet-5": { - "max_tokens": 1000000, - "max_input_tokens": 1000000, "input_cost_per_token": 2e-06, - "output_cost_per_token": 1e-05, "litellm_provider": "deepinfra", + "max_input_tokens": 1000000, + "max_tokens": 1000000, "mode": "chat", + "output_cost_per_token": 1e-05, + "prompt_cache_min_tokens": 1024, + "source": "https://deepinfra.com/pricing", + "supports_adaptive_thinking": true, "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, "supports_reasoning": true, - "supports_vision": true, - "source": "https://deepinfra.com/pricing" + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true }, "deepinfra/Qwen/Qwen3.5-397B-A17B": { "max_tokens": 262144, @@ -53288,18 +53297,22 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/anthropic/claude-fable-5": { - "max_tokens": 1000000, - "max_input_tokens": 1000000, "input_cost_per_token": 1e-05, - "output_cost_per_token": 5e-05, "litellm_provider": "deepinfra", + "max_input_tokens": 1000000, + "max_tokens": 1000000, "mode": "chat", + "output_cost_per_token": 5e-05, + "prompt_cache_min_tokens": 512, + "source": "https://deepinfra.com/pricing", + "supports_adaptive_thinking": true, "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, "supports_vision": true, - "source": "https://deepinfra.com/pricing" + "thinking_always_on": true }, "deepinfra/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { "max_tokens": 262144, @@ -53408,18 +53421,21 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/anthropic/claude-opus-4-7": { - "max_tokens": 1000000, - "max_input_tokens": 1000000, "input_cost_per_token": 5e-06, - "output_cost_per_token": 2.5e-05, "litellm_provider": "deepinfra", + "max_input_tokens": 1000000, + "max_tokens": 1000000, "mode": "chat", + "output_cost_per_token": 2.5e-05, + "prompt_cache_min_tokens": 2048, + "source": "https://deepinfra.com/pricing", + "supports_adaptive_thinking": true, "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, "supports_reasoning": true, - "supports_vision": true, - "source": "https://deepinfra.com/pricing" + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true }, "deepinfra/Qwen/Qwen3.6-27B": { "max_tokens": 262144, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1ca0e748d38..5f565b84d38 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -52689,18 +52689,21 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/anthropic/claude-opus-4-8": { - "max_tokens": 1000000, - "max_input_tokens": 1000000, "input_cost_per_token": 5e-06, - "output_cost_per_token": 2.5e-05, "litellm_provider": "deepinfra", + "max_input_tokens": 1000000, + "max_tokens": 1000000, "mode": "chat", + "output_cost_per_token": 2.5e-05, + "prompt_cache_min_tokens": 1024, + "source": "https://deepinfra.com/pricing", + "supports_adaptive_thinking": true, "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, "supports_reasoning": true, - "supports_vision": true, - "source": "https://deepinfra.com/pricing" + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true }, "deepinfra/anthropic/claude-sonnet-4-6": { "max_tokens": 1000000, @@ -52890,18 +52893,21 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/anthropic/claude-opus-5": { - "max_tokens": 1000000, - "max_input_tokens": 1000000, "input_cost_per_token": 5e-06, - "output_cost_per_token": 2.5e-05, "litellm_provider": "deepinfra", + "max_input_tokens": 1000000, + "max_tokens": 1000000, "mode": "chat", + "output_cost_per_token": 2.5e-05, + "prompt_cache_min_tokens": 512, + "source": "https://deepinfra.com/pricing", + "supports_adaptive_thinking": true, "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, "supports_reasoning": true, - "supports_vision": true, - "source": "https://deepinfra.com/pricing" + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true }, "deepinfra/thinkingmachines/Inkling": { "max_tokens": 524288, @@ -53199,18 +53205,21 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/anthropic/claude-sonnet-5": { - "max_tokens": 1000000, - "max_input_tokens": 1000000, "input_cost_per_token": 2e-06, - "output_cost_per_token": 1e-05, "litellm_provider": "deepinfra", + "max_input_tokens": 1000000, + "max_tokens": 1000000, "mode": "chat", + "output_cost_per_token": 1e-05, + "prompt_cache_min_tokens": 1024, + "source": "https://deepinfra.com/pricing", + "supports_adaptive_thinking": true, "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, "supports_reasoning": true, - "supports_vision": true, - "source": "https://deepinfra.com/pricing" + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true }, "deepinfra/Qwen/Qwen3.5-397B-A17B": { "max_tokens": 262144, @@ -53288,18 +53297,22 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/anthropic/claude-fable-5": { - "max_tokens": 1000000, - "max_input_tokens": 1000000, "input_cost_per_token": 1e-05, - "output_cost_per_token": 5e-05, "litellm_provider": "deepinfra", + "max_input_tokens": 1000000, + "max_tokens": 1000000, "mode": "chat", + "output_cost_per_token": 5e-05, + "prompt_cache_min_tokens": 512, + "source": "https://deepinfra.com/pricing", + "supports_adaptive_thinking": true, "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, "supports_vision": true, - "source": "https://deepinfra.com/pricing" + "thinking_always_on": true }, "deepinfra/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { "max_tokens": 262144, @@ -53408,18 +53421,21 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/anthropic/claude-opus-4-7": { - "max_tokens": 1000000, - "max_input_tokens": 1000000, "input_cost_per_token": 5e-06, - "output_cost_per_token": 2.5e-05, "litellm_provider": "deepinfra", + "max_input_tokens": 1000000, + "max_tokens": 1000000, "mode": "chat", + "output_cost_per_token": 2.5e-05, + "prompt_cache_min_tokens": 2048, + "source": "https://deepinfra.com/pricing", + "supports_adaptive_thinking": true, "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, "supports_reasoning": true, - "supports_vision": true, - "source": "https://deepinfra.com/pricing" + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true }, "deepinfra/Qwen/Qwen3.6-27B": { "max_tokens": 262144, diff --git a/tests/test_litellm/test_together_ai_model_metadata.py b/tests/test_litellm/test_together_ai_model_metadata.py index 5a0aadf4737..60e4b8ddb4d 100644 --- a/tests/test_litellm/test_together_ai_model_metadata.py +++ b/tests/test_litellm/test_together_ai_model_metadata.py @@ -31,16 +31,16 @@ SERVERLESS_CHAT_MODELS: Final = ( "together_ai/meta-models/Muse-Glimmer-30B", "together_ai/google/gemma-4-31B-it", "together_ai/pearl-ai/gemma-4-31b-it", - "together_ai/google/gemma-3n-E4B-it", "together_ai/arize-ai/qwen-2-1.5b-instruct", "together_ai/Prism-ML/Ternary-Bonsai-27B", - "together_ai/meta-llama/Llama-Guard-4-12B", "together_ai/openai/gpt-oss-120b", "together_ai/openai/gpt-oss-20b", "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo", ) DEPRECATED_MODELS: Final = { + "together_ai/google/gemma-3n-E4B-it": "2026-08-25", + "together_ai/meta-llama/Llama-Guard-4-12B": "2026-08-25", "together_ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": "2026-07-10", "together_ai/Qwen/Qwen3.5-397B-A17B": "2026-06-29", "together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": "2026-06-04", From 4456a4407feb474439d344ee5116d7d82d8701ab Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 26 Aug 2026 19:20:14 +0000 Subject: [PATCH 13/29] fix(model_prices): azure gpt-5.6 cache writes, mistral missing models, together cache reads Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 272 ++++++++++++++++-- model_prices_and_context_window.json | 272 ++++++++++++++++-- tests/test_litellm/test_cost_calculator.py | 78 +++++ 3 files changed, 560 insertions(+), 62 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 7515e7ad396..1526319ecf1 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -6592,6 +6592,9 @@ "supports_web_search": true }, "azure/gpt-5.6": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05, + "cache_creation_input_token_cost_priority": 1.25e-05, "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1e-06, @@ -6642,6 +6645,9 @@ "supports_minimal_reasoning_effort": false }, "azure/gpt-5.6-sol": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05, + "cache_creation_input_token_cost_priority": 1.25e-05, "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1e-06, @@ -6693,6 +6699,9 @@ "supports_minimal_reasoning_effort": false }, "azure/gpt-5.6-terra": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5e-06, + "cache_creation_input_token_cost_priority": 5e-06, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_272k_tokens": 4e-07, "cache_read_input_token_cost_priority": 4e-07, @@ -6744,6 +6753,9 @@ "supports_minimal_reasoning_effort": false }, "azure/gpt-5.6-luna": { + "cache_creation_input_token_cost": 2.5e-07, + "cache_creation_input_token_cost_above_272k_tokens": 5e-07, + "cache_creation_input_token_cost_priority": 5e-07, "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_above_272k_tokens": 4e-08, "cache_read_input_token_cost_priority": 4e-08, @@ -6795,12 +6807,15 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.6": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, + "cache_creation_input_token_cost_priority": 1.375e-05, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, - "cache_read_input_token_cost_priority": 1.375e-06, + "cache_read_input_token_cost_priority": 1.1e-06, "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, - "input_cost_per_token_priority": 1.375e-05, + "input_cost_per_token_priority": 1.1e-05, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, @@ -6808,7 +6823,7 @@ "mode": "chat", "output_cost_per_token": 3.3e-05, "output_cost_per_token_above_272k_tokens": 4.95e-05, - "output_cost_per_token_priority": 8.25e-05, + "output_cost_per_token_priority": 6.6e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -6842,13 +6857,16 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.6-sol": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, + "cache_creation_input_token_cost_priority": 1.375e-05, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, - "cache_read_input_token_cost_priority": 1.375e-06, + "cache_read_input_token_cost_priority": 1.1e-06, "deprecation_date": "2028-01-11", "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, - "input_cost_per_token_priority": 1.375e-05, + "input_cost_per_token_priority": 1.1e-05, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, @@ -6856,7 +6874,7 @@ "mode": "chat", "output_cost_per_token": 3.3e-05, "output_cost_per_token_above_272k_tokens": 4.95e-05, - "output_cost_per_token_priority": 8.25e-05, + "output_cost_per_token_priority": 6.6e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -6890,13 +6908,16 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.6-terra": { + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5.5e-06, + "cache_creation_input_token_cost_priority": 5.5e-06, "cache_read_input_token_cost": 2.2e-07, "cache_read_input_token_cost_above_272k_tokens": 4.4e-07, - "cache_read_input_token_cost_priority": 5.5e-07, + "cache_read_input_token_cost_priority": 4.4e-07, "deprecation_date": "2028-01-11", "input_cost_per_token": 2.2e-06, "input_cost_per_token_above_272k_tokens": 4.4e-06, - "input_cost_per_token_priority": 5.5e-06, + "input_cost_per_token_priority": 4.4e-06, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, @@ -6904,7 +6925,7 @@ "mode": "chat", "output_cost_per_token": 1.32e-05, "output_cost_per_token_above_272k_tokens": 1.98e-05, - "output_cost_per_token_priority": 3.3e-05, + "output_cost_per_token_priority": 2.64e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -6938,13 +6959,16 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.6-luna": { + "cache_creation_input_token_cost": 2.75e-07, + "cache_creation_input_token_cost_above_272k_tokens": 5.5e-07, + "cache_creation_input_token_cost_priority": 5.5e-07, "cache_read_input_token_cost": 2.2e-08, "cache_read_input_token_cost_above_272k_tokens": 4.4e-08, - "cache_read_input_token_cost_priority": 5.5e-08, + "cache_read_input_token_cost_priority": 4.4e-08, "deprecation_date": "2028-01-11", "input_cost_per_token": 2.2e-07, "input_cost_per_token_above_272k_tokens": 4.4e-07, - "input_cost_per_token_priority": 5.5e-07, + "input_cost_per_token_priority": 4.4e-07, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, @@ -6952,7 +6976,7 @@ "mode": "chat", "output_cost_per_token": 1.32e-06, "output_cost_per_token_above_272k_tokens": 1.98e-06, - "output_cost_per_token_priority": 3.3e-06, + "output_cost_per_token_priority": 2.64e-06, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -6986,12 +7010,15 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.6": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, + "cache_creation_input_token_cost_priority": 1.375e-05, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, - "cache_read_input_token_cost_priority": 1.375e-06, + "cache_read_input_token_cost_priority": 1.1e-06, "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, - "input_cost_per_token_priority": 1.375e-05, + "input_cost_per_token_priority": 1.1e-05, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, @@ -6999,7 +7026,7 @@ "mode": "chat", "output_cost_per_token": 3.3e-05, "output_cost_per_token_above_272k_tokens": 4.95e-05, - "output_cost_per_token_priority": 8.25e-05, + "output_cost_per_token_priority": 6.6e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -7033,13 +7060,16 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.6-sol": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, + "cache_creation_input_token_cost_priority": 1.375e-05, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, - "cache_read_input_token_cost_priority": 1.375e-06, + "cache_read_input_token_cost_priority": 1.1e-06, "deprecation_date": "2028-01-11", "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, - "input_cost_per_token_priority": 1.375e-05, + "input_cost_per_token_priority": 1.1e-05, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, @@ -7047,7 +7077,7 @@ "mode": "chat", "output_cost_per_token": 3.3e-05, "output_cost_per_token_above_272k_tokens": 4.95e-05, - "output_cost_per_token_priority": 8.25e-05, + "output_cost_per_token_priority": 6.6e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -7081,13 +7111,16 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.6-terra": { + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5.5e-06, + "cache_creation_input_token_cost_priority": 5.5e-06, "cache_read_input_token_cost": 2.2e-07, "cache_read_input_token_cost_above_272k_tokens": 4.4e-07, - "cache_read_input_token_cost_priority": 5.5e-07, + "cache_read_input_token_cost_priority": 4.4e-07, "deprecation_date": "2028-01-11", "input_cost_per_token": 2.2e-06, "input_cost_per_token_above_272k_tokens": 4.4e-06, - "input_cost_per_token_priority": 5.5e-06, + "input_cost_per_token_priority": 4.4e-06, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, @@ -7095,7 +7128,7 @@ "mode": "chat", "output_cost_per_token": 1.32e-05, "output_cost_per_token_above_272k_tokens": 1.98e-05, - "output_cost_per_token_priority": 3.3e-05, + "output_cost_per_token_priority": 2.64e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -7129,13 +7162,16 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.6-luna": { + "cache_creation_input_token_cost": 2.75e-07, + "cache_creation_input_token_cost_above_272k_tokens": 5.5e-07, + "cache_creation_input_token_cost_priority": 5.5e-07, "cache_read_input_token_cost": 2.2e-08, "cache_read_input_token_cost_above_272k_tokens": 4.4e-08, - "cache_read_input_token_cost_priority": 5.5e-08, + "cache_read_input_token_cost_priority": 4.4e-08, "deprecation_date": "2028-01-11", "input_cost_per_token": 2.2e-07, "input_cost_per_token_above_272k_tokens": 4.4e-07, - "input_cost_per_token_priority": 5.5e-07, + "input_cost_per_token_priority": 4.4e-07, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, @@ -7143,7 +7179,7 @@ "mode": "chat", "output_cost_per_token": 1.32e-06, "output_cost_per_token_above_272k_tokens": 1.98e-06, - "output_cost_per_token_priority": 3.3e-06, + "output_cost_per_token_priority": 2.64e-06, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -30721,6 +30757,152 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "mistral/ministral-14b-2512": { + "input_cost_per_token": 2e-07, + "litellm_provider": "mistral", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2e-07, + "source": "https://docs.mistral.ai/models/ministral-3-14b-25-12", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "mistral/ministral-14b-latest": { + "input_cost_per_token": 2e-07, + "litellm_provider": "mistral", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2e-07, + "source": "https://docs.mistral.ai/models/ministral-3-14b-25-12", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "mistral/ministral-3b-2512": { + "input_cost_per_token": 1e-07, + "litellm_provider": "mistral", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1e-07, + "source": "https://docs.mistral.ai/models/ministral-3-3b-25-12", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "mistral/ministral-3b-latest": { + "input_cost_per_token": 1e-07, + "litellm_provider": "mistral", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1e-07, + "source": "https://docs.mistral.ai/models/ministral-3-3b-25-12", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "mistral/mistral-embed-2312": { + "input_cost_per_token": 1e-07, + "litellm_provider": "mistral", + "max_input_tokens": 8192, + "max_tokens": 8192, + "mode": "embedding", + "source": "https://docs.mistral.ai/models/mistral-embed-23-12" + }, + "mistral/mistral-medium-3": { + "input_cost_per_token": 1.5e-06, + "litellm_provider": "mistral", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 7.5e-06, + "source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "mistral/voxtral-mini-transcribe-realtime-latest": { + "input_cost_per_second": 0.0001, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/model-cards/voxtral-mini-transcribe-realtime-26-02", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-mini-tts-latest": { + "litellm_provider": "mistral", + "mode": "audio_speech", + "output_cost_per_character": 1.6e-05, + "source": "https://docs.mistral.ai/models/model-cards/voxtral-tts-26-03", + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_output": true + }, + "mistral/voxtral-small-2507": { + "input_cost_per_second": 6.666666666666667e-05, + "input_cost_per_token": 1e-07, + "litellm_provider": "mistral", + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 4e-07, + "source": "https://docs.mistral.ai/models/voxtral-small-25-07", + "supports_audio_input": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "mistral/voxtral-small-latest": { + "input_cost_per_second": 6.666666666666667e-05, + "input_cost_per_token": 1e-07, + "litellm_provider": "mistral", + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 4e-07, + "source": "https://docs.mistral.ai/models/voxtral-small-25-07", + "supports_audio_input": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, "mistral/zai-glm-5-2": { "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.4e-06, @@ -31191,9 +31373,9 @@ "mistral/ministral-3-3b-2512": { "input_cost_per_token": 1e-07, "litellm_provider": "mistral", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://mistral.ai/pricing", @@ -38342,6 +38524,7 @@ "supports_tool_choice": true }, "together_ai/Qwen/Qwen3.5-397B-A17B": { + "cache_read_input_token_cost": 3.5e-07, "deprecation_date": "2026-06-29", "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", @@ -38351,10 +38534,12 @@ "source": "https://www.together.ai/models/Qwen/Qwen3.5-397B-A17B", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/MiniMaxAI/MiniMax-M3": { + "cache_read_input_token_cost": 6e-08, "input_cost_per_token": 3e-07, "litellm_provider": "together_ai", "max_input_tokens": 524288, @@ -38365,6 +38550,7 @@ "source": "https://docs.together.ai/docs/serverless-models", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -38408,6 +38594,7 @@ "supports_reasoning": true }, "together_ai/Qwen/Qwen3.7-Max": { + "cache_read_input_token_cost": 1.3e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "together_ai", "max_input_tokens": 1000000, @@ -38415,7 +38602,8 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 3.75e-06, - "source": "https://docs.together.ai/docs/serverless-models" + "source": "https://docs.together.ai/docs/serverless-models", + "supports_prompt_caching": true }, "together_ai/Qwen/Qwen3.7-Plus": { "input_cost_per_token": 3.2e-07, @@ -38428,6 +38616,7 @@ "source": "https://docs.together.ai/docs/serverless-models" }, "together_ai/Qwen/Qwen3.8-2.4T-A95B": { + "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2.5e-06, "litellm_provider": "together_ai", "max_input_tokens": 1010000, @@ -38435,7 +38624,8 @@ "max_tokens": 1010000, "mode": "chat", "output_cost_per_token": 6.25e-06, - "source": "https://docs.together.ai/docs/serverless-models" + "source": "https://docs.together.ai/docs/serverless-models", + "supports_prompt_caching": true }, "together_ai/arize-ai/qwen-2-1.5b-instruct": { "input_cost_per_token": 1e-07, @@ -38448,6 +38638,7 @@ "source": "https://docs.together.ai/docs/serverless-models" }, "together_ai/deepseek-ai/DeepSeek-V4-Flash-0731": { + "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 1.4e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, @@ -38458,10 +38649,12 @@ "source": "https://docs.together.ai/docs/serverless-models", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/deepseek-ai/DeepSeek-V4-Pro": { + "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 1.74e-06, "litellm_provider": "together_ai", "max_input_tokens": 512000, @@ -38472,11 +38665,13 @@ "source": "https://docs.together.ai/docs/serverless-models", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813": { + "cache_read_input_token_cost": 1.3e-07, "input_cost_per_token": 1.32e-06, "litellm_provider": "together_ai", "max_input_tokens": 1048576, @@ -38487,6 +38682,7 @@ "source": "https://docs.together.ai/docs/serverless-models", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true }, @@ -38538,6 +38734,7 @@ "source": "https://docs.together.ai/docs/serverless-models" }, "together_ai/meta-models/Muse-Glimmer-30B": { + "cache_read_input_token_cost": 4e-08, "input_cost_per_token": 3.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, @@ -38545,9 +38742,11 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-06, - "source": "https://docs.together.ai/docs/serverless-models" + "source": "https://docs.together.ai/docs/serverless-models", + "supports_prompt_caching": true }, "together_ai/moonshotai/Kimi-K2.7-Code": { + "cache_read_input_token_cost": 1.9e-07, "input_cost_per_token": 9.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, @@ -38558,11 +38757,13 @@ "source": "https://docs.together.ai/docs/serverless-models", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true }, "together_ai/moonshotai/Kimi-K3": { + "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "litellm_provider": "together_ai", "max_input_tokens": 1048576, @@ -38573,12 +38774,14 @@ "source": "https://docs.together.ai/docs/serverless-models", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true }, "together_ai/nvidia/nemotron-3-ultra-550b-a55b": { + "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 512288, @@ -38589,6 +38792,7 @@ "source": "https://docs.together.ai/docs/serverless-models", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true @@ -38604,6 +38808,7 @@ "source": "https://docs.together.ai/docs/serverless-models" }, "together_ai/thinkingmachines/Inkling": { + "cache_read_input_token_cost": 1.7e-07, "input_cost_per_token": 1e-06, "litellm_provider": "together_ai", "max_input_tokens": 524288, @@ -38614,10 +38819,12 @@ "source": "https://docs.together.ai/docs/serverless-models", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/thinkingmachines/Inkling-Small": { + "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 5e-07, "litellm_provider": "together_ai", "max_input_tokens": 524288, @@ -38625,9 +38832,11 @@ "max_tokens": 524288, "mode": "chat", "output_cost_per_token": 1.2e-06, - "source": "https://docs.together.ai/docs/serverless-models" + "source": "https://docs.together.ai/docs/serverless-models", + "supports_prompt_caching": true }, "together_ai/zai-org/GLM-5.2": { + "cache_read_input_token_cost": 2.6e-07, "input_cost_per_token": 1.4e-06, "litellm_provider": "together_ai", "max_input_tokens": 1048575, @@ -38638,6 +38847,7 @@ "source": "https://docs.together.ai/docs/serverless-models", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 7515e7ad396..1526319ecf1 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -6592,6 +6592,9 @@ "supports_web_search": true }, "azure/gpt-5.6": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05, + "cache_creation_input_token_cost_priority": 1.25e-05, "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1e-06, @@ -6642,6 +6645,9 @@ "supports_minimal_reasoning_effort": false }, "azure/gpt-5.6-sol": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05, + "cache_creation_input_token_cost_priority": 1.25e-05, "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1e-06, @@ -6693,6 +6699,9 @@ "supports_minimal_reasoning_effort": false }, "azure/gpt-5.6-terra": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5e-06, + "cache_creation_input_token_cost_priority": 5e-06, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_272k_tokens": 4e-07, "cache_read_input_token_cost_priority": 4e-07, @@ -6744,6 +6753,9 @@ "supports_minimal_reasoning_effort": false }, "azure/gpt-5.6-luna": { + "cache_creation_input_token_cost": 2.5e-07, + "cache_creation_input_token_cost_above_272k_tokens": 5e-07, + "cache_creation_input_token_cost_priority": 5e-07, "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_above_272k_tokens": 4e-08, "cache_read_input_token_cost_priority": 4e-08, @@ -6795,12 +6807,15 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.6": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, + "cache_creation_input_token_cost_priority": 1.375e-05, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, - "cache_read_input_token_cost_priority": 1.375e-06, + "cache_read_input_token_cost_priority": 1.1e-06, "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, - "input_cost_per_token_priority": 1.375e-05, + "input_cost_per_token_priority": 1.1e-05, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, @@ -6808,7 +6823,7 @@ "mode": "chat", "output_cost_per_token": 3.3e-05, "output_cost_per_token_above_272k_tokens": 4.95e-05, - "output_cost_per_token_priority": 8.25e-05, + "output_cost_per_token_priority": 6.6e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -6842,13 +6857,16 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.6-sol": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, + "cache_creation_input_token_cost_priority": 1.375e-05, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, - "cache_read_input_token_cost_priority": 1.375e-06, + "cache_read_input_token_cost_priority": 1.1e-06, "deprecation_date": "2028-01-11", "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, - "input_cost_per_token_priority": 1.375e-05, + "input_cost_per_token_priority": 1.1e-05, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, @@ -6856,7 +6874,7 @@ "mode": "chat", "output_cost_per_token": 3.3e-05, "output_cost_per_token_above_272k_tokens": 4.95e-05, - "output_cost_per_token_priority": 8.25e-05, + "output_cost_per_token_priority": 6.6e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -6890,13 +6908,16 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.6-terra": { + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5.5e-06, + "cache_creation_input_token_cost_priority": 5.5e-06, "cache_read_input_token_cost": 2.2e-07, "cache_read_input_token_cost_above_272k_tokens": 4.4e-07, - "cache_read_input_token_cost_priority": 5.5e-07, + "cache_read_input_token_cost_priority": 4.4e-07, "deprecation_date": "2028-01-11", "input_cost_per_token": 2.2e-06, "input_cost_per_token_above_272k_tokens": 4.4e-06, - "input_cost_per_token_priority": 5.5e-06, + "input_cost_per_token_priority": 4.4e-06, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, @@ -6904,7 +6925,7 @@ "mode": "chat", "output_cost_per_token": 1.32e-05, "output_cost_per_token_above_272k_tokens": 1.98e-05, - "output_cost_per_token_priority": 3.3e-05, + "output_cost_per_token_priority": 2.64e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -6938,13 +6959,16 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.6-luna": { + "cache_creation_input_token_cost": 2.75e-07, + "cache_creation_input_token_cost_above_272k_tokens": 5.5e-07, + "cache_creation_input_token_cost_priority": 5.5e-07, "cache_read_input_token_cost": 2.2e-08, "cache_read_input_token_cost_above_272k_tokens": 4.4e-08, - "cache_read_input_token_cost_priority": 5.5e-08, + "cache_read_input_token_cost_priority": 4.4e-08, "deprecation_date": "2028-01-11", "input_cost_per_token": 2.2e-07, "input_cost_per_token_above_272k_tokens": 4.4e-07, - "input_cost_per_token_priority": 5.5e-07, + "input_cost_per_token_priority": 4.4e-07, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, @@ -6952,7 +6976,7 @@ "mode": "chat", "output_cost_per_token": 1.32e-06, "output_cost_per_token_above_272k_tokens": 1.98e-06, - "output_cost_per_token_priority": 3.3e-06, + "output_cost_per_token_priority": 2.64e-06, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -6986,12 +7010,15 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.6": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, + "cache_creation_input_token_cost_priority": 1.375e-05, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, - "cache_read_input_token_cost_priority": 1.375e-06, + "cache_read_input_token_cost_priority": 1.1e-06, "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, - "input_cost_per_token_priority": 1.375e-05, + "input_cost_per_token_priority": 1.1e-05, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, @@ -6999,7 +7026,7 @@ "mode": "chat", "output_cost_per_token": 3.3e-05, "output_cost_per_token_above_272k_tokens": 4.95e-05, - "output_cost_per_token_priority": 8.25e-05, + "output_cost_per_token_priority": 6.6e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -7033,13 +7060,16 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.6-sol": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, + "cache_creation_input_token_cost_priority": 1.375e-05, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, - "cache_read_input_token_cost_priority": 1.375e-06, + "cache_read_input_token_cost_priority": 1.1e-06, "deprecation_date": "2028-01-11", "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, - "input_cost_per_token_priority": 1.375e-05, + "input_cost_per_token_priority": 1.1e-05, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, @@ -7047,7 +7077,7 @@ "mode": "chat", "output_cost_per_token": 3.3e-05, "output_cost_per_token_above_272k_tokens": 4.95e-05, - "output_cost_per_token_priority": 8.25e-05, + "output_cost_per_token_priority": 6.6e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -7081,13 +7111,16 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.6-terra": { + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5.5e-06, + "cache_creation_input_token_cost_priority": 5.5e-06, "cache_read_input_token_cost": 2.2e-07, "cache_read_input_token_cost_above_272k_tokens": 4.4e-07, - "cache_read_input_token_cost_priority": 5.5e-07, + "cache_read_input_token_cost_priority": 4.4e-07, "deprecation_date": "2028-01-11", "input_cost_per_token": 2.2e-06, "input_cost_per_token_above_272k_tokens": 4.4e-06, - "input_cost_per_token_priority": 5.5e-06, + "input_cost_per_token_priority": 4.4e-06, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, @@ -7095,7 +7128,7 @@ "mode": "chat", "output_cost_per_token": 1.32e-05, "output_cost_per_token_above_272k_tokens": 1.98e-05, - "output_cost_per_token_priority": 3.3e-05, + "output_cost_per_token_priority": 2.64e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -7129,13 +7162,16 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.6-luna": { + "cache_creation_input_token_cost": 2.75e-07, + "cache_creation_input_token_cost_above_272k_tokens": 5.5e-07, + "cache_creation_input_token_cost_priority": 5.5e-07, "cache_read_input_token_cost": 2.2e-08, "cache_read_input_token_cost_above_272k_tokens": 4.4e-08, - "cache_read_input_token_cost_priority": 5.5e-08, + "cache_read_input_token_cost_priority": 4.4e-08, "deprecation_date": "2028-01-11", "input_cost_per_token": 2.2e-07, "input_cost_per_token_above_272k_tokens": 4.4e-07, - "input_cost_per_token_priority": 5.5e-07, + "input_cost_per_token_priority": 4.4e-07, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, @@ -7143,7 +7179,7 @@ "mode": "chat", "output_cost_per_token": 1.32e-06, "output_cost_per_token_above_272k_tokens": 1.98e-06, - "output_cost_per_token_priority": 3.3e-06, + "output_cost_per_token_priority": 2.64e-06, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -30721,6 +30757,152 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "mistral/ministral-14b-2512": { + "input_cost_per_token": 2e-07, + "litellm_provider": "mistral", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2e-07, + "source": "https://docs.mistral.ai/models/ministral-3-14b-25-12", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "mistral/ministral-14b-latest": { + "input_cost_per_token": 2e-07, + "litellm_provider": "mistral", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2e-07, + "source": "https://docs.mistral.ai/models/ministral-3-14b-25-12", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "mistral/ministral-3b-2512": { + "input_cost_per_token": 1e-07, + "litellm_provider": "mistral", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1e-07, + "source": "https://docs.mistral.ai/models/ministral-3-3b-25-12", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "mistral/ministral-3b-latest": { + "input_cost_per_token": 1e-07, + "litellm_provider": "mistral", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1e-07, + "source": "https://docs.mistral.ai/models/ministral-3-3b-25-12", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "mistral/mistral-embed-2312": { + "input_cost_per_token": 1e-07, + "litellm_provider": "mistral", + "max_input_tokens": 8192, + "max_tokens": 8192, + "mode": "embedding", + "source": "https://docs.mistral.ai/models/mistral-embed-23-12" + }, + "mistral/mistral-medium-3": { + "input_cost_per_token": 1.5e-06, + "litellm_provider": "mistral", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 7.5e-06, + "source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "mistral/voxtral-mini-transcribe-realtime-latest": { + "input_cost_per_second": 0.0001, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/model-cards/voxtral-mini-transcribe-realtime-26-02", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-mini-tts-latest": { + "litellm_provider": "mistral", + "mode": "audio_speech", + "output_cost_per_character": 1.6e-05, + "source": "https://docs.mistral.ai/models/model-cards/voxtral-tts-26-03", + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_output": true + }, + "mistral/voxtral-small-2507": { + "input_cost_per_second": 6.666666666666667e-05, + "input_cost_per_token": 1e-07, + "litellm_provider": "mistral", + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 4e-07, + "source": "https://docs.mistral.ai/models/voxtral-small-25-07", + "supports_audio_input": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "mistral/voxtral-small-latest": { + "input_cost_per_second": 6.666666666666667e-05, + "input_cost_per_token": 1e-07, + "litellm_provider": "mistral", + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 4e-07, + "source": "https://docs.mistral.ai/models/voxtral-small-25-07", + "supports_audio_input": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, "mistral/zai-glm-5-2": { "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.4e-06, @@ -31191,9 +31373,9 @@ "mistral/ministral-3-3b-2512": { "input_cost_per_token": 1e-07, "litellm_provider": "mistral", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://mistral.ai/pricing", @@ -38342,6 +38524,7 @@ "supports_tool_choice": true }, "together_ai/Qwen/Qwen3.5-397B-A17B": { + "cache_read_input_token_cost": 3.5e-07, "deprecation_date": "2026-06-29", "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", @@ -38351,10 +38534,12 @@ "source": "https://www.together.ai/models/Qwen/Qwen3.5-397B-A17B", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/MiniMaxAI/MiniMax-M3": { + "cache_read_input_token_cost": 6e-08, "input_cost_per_token": 3e-07, "litellm_provider": "together_ai", "max_input_tokens": 524288, @@ -38365,6 +38550,7 @@ "source": "https://docs.together.ai/docs/serverless-models", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -38408,6 +38594,7 @@ "supports_reasoning": true }, "together_ai/Qwen/Qwen3.7-Max": { + "cache_read_input_token_cost": 1.3e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "together_ai", "max_input_tokens": 1000000, @@ -38415,7 +38602,8 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 3.75e-06, - "source": "https://docs.together.ai/docs/serverless-models" + "source": "https://docs.together.ai/docs/serverless-models", + "supports_prompt_caching": true }, "together_ai/Qwen/Qwen3.7-Plus": { "input_cost_per_token": 3.2e-07, @@ -38428,6 +38616,7 @@ "source": "https://docs.together.ai/docs/serverless-models" }, "together_ai/Qwen/Qwen3.8-2.4T-A95B": { + "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2.5e-06, "litellm_provider": "together_ai", "max_input_tokens": 1010000, @@ -38435,7 +38624,8 @@ "max_tokens": 1010000, "mode": "chat", "output_cost_per_token": 6.25e-06, - "source": "https://docs.together.ai/docs/serverless-models" + "source": "https://docs.together.ai/docs/serverless-models", + "supports_prompt_caching": true }, "together_ai/arize-ai/qwen-2-1.5b-instruct": { "input_cost_per_token": 1e-07, @@ -38448,6 +38638,7 @@ "source": "https://docs.together.ai/docs/serverless-models" }, "together_ai/deepseek-ai/DeepSeek-V4-Flash-0731": { + "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 1.4e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, @@ -38458,10 +38649,12 @@ "source": "https://docs.together.ai/docs/serverless-models", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/deepseek-ai/DeepSeek-V4-Pro": { + "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 1.74e-06, "litellm_provider": "together_ai", "max_input_tokens": 512000, @@ -38472,11 +38665,13 @@ "source": "https://docs.together.ai/docs/serverless-models", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813": { + "cache_read_input_token_cost": 1.3e-07, "input_cost_per_token": 1.32e-06, "litellm_provider": "together_ai", "max_input_tokens": 1048576, @@ -38487,6 +38682,7 @@ "source": "https://docs.together.ai/docs/serverless-models", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true }, @@ -38538,6 +38734,7 @@ "source": "https://docs.together.ai/docs/serverless-models" }, "together_ai/meta-models/Muse-Glimmer-30B": { + "cache_read_input_token_cost": 4e-08, "input_cost_per_token": 3.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, @@ -38545,9 +38742,11 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-06, - "source": "https://docs.together.ai/docs/serverless-models" + "source": "https://docs.together.ai/docs/serverless-models", + "supports_prompt_caching": true }, "together_ai/moonshotai/Kimi-K2.7-Code": { + "cache_read_input_token_cost": 1.9e-07, "input_cost_per_token": 9.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, @@ -38558,11 +38757,13 @@ "source": "https://docs.together.ai/docs/serverless-models", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true }, "together_ai/moonshotai/Kimi-K3": { + "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "litellm_provider": "together_ai", "max_input_tokens": 1048576, @@ -38573,12 +38774,14 @@ "source": "https://docs.together.ai/docs/serverless-models", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true }, "together_ai/nvidia/nemotron-3-ultra-550b-a55b": { + "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 512288, @@ -38589,6 +38792,7 @@ "source": "https://docs.together.ai/docs/serverless-models", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true @@ -38604,6 +38808,7 @@ "source": "https://docs.together.ai/docs/serverless-models" }, "together_ai/thinkingmachines/Inkling": { + "cache_read_input_token_cost": 1.7e-07, "input_cost_per_token": 1e-06, "litellm_provider": "together_ai", "max_input_tokens": 524288, @@ -38614,10 +38819,12 @@ "source": "https://docs.together.ai/docs/serverless-models", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/thinkingmachines/Inkling-Small": { + "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 5e-07, "litellm_provider": "together_ai", "max_input_tokens": 524288, @@ -38625,9 +38832,11 @@ "max_tokens": 524288, "mode": "chat", "output_cost_per_token": 1.2e-06, - "source": "https://docs.together.ai/docs/serverless-models" + "source": "https://docs.together.ai/docs/serverless-models", + "supports_prompt_caching": true }, "together_ai/zai-org/GLM-5.2": { + "cache_read_input_token_cost": 2.6e-07, "input_cost_per_token": 1.4e-06, "litellm_provider": "together_ai", "max_input_tokens": 1048575, @@ -38638,6 +38847,7 @@ "source": "https://docs.together.ai/docs/serverless-models", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 3d0921c6fd8..578282efee5 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -1727,6 +1727,84 @@ def test_azure_ai_cache_cost_calculation(_local_model_cost_map): ), f"Output cost mismatch: got {output_cost}, expected {expected_output_cost}" +AZURE_GPT_5_6_MAP_KEYS = ( + "azure/gpt-5.6", + "azure/gpt-5.6-sol", + "azure/gpt-5.6-terra", + "azure/gpt-5.6-luna", + "azure/us/gpt-5.6", + "azure/us/gpt-5.6-sol", + "azure/us/gpt-5.6-terra", + "azure/us/gpt-5.6-luna", + "azure/eu/gpt-5.6", + "azure/eu/gpt-5.6-sol", + "azure/eu/gpt-5.6-terra", + "azure/eu/gpt-5.6-luna", +) + + +def test_azure_gpt_5_6_cache_write_tokens_are_billed(_local_model_cost_map): + """ + Azure bills gpt-5.6 prompt cache writes at 1.25x the input rate, but the + azure entries carried no ``cache_creation_input_token_cost``, so + cache-write tokens were billed at the plain input rate instead. + """ + from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token + from litellm.types.utils import PromptTokensDetailsWrapper, Usage + + usage = Usage( + completion_tokens=100, + prompt_tokens=2000, + total_tokens=2100, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=0, text_tokens=687), + cache_creation_input_tokens=1313, + ) + + input_cost, output_cost = generic_cost_per_token( + model="azure/gpt-5.6-luna", usage=usage, custom_llm_provider="azure" + ) + + assert input_cost == pytest.approx(687 * 2e-07 + 1313 * 2.5e-07) + assert output_cost == pytest.approx(100 * 1.2e-06) + + +@pytest.mark.parametrize("model", AZURE_GPT_5_6_MAP_KEYS) +def test_azure_gpt_5_6_rates_match_azure_price_page(_local_model_cost_map, model): + """ + Per the Azure retail price API (2026-08-26): cache writes cost 1.25x input + on every published gpt-5.6 meter, and Data Zone standard and priority + rates cost 1.1x Global (us/eu priority rates previously sat at 1.25x). + Azure publishes no long-context priority meters, so the + ``*_above_272k_tokens_priority`` suffix is excluded. + """ + entry = litellm.model_cost[model] + input_keys = [ + key + for key in entry + if key.startswith("input_cost_per_token") + and not key.endswith("_above_272k_tokens_priority") + ] + assert input_keys + for key in input_keys: + suffix = key[len("input_cost_per_token") :] + assert entry["cache_creation_input_token_cost" + suffix] == pytest.approx( + entry[key] * 1.25 + ), key + + zone = model.split("/")[1] + if zone in ("us", "eu"): + global_entry = litellm.model_cost["azure/" + model.split("/", 2)[2]] + prefixes = ("input_cost_per_token", "output_cost_per_token", "cache_read", "cache_creation") + token_cost_keys = [ + key + for key in entry + if key.startswith(prefixes) and not key.endswith("_above_272k_tokens_priority") + ] + assert len(token_cost_keys) >= 9 + for key in token_cost_keys: + assert entry[key] == pytest.approx(global_entry[key] * 1.1), key + + def test_vertex_regional_deployment_costs_uplift_over_global(monkeypatch): """ Regression for https://github.com/BerriAI/litellm/issues/34393: two Vertex From d53c2c818b962232699a9126e30c6459d53f69a0 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 26 Aug 2026 19:08:21 -0700 Subject: [PATCH 14/29] test(e2e): cover key generate and update on the Admin UI path The two `surface: ui` cells in the coverage registry, mgmt.key.generate.happy_path and mgmt.key.update.happy_path, had no covering test. The existing key tests all call /key/generate and /key/update with the master key, which is not how the dashboard reaches those routes: an admin signs in, the proxy mints a UI session key scoped to the litellm-dashboard team, and every subsequent create or edit is written under that session key. TestDashboardKeyRoutes covers that path. The first test signs in through /v2/login, decodes the master-key-signed session JWT the way the dashboard does, and asserts the minted key carries the admin role and the dashboard team, then that it can actually read the key inventory the Virtual Keys page renders. The second edits a key under that session key and asserts both halves of the contract: /key/info reports the new models and limits with the alias untouched, and the gateway flips enforcement to match. ManagementClient grows dashboard_login plus caller-aware key_list and update_key, so a test can say who is driving a management route instead of always implying the master key. update_key returns its Result rather than raising, which lets a caller poll a route that is only transiently refusing; a freshly minted session key is briefly unauthorized while the auth cache picks up its user row. --- tests/e2e/management/management_client.py | 104 ++++++++++++++++---- tests/e2e/management/test_management_e2e.py | 101 ++++++++++++++++++- tests/e2e/models.py | 26 ++++- 3 files changed, 207 insertions(+), 24 deletions(-) diff --git a/tests/e2e/management/management_client.py b/tests/e2e/management/management_client.py index b2bd41e19ba..d3be1e9f39c 100644 --- a/tests/e2e/management/management_client.py +++ b/tests/e2e/management/management_client.py @@ -9,8 +9,21 @@ from __future__ import annotations import time from dataclasses import dataclass +import jwt + +from e2e_config import MASTER_KEY from proxy_client import ProxyClient -from e2e_http import NoBody, ProbeResult, Result, StreamingResponse, Success, UnknownApiError, unwrap +from e2e_http import ( + AuthHeaders, + NetworkError, + NoBody, + ProbeResult, + Result, + StreamingResponse, + Success, + UnknownApiError, + unwrap, +) from models import ( ChatBody, ChatMessage, @@ -50,6 +63,9 @@ from models import ( TeamNewBody, TeamNewResponse, TeamUpdateBody, + UiLoginBody, + UiLoginResponse, + UiSessionClaims, UserDeleteBody, UserDeleteResponse, UserInfoParams, @@ -63,38 +79,59 @@ from models import ( MODEL_ACCESS_DENIED_MARKER = "key_model_access_denied" ROUTE_NOT_ALLOWED_MARKER = "not allowed to call this route" +DASHBOARD_SESSION_TEAM_ID = "litellm-dashboard" _TEAM_READY_ATTEMPTS = 15 _TEAM_READY_SLEEP_SECONDS = 0.4 +_KEY_WRITE_ATTEMPTS = 5 +_TRANSIENT_BACKEND_MARKERS = ("connecting to redis", "name resolution") + + +@dataclass(frozen=True, slots=True) +class DashboardSession: + """What a dashboard sign-in hands the Admin UI: the session key it sends as + its bearer on every subsequent call, the claims it renders the signed-in user + from, and where it lands the browser.""" + + session_key: str + claims: UiSessionClaims + redirect_url: str @dataclass(frozen=True, slots=True) class ManagementClient: proxy: ProxyClient + master_key: str def llm_only_key(self) -> str: return self.proxy.generate_key(KeyGenerateBody(models=[], allowed_routes=["llm_api_routes"])) - def update_key_models(self, key: str, models: list[str]) -> None: - last: Result[NoBody] | None = None - for attempt in range(5): + def update_key(self, body: KeyUpdateBody, *, caller_key: str | None = None) -> Result[NoBody]: + """POST /key/update. `caller_key` is who is editing: the master key by + default, or a virtual key (the dashboard edits under the session key its + sign-in minted, never the master key). Returns the outcome rather than + unwrapping it, so a caller can poll a route that is only transiently + refusing; `update_key_models` is the unwrapping shorthand.""" + headers = self.proxy.transport.master if caller_key is None else self.proxy.transport.bearer(caller_key) + last: Result[NoBody] = NetworkError(message="/key/update was never attempted") + for attempt in range(_KEY_WRITE_ATTEMPTS): last = self.proxy.transport.post( "/key/update", - headers=self.proxy.transport.master, - json=KeyUpdateBody(key=key, models=models), + headers=headers, + json=body, response_type=NoBody, ) match last: - case Success(): - return - case UnknownApiError(body=body) if ( - "connecting to redis" in body.lower() or "name resolution" in body.lower() + case UnknownApiError(body=error_body) if any( + marker in error_body.lower() for marker in _TRANSIENT_BACKEND_MARKERS ): time.sleep(0.5 * (attempt + 1)) continue case _: break - assert last is not None - raise AssertionError(last) + return last + + def update_key_models(self, key: str, models: list[str]) -> None: + _ = unwrap(self.update_key(KeyUpdateBody(key=key, models=models))) def delete_key_strict(self, key: str) -> None: """Strict delete for the act phase of a test: a failed delete is a hard @@ -150,15 +187,42 @@ class ManagementClient: ) ).key + def key_list(self, key_alias: str, *, caller_key: str | None = None) -> Result[KeyListResponse]: + """GET /key/list, the Virtual Keys page's own inventory call. `caller_key` is + who is asking: the master key by default, or a virtual key.""" + headers = self.proxy.transport.master if caller_key is None else self.proxy.transport.bearer(caller_key) + return self.proxy.transport.get( + "/key/list", + headers=headers, + params=KeyListParams(key_alias=key_alias), + response_type=KeyListResponse, + ) + def key_alias_count(self, key_alias: str) -> int: - return unwrap( - self.proxy.transport.get( - "/key/list", - headers=self.proxy.transport.master, - params=KeyListParams(key_alias=key_alias), - response_type=KeyListResponse, + return unwrap(self.key_list(key_alias)).total_count + + def dashboard_login(self, username: str, password: str) -> DashboardSession: + """POST /v2/login, the call the Admin UI's sign-in form makes. + + The proxy authenticates the credentials, mints a UI session key for the + signed-in user, and hands it back inside a JWT signed with the master key. + Decoding that JWT is the only way to reach the session key, and it is what + the dashboard itself does before it can call a single management route.""" + response = unwrap( + self.proxy.transport.post( + "/v2/login", + headers=AuthHeaders(), + json=UiLoginBody(username=username, password=password), + response_type=UiLoginResponse, ) - ).total_count + ) + decoded: object = jwt.decode(response.token, self.master_key, algorithms=["HS256"]) + claims = UiSessionClaims.model_validate(decoded) + return DashboardSession( + session_key=claims.key, + claims=claims, + redirect_url=response.redirect_url, + ) def create_team(self, body: TeamNewBody) -> str: team_id = unwrap( @@ -465,4 +529,4 @@ class ManagementClient: def build_client(proxy: ProxyClient) -> ManagementClient: - return ManagementClient(proxy=proxy) + return ManagementClient(proxy=proxy, master_key=MASTER_KEY) diff --git a/tests/e2e/management/test_management_e2e.py b/tests/e2e/management/test_management_e2e.py index 9b398963ac9..a381f320cdc 100644 --- a/tests/e2e/management/test_management_e2e.py +++ b/tests/e2e/management/test_management_e2e.py @@ -15,15 +15,16 @@ from collections.abc import Callable import pytest -from e2e_config import unique_marker -from e2e_http import StreamingResponse +from e2e_config import UI_PASSWORD, UI_USERNAME, unique_marker +from e2e_http import StreamingResponse, Success from lifecycle import ResourceManager from management_client import ( + DASHBOARD_SESSION_TEAM_ID, MODEL_ACCESS_DENIED_MARKER, ROUTE_NOT_ALLOWED_MARKER, ManagementClient, ) -from models import KeyGenerateBody, OrgInfoResponse, OrgNewBody, OrgUpdateBody, TagListEntry, TagNewBody, TeamNewBody, TeamUpdateBody, UserNewBody, UserUpdateBody, LiteLLMParamsBody, ModelInfoEntry +from models import KeyGenerateBody, KeyUpdateBody, OrgInfoResponse, OrgNewBody, OrgUpdateBody, TagListEntry, TagNewBody, TeamNewBody, TeamUpdateBody, UserNewBody, UserUpdateBody, LiteLLMParamsBody, ModelInfoEntry pytestmark = pytest.mark.e2e @@ -199,6 +200,100 @@ class TestKeyRoutes: return True if client.proxy.key_info(key).blocked else None _ = _poll(client, blocked, "/key/info never reported the key blocked after /key/block before the deadline") + + +class TestDashboardKeyRoutes: + """The /key writes as the Admin UI makes them. Signing in mints the session key + the dashboard authenticates with, and every key an admin creates or edits in the + browser is written under that session key rather than the master key, so these + are the same routes the API-surface tests cover with a different caller.""" + + @pytest.mark.covers("mgmt.key.generate.happy_path") + def test_sign_in_mints_a_session_key_that_drives_the_dashboard( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + alias = f"e2e-mgmt-uisession-{unique_marker()}" + _ = _generate_key(client, resources, KeyGenerateBody(models=["gemini-2.5-flash"], key_alias=alias)) + + session = client.dashboard_login(UI_USERNAME, UI_PASSWORD) + resources.defer(lambda: client.proxy.delete_key(session.session_key)) + + assert session.claims.login_method == "username_password", ( + f"/v2/login reports login_method {session.claims.login_method!r} for a username/password sign-in" + ) + assert session.claims.user_role == "proxy_admin", ( + f"/v2/login reports user_role {session.claims.user_role!r} for the admin credentials, expected 'proxy_admin'" + ) + assert session.redirect_url.endswith("/ui?login=success"), ( + f"/v2/login sends the browser to {session.redirect_url!r} instead of the dashboard" + ) + + info = client.proxy.key_info(session.session_key) + assert info.team_id == DASHBOARD_SESSION_TEAM_ID, ( + f"the minted session key reports team_id {info.team_id!r}, expected the dashboard's " + f"{DASHBOARD_SESSION_TEAM_ID!r}" + ) + + def dashboard_lists_the_key() -> bool | None: + match client.key_list(alias, caller_key=session.session_key): + case Success(data=listing) if listing.total_count == 1: + return True + case _: + return None + + _ = _poll( + client, + dashboard_lists_the_key, + f"the session key never saw {alias!r} in /key/list before the deadline, so the dashboard " + "would render no keys", + ) + + @pytest.mark.covers("mgmt.key.update.happy_path") + def test_editing_a_key_from_the_dashboard_persists_and_is_enforced( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + alias = f"e2e-mgmt-uiedit-{unique_marker()}" + target = _generate_key( + client, + resources, + KeyGenerateBody(models=["gemini-2.5-flash"], key_alias=alias, tpm_limit=100, rpm_limit=200), + ) + _poll_chat_ok(client, target, "gemini-2.5-flash") + _assert_model_denied(client.chat_status(target, "gpt-5.5", f"say hi {unique_marker()}"), "gpt-5.5") + + session = client.dashboard_login(UI_USERNAME, UI_PASSWORD) + resources.defer(lambda: client.proxy.delete_key(session.session_key)) + + def dashboard_saves_the_edit() -> bool | None: + match client.update_key( + KeyUpdateBody(key=target, models=["gpt-5.5"], tpm_limit=300, rpm_limit=400), + caller_key=session.session_key, + ): + case Success(): + return True + case _: + return None + + _ = _poll( + client, + dashboard_saves_the_edit, + "the dashboard session key was never accepted on /key/update before the deadline", + ) + + info = client.proxy.key_info(target) + assert info.models == ["gpt-5.5"], ( + f"/key/info reports models {info.models} after the dashboard edit to ['gpt-5.5']" + ) + assert info.tpm_limit == 300, f"/key/info reports tpm_limit {info.tpm_limit} after the dashboard edit to 300" + assert info.rpm_limit == 400, f"/key/info reports rpm_limit {info.rpm_limit} after the dashboard edit to 400" + assert info.key_alias == alias, ( + f"the dashboard edit renamed the key to {info.key_alias!r}, it should still be {alias!r}" + ) + + _poll_model_access_granted(client, target, "gpt-5.5") + _poll_chat_denied(client, target, "gemini-2.5-flash") + + class TestKeyRegeneration: @pytest.mark.covers("mgmt.key.regenerate.happy_path") def test_regenerate_rotates_to_a_working_new_key( diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 95a02b58824..0dc8c515720 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -892,7 +892,10 @@ class CredentialCreateResponse(BaseModel): class KeyUpdateBody(BaseModel): key: str - models: list[str] + models: list[str] | None = None + key_alias: str | None = None + tpm_limit: int | None = None + rpm_limit: int | None = None class KeyBlockBody(BaseModel): @@ -907,6 +910,27 @@ class KeyListResponse(BaseModel): total_count: int +# ---------- admin UI session ---------- + + +class UiLoginBody(BaseModel): + username: str + password: str + + +class UiLoginResponse(BaseModel): + token: str + redirect_url: str + + +class UiSessionClaims(BaseModel): + user_id: str + key: str + user_role: str + login_method: Literal["sso", "username_password"] + exp: int + + class TeamMemberEntry(BaseModel): role: Literal["admin", "user"] user_id: str From 8741e8a1adc940da1c78d6c0f7f2f1a5efb10475 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 26 Aug 2026 22:52:44 -0700 Subject: [PATCH 15/29] test(e2e): create the key under the dashboard session key The test claiming mgmt.key.generate.happy_path signed in and then only read /key/list, so nothing proved the session key an admin's sign-in mints is actually accepted on /key/generate. It now does what an admin filling in Create New Key does: POST /key/generate under the session key, read the new key back from /key/info, see it in the dashboard's own /key/list, and drive real traffic through it to confirm its model scope is enforced. Adds ManagementClient.generate_key with the same caller_key seam update_key and key_list already use, so the suite can call the route as the master key or as a virtual key. Also wraps the over-long models import. Refusing the dashboard session key on /key/generate turns only this test red; the master-key generate, the key edit, and regenerate stay green. --- tests/e2e/management/management_client.py | 14 +++++ tests/e2e/management/test_management_e2e.py | 64 ++++++++++++++++++--- 2 files changed, 69 insertions(+), 9 deletions(-) diff --git a/tests/e2e/management/management_client.py b/tests/e2e/management/management_client.py index d3be1e9f39c..387280c8023 100644 --- a/tests/e2e/management/management_client.py +++ b/tests/e2e/management/management_client.py @@ -105,6 +105,20 @@ class ManagementClient: def llm_only_key(self) -> str: return self.proxy.generate_key(KeyGenerateBody(models=[], allowed_routes=["llm_api_routes"])) + def generate_key(self, body: KeyGenerateBody, *, caller_key: str | None = None) -> Result[KeyGenerateResponse]: + """POST /key/generate. `caller_key` is who is creating the key: the master + key by default, or a virtual key (an admin filling in Create New Key on the + dashboard creates it under the session key their sign-in minted). Returns + the outcome rather than unwrapping it, so a caller can poll a route that is + only transiently refusing.""" + headers = self.proxy.transport.master if caller_key is None else self.proxy.transport.bearer(caller_key) + return self.proxy.transport.post( + "/key/generate", + headers=headers, + json=body, + response_type=KeyGenerateResponse, + ) + def update_key(self, body: KeyUpdateBody, *, caller_key: str | None = None) -> Result[NoBody]: """POST /key/update. `caller_key` is who is editing: the master key by default, or a virtual key (the dashboard edits under the session key its diff --git a/tests/e2e/management/test_management_e2e.py b/tests/e2e/management/test_management_e2e.py index a381f320cdc..a56eb853823 100644 --- a/tests/e2e/management/test_management_e2e.py +++ b/tests/e2e/management/test_management_e2e.py @@ -24,7 +24,21 @@ from management_client import ( ROUTE_NOT_ALLOWED_MARKER, ManagementClient, ) -from models import KeyGenerateBody, KeyUpdateBody, OrgInfoResponse, OrgNewBody, OrgUpdateBody, TagListEntry, TagNewBody, TeamNewBody, TeamUpdateBody, UserNewBody, UserUpdateBody, LiteLLMParamsBody, ModelInfoEntry +from models import ( + KeyGenerateBody, + KeyUpdateBody, + LiteLLMParamsBody, + ModelInfoEntry, + OrgInfoResponse, + OrgNewBody, + OrgUpdateBody, + TagListEntry, + TagNewBody, + TeamNewBody, + TeamUpdateBody, + UserNewBody, + UserUpdateBody, +) pytestmark = pytest.mark.e2e @@ -209,12 +223,9 @@ class TestDashboardKeyRoutes: are the same routes the API-surface tests cover with a different caller.""" @pytest.mark.covers("mgmt.key.generate.happy_path") - def test_sign_in_mints_a_session_key_that_drives_the_dashboard( + def test_creating_a_key_from_the_dashboard_persists_and_works( self, client: ManagementClient, resources: ResourceManager ) -> None: - alias = f"e2e-mgmt-uisession-{unique_marker()}" - _ = _generate_key(client, resources, KeyGenerateBody(models=["gemini-2.5-flash"], key_alias=alias)) - session = client.dashboard_login(UI_USERNAME, UI_PASSWORD) resources.defer(lambda: client.proxy.delete_key(session.session_key)) @@ -222,18 +233,50 @@ class TestDashboardKeyRoutes: f"/v2/login reports login_method {session.claims.login_method!r} for a username/password sign-in" ) assert session.claims.user_role == "proxy_admin", ( - f"/v2/login reports user_role {session.claims.user_role!r} for the admin credentials, expected 'proxy_admin'" + f"/v2/login reports user_role {session.claims.user_role!r} for the admin credentials, " + "expected 'proxy_admin'" ) assert session.redirect_url.endswith("/ui?login=success"), ( f"/v2/login sends the browser to {session.redirect_url!r} instead of the dashboard" ) - info = client.proxy.key_info(session.session_key) - assert info.team_id == DASHBOARD_SESSION_TEAM_ID, ( - f"the minted session key reports team_id {info.team_id!r}, expected the dashboard's " + session_info = client.proxy.key_info(session.session_key) + assert session_info.team_id == DASHBOARD_SESSION_TEAM_ID, ( + f"the minted session key reports team_id {session_info.team_id!r}, expected the dashboard's " f"{DASHBOARD_SESSION_TEAM_ID!r}" ) + alias = f"e2e-mgmt-uicreate-{unique_marker()}" + + def dashboard_creates_the_key() -> str | None: + match client.generate_key( + KeyGenerateBody(models=["gemini-2.5-flash"], key_alias=alias, tpm_limit=100), + caller_key=session.session_key, + ): + case Success(data=created): + return created.key + case _: + return None + + created = _poll( + client, + dashboard_creates_the_key, + "the dashboard session key was never accepted on /key/generate before the deadline", + ) + resources.defer(lambda: client.proxy.delete_key(created)) + + created_info = client.proxy.key_info(created) + assert created_info.key_alias == alias, ( + f"/key/info reports key_alias {created_info.key_alias!r} for the key the dashboard created, " + f"expected {alias!r}" + ) + assert created_info.models == ["gemini-2.5-flash"], ( + f"/key/info reports models {created_info.models} for the key the dashboard created" + ) + assert created_info.tpm_limit == 100, ( + f"/key/info reports tpm_limit {created_info.tpm_limit} for the key the dashboard created, expected 100" + ) + def dashboard_lists_the_key() -> bool | None: match client.key_list(alias, caller_key=session.session_key): case Success(data=listing) if listing.total_count == 1: @@ -248,6 +291,9 @@ class TestDashboardKeyRoutes: "would render no keys", ) + _poll_chat_ok(client, created, "gemini-2.5-flash") + _assert_model_denied(client.chat_status(created, "gpt-5.5", f"say hi {unique_marker()}"), "gpt-5.5") + @pytest.mark.covers("mgmt.key.update.happy_path") def test_editing_a_key_from_the_dashboard_persists_and_is_enforced( self, client: ManagementClient, resources: ResourceManager From 5d26ae0fcd77f50ec5e210b8b529dc77370da594 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2026 13:16:12 +0000 Subject: [PATCH 16/29] fix(model_prices): absorb Databricks/Z.AI and xAI registry PRs, add Together and Azure deprecation dates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 203 ++++++++---------- model_prices_and_context_window.json | 203 ++++++++---------- .../test_databricks_cost_calculator.py | 2 + .../llms/xai/test_xai_model_registry.py | 75 +++++++ .../test_together_ai_model_metadata.py | 8 +- 5 files changed, 261 insertions(+), 230 deletions(-) create mode 100644 tests/test_litellm/llms/xai/test_xai_model_registry.py diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index cfa06ff5f81..0d867955544 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -15167,6 +15167,34 @@ "output_dbu_cost_per_token": 7.143e-06, "source": "https://www.databricks.com/product/pricing/foundation-model-serving" }, + "databricks/databricks-glm-5-2": { + "cache_creation_input_token_cost": 1.4e-06, + "cache_read_input_token_cost": 2.5998e-07, + "input_cost_per_token": 1.4e-06, + "input_dbu_cost_per_token": 2e-05, + "litellm_provider": "databricks", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "max_tokens": 131072, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference." + }, + "mode": "chat", + "output_cost_per_token": 4.39999e-06, + "output_dbu_cost_per_token": 6.2857e-05, + "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, "databricks/databricks-gpt-5": { "cache_creation_input_token_cost": 1.24999e-06, "cache_read_input_token_cost": 1.2502e-07, @@ -15434,6 +15462,35 @@ "output_vector_size": 1024, "source": "https://www.databricks.com/product/pricing/foundation-model-serving" }, + "databricks/databricks-kimi-k3": { + "cache_creation_input_token_cost": 2.99999e-06, + "cache_read_input_token_cost": 3.0002e-07, + "input_cost_per_token": 2.99999e-06, + "input_dbu_cost_per_token": 4.2857e-05, + "litellm_provider": "databricks", + "max_input_tokens": 1000000, + "max_output_tokens": 1048576, + "max_tokens": 1048576, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference." + }, + "mode": "chat", + "output_cost_per_token": 1.500002e-05, + "output_dbu_cost_per_token": 0.000214286, + "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "databricks/databricks-llama-2-70b-chat": { "cache_creation_input_token_cost": 5.0001e-07, "cache_read_input_token_cost": 5.0001e-07, @@ -38329,7 +38386,7 @@ "max_output_tokens": 20480, "max_tokens": 20480, "metadata": { - "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro" + "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813" }, "mode": "chat", "output_cost_per_token": 7e-06, @@ -38358,7 +38415,7 @@ "max_output_tokens": 8192, "max_tokens": 8192, "metadata": { - "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro" + "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813" }, "mode": "chat", "output_cost_per_token": 1.25e-06, @@ -38373,7 +38430,7 @@ "litellm_provider": "together_ai", "max_tokens": 16384, "metadata": { - "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro" + "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813" }, "mode": "chat", "output_cost_per_token": 1.7e-06, @@ -38794,6 +38851,7 @@ "supports_tool_choice": true }, "together_ai/deepseek-ai/DeepSeek-V4-Pro": { + "deprecation_date": "2026-08-27", "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 1.74e-06, "litellm_provider": "together_ai", @@ -38886,6 +38944,7 @@ "supports_prompt_caching": true }, "together_ai/moonshotai/Kimi-K2.7-Code": { + "deprecation_date": "2026-08-27", "cache_read_input_token_cost": 1.9e-07, "input_cost_per_token": 9.5e-07, "litellm_provider": "together_ai", @@ -38921,6 +38980,7 @@ "supports_vision": true }, "together_ai/nvidia/nemotron-3-ultra-550b-a55b": { + "deprecation_date": "2026-08-27", "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", @@ -38938,6 +38998,7 @@ "supports_tool_choice": true }, "together_ai/pearl-ai/gemma-4-31b-it": { + "deprecation_date": "2026-08-27", "input_cost_per_token": 2.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, @@ -43788,85 +43849,6 @@ "/v1/audio/transcriptions" ] }, - "xai/grok-2": { - "input_cost_per_token": 2e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_web_search": true - }, - "xai/grok-2-1212": { - "input_cost_per_token": 2e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_web_search": true - }, - "xai/grok-2-latest": { - "input_cost_per_token": 2e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_web_search": true - }, - "xai/grok-2-vision": { - "input_cost_per_image": 2e-06, - "input_cost_per_token": 2e-06, - "litellm_provider": "xai", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, - "xai/grok-2-vision-1212": { - "deprecation_date": "2026-02-28", - "input_cost_per_image": 2e-06, - "input_cost_per_token": 2e-06, - "litellm_provider": "xai", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, - "xai/grok-2-vision-latest": { - "input_cost_per_image": 2e-06, - "input_cost_per_token": 2e-06, - "litellm_provider": "xai", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, "xai/grok-3": { "cache_read_input_token_cost": 7.5e-07, "input_cost_per_token": 3e-06, @@ -44252,7 +44234,7 @@ "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, - "mode": "chat", + "mode": "responses", "output_cost_per_token": 2.5e-06, "source": "https://docs.x.ai/docs/models", "supports_function_calling": true, @@ -44264,7 +44246,10 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, - "supports_response_schema": true + "supports_response_schema": true, + "supported_endpoints": [ + "/v1/responses" + ] }, "xai/grok-4.20-beta-0309-reasoning": { "cache_read_input_token_cost": 2e-07, @@ -44433,19 +44418,6 @@ "supports_vision": true, "supports_web_search": true }, - "xai/grok-beta": { - "input_cost_per_token": 5e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, "xai/grok-code-fast": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 1e-06, @@ -44509,20 +44481,6 @@ "supports_vision": true, "deprecation_date": "2026-05-15" }, - "xai/grok-vision-beta": { - "input_cost_per_image": 5e-06, - "input_cost_per_token": 5e-06, - "litellm_provider": "xai", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, "zai.glm-4.7": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock_converse", @@ -44581,6 +44539,21 @@ "supports_tool_choice": true, "source": "https://docs.z.ai/guides/overview/pricing" }, + "zai/glm-5.3": { + "cache_creation_input_token_cost": 0, + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "zai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://docs.z.ai/guides/overview/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "zai/glm-5.1": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 2.6e-07, @@ -44786,6 +44759,7 @@ ] }, "azure/sora-2": { + "deprecation_date": "2026-10-15", "litellm_provider": "azure", "mode": "video_generation", "output_cost_per_video_per_second": 0.1, @@ -51240,7 +51214,7 @@ "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, - "mode": "chat", + "mode": "responses", "output_cost_per_token": 2.5e-06, "source": "https://docs.x.ai/docs/models", "supports_function_calling": true, @@ -51252,7 +51226,10 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, - "supports_response_schema": true + "supports_response_schema": true, + "supported_endpoints": [ + "/v1/responses" + ] }, "xai/grok-build-0.1": { "cache_read_input_token_cost": 2e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index cfa06ff5f81..0d867955544 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -15167,6 +15167,34 @@ "output_dbu_cost_per_token": 7.143e-06, "source": "https://www.databricks.com/product/pricing/foundation-model-serving" }, + "databricks/databricks-glm-5-2": { + "cache_creation_input_token_cost": 1.4e-06, + "cache_read_input_token_cost": 2.5998e-07, + "input_cost_per_token": 1.4e-06, + "input_dbu_cost_per_token": 2e-05, + "litellm_provider": "databricks", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "max_tokens": 131072, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference." + }, + "mode": "chat", + "output_cost_per_token": 4.39999e-06, + "output_dbu_cost_per_token": 6.2857e-05, + "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, "databricks/databricks-gpt-5": { "cache_creation_input_token_cost": 1.24999e-06, "cache_read_input_token_cost": 1.2502e-07, @@ -15434,6 +15462,35 @@ "output_vector_size": 1024, "source": "https://www.databricks.com/product/pricing/foundation-model-serving" }, + "databricks/databricks-kimi-k3": { + "cache_creation_input_token_cost": 2.99999e-06, + "cache_read_input_token_cost": 3.0002e-07, + "input_cost_per_token": 2.99999e-06, + "input_dbu_cost_per_token": 4.2857e-05, + "litellm_provider": "databricks", + "max_input_tokens": 1000000, + "max_output_tokens": 1048576, + "max_tokens": 1048576, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference." + }, + "mode": "chat", + "output_cost_per_token": 1.500002e-05, + "output_dbu_cost_per_token": 0.000214286, + "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "databricks/databricks-llama-2-70b-chat": { "cache_creation_input_token_cost": 5.0001e-07, "cache_read_input_token_cost": 5.0001e-07, @@ -38329,7 +38386,7 @@ "max_output_tokens": 20480, "max_tokens": 20480, "metadata": { - "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro" + "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813" }, "mode": "chat", "output_cost_per_token": 7e-06, @@ -38358,7 +38415,7 @@ "max_output_tokens": 8192, "max_tokens": 8192, "metadata": { - "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro" + "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813" }, "mode": "chat", "output_cost_per_token": 1.25e-06, @@ -38373,7 +38430,7 @@ "litellm_provider": "together_ai", "max_tokens": 16384, "metadata": { - "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro" + "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813" }, "mode": "chat", "output_cost_per_token": 1.7e-06, @@ -38794,6 +38851,7 @@ "supports_tool_choice": true }, "together_ai/deepseek-ai/DeepSeek-V4-Pro": { + "deprecation_date": "2026-08-27", "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 1.74e-06, "litellm_provider": "together_ai", @@ -38886,6 +38944,7 @@ "supports_prompt_caching": true }, "together_ai/moonshotai/Kimi-K2.7-Code": { + "deprecation_date": "2026-08-27", "cache_read_input_token_cost": 1.9e-07, "input_cost_per_token": 9.5e-07, "litellm_provider": "together_ai", @@ -38921,6 +38980,7 @@ "supports_vision": true }, "together_ai/nvidia/nemotron-3-ultra-550b-a55b": { + "deprecation_date": "2026-08-27", "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", @@ -38938,6 +38998,7 @@ "supports_tool_choice": true }, "together_ai/pearl-ai/gemma-4-31b-it": { + "deprecation_date": "2026-08-27", "input_cost_per_token": 2.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, @@ -43788,85 +43849,6 @@ "/v1/audio/transcriptions" ] }, - "xai/grok-2": { - "input_cost_per_token": 2e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_web_search": true - }, - "xai/grok-2-1212": { - "input_cost_per_token": 2e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_web_search": true - }, - "xai/grok-2-latest": { - "input_cost_per_token": 2e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_web_search": true - }, - "xai/grok-2-vision": { - "input_cost_per_image": 2e-06, - "input_cost_per_token": 2e-06, - "litellm_provider": "xai", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, - "xai/grok-2-vision-1212": { - "deprecation_date": "2026-02-28", - "input_cost_per_image": 2e-06, - "input_cost_per_token": 2e-06, - "litellm_provider": "xai", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, - "xai/grok-2-vision-latest": { - "input_cost_per_image": 2e-06, - "input_cost_per_token": 2e-06, - "litellm_provider": "xai", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, "xai/grok-3": { "cache_read_input_token_cost": 7.5e-07, "input_cost_per_token": 3e-06, @@ -44252,7 +44234,7 @@ "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, - "mode": "chat", + "mode": "responses", "output_cost_per_token": 2.5e-06, "source": "https://docs.x.ai/docs/models", "supports_function_calling": true, @@ -44264,7 +44246,10 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, - "supports_response_schema": true + "supports_response_schema": true, + "supported_endpoints": [ + "/v1/responses" + ] }, "xai/grok-4.20-beta-0309-reasoning": { "cache_read_input_token_cost": 2e-07, @@ -44433,19 +44418,6 @@ "supports_vision": true, "supports_web_search": true }, - "xai/grok-beta": { - "input_cost_per_token": 5e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, "xai/grok-code-fast": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 1e-06, @@ -44509,20 +44481,6 @@ "supports_vision": true, "deprecation_date": "2026-05-15" }, - "xai/grok-vision-beta": { - "input_cost_per_image": 5e-06, - "input_cost_per_token": 5e-06, - "litellm_provider": "xai", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, "zai.glm-4.7": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock_converse", @@ -44581,6 +44539,21 @@ "supports_tool_choice": true, "source": "https://docs.z.ai/guides/overview/pricing" }, + "zai/glm-5.3": { + "cache_creation_input_token_cost": 0, + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "zai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://docs.z.ai/guides/overview/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "zai/glm-5.1": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 2.6e-07, @@ -44786,6 +44759,7 @@ ] }, "azure/sora-2": { + "deprecation_date": "2026-10-15", "litellm_provider": "azure", "mode": "video_generation", "output_cost_per_video_per_second": 0.1, @@ -51240,7 +51214,7 @@ "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, - "mode": "chat", + "mode": "responses", "output_cost_per_token": 2.5e-06, "source": "https://docs.x.ai/docs/models", "supports_function_calling": true, @@ -51252,7 +51226,10 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, - "supports_response_schema": true + "supports_response_schema": true, + "supported_endpoints": [ + "/v1/responses" + ] }, "xai/grok-build-0.1": { "cache_read_input_token_cost": 2e-07, diff --git a/tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py b/tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py index 21f047b753c..29ad8ee4b6e 100644 --- a/tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py +++ b/tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py @@ -61,6 +61,8 @@ PUBLISHED_DBU_PER_MILLION: Final = { "databricks/databricks-gemini-3-1-flash-lite": ("4.464", "26.786", "4.464", "0.446"), "databricks/databricks-gemini-2-5-pro": ("22.321", "178.571", "22.321", "2.232"), "databricks/databricks-gemini-2-5-flash": ("5.357", "44.643", "5.357", "0.536"), + "databricks/databricks-kimi-k3": ("42.857", "214.286", "42.857", "4.286"), + "databricks/databricks-glm-5-2": ("20.000", "62.857", "20.000", "3.714"), } PROMOTIONAL_DISCOUNT: Final = 0.80 PROMOTION_EXPIRES: Final = "2027-01-31" diff --git a/tests/test_litellm/llms/xai/test_xai_model_registry.py b/tests/test_litellm/llms/xai/test_xai_model_registry.py new file mode 100644 index 00000000000..25b2002968d --- /dev/null +++ b/tests/test_litellm/llms/xai/test_xai_model_registry.py @@ -0,0 +1,75 @@ +""" +Registry regression tests for xAI entries in the model cost map. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).parents[4] +PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json" +BACKUP_PRICES_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" + +# Retired by xAI and no longer served: requests to these slugs 404 rather than +# redirecting, and they are absent from https://docs.x.ai/docs/models +RETIRED_MODELS = ( + "xai/grok-2", + "xai/grok-2-1212", + "xai/grok-2-latest", + "xai/grok-2-vision", + "xai/grok-2-vision-1212", + "xai/grok-2-vision-latest", + "xai/grok-beta", + "xai/grok-vision-beta", +) + +# https://docs.x.ai/developers/model-capabilities/text/multi-agent +# "The multi-agent model does not work with the OpenAI Chat Completions API." +RESPONSES_ONLY_MODELS = ( + "xai/grok-4.20-multi-agent-0309", + "xai/grok-4.20-multi-agent-beta-0309", +) + +MAP_PATHS = (PRICES_PATH, BACKUP_PRICES_PATH) + + +@pytest.fixture(scope="module", params=[p.name for p in MAP_PATHS]) +def cost_map(request: pytest.FixtureRequest) -> dict: + path = next(p for p in MAP_PATHS if p.name == request.param) + return json.loads(path.read_text(encoding="utf-8")) + + +@pytest.mark.parametrize("model", RETIRED_MODELS) +def test_retired_xai_models_are_not_advertised(cost_map: dict, model: str): + assert model not in cost_map + + +@pytest.mark.parametrize("model", RESPONSES_ONLY_MODELS) +def test_multi_agent_models_are_responses_only(cost_map: dict, model: str): + entry = cost_map[model] + assert entry["supported_endpoints"] == ["/v1/responses"] + assert entry["mode"] == "responses" + assert "/v1/chat/completions" not in entry["supported_endpoints"] + + +def test_surviving_xai_chat_models_still_serve_chat_completions(cost_map: dict): + """Guard against the removal above over-reaching into live models.""" + chat_models = [ + key + for key, value in cost_map.items() + if isinstance(value, dict) and value.get("litellm_provider") == "xai" and value.get("mode") == "chat" + ] + assert "xai/grok-4.3" in chat_models + assert "xai/grok-4.6" in chat_models + assert not any(key.startswith("xai/grok-2") for key in chat_models) + + +def test_both_cost_maps_agree_on_xai_entries(): + prices = json.loads(PRICES_PATH.read_text(encoding="utf-8")) + backup = json.loads(BACKUP_PRICES_PATH.read_text(encoding="utf-8")) + xai_keys = {k for k, v in prices.items() if isinstance(v, dict) and v.get("litellm_provider") == "xai"} + assert xai_keys + assert {k: prices[k] for k in xai_keys} == {k: backup[k] for k in xai_keys} diff --git a/tests/test_litellm/test_together_ai_model_metadata.py b/tests/test_litellm/test_together_ai_model_metadata.py index 60e4b8ddb4d..32d0769becc 100644 --- a/tests/test_litellm/test_together_ai_model_metadata.py +++ b/tests/test_litellm/test_together_ai_model_metadata.py @@ -15,10 +15,8 @@ COST_MAP_ADAPTER: Final = TypeAdapter(CostMap) SERVERLESS_CHAT_MODELS: Final = ( "together_ai/moonshotai/Kimi-K3", "together_ai/zai-org/GLM-5.2", - "together_ai/deepseek-ai/DeepSeek-V4-Pro", "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813", "together_ai/deepseek-ai/DeepSeek-V4-Flash-0731", - "together_ai/moonshotai/Kimi-K2.7-Code", "together_ai/MiniMaxAI/MiniMax-M3", "together_ai/thinkingmachines/Inkling", "together_ai/thinkingmachines/Inkling-Small", @@ -27,10 +25,8 @@ SERVERLESS_CHAT_MODELS: Final = ( "together_ai/Qwen/Qwen3.7-Plus", "together_ai/Qwen/Qwen3.6-Plus", "together_ai/Qwen/Qwen3.5-9B", - "together_ai/nvidia/nemotron-3-ultra-550b-a55b", "together_ai/meta-models/Muse-Glimmer-30B", "together_ai/google/gemma-4-31B-it", - "together_ai/pearl-ai/gemma-4-31b-it", "together_ai/arize-ai/qwen-2-1.5b-instruct", "together_ai/Prism-ML/Ternary-Bonsai-27B", "together_ai/openai/gpt-oss-120b", @@ -39,6 +35,10 @@ SERVERLESS_CHAT_MODELS: Final = ( ) DEPRECATED_MODELS: Final = { + "together_ai/nvidia/nemotron-3-ultra-550b-a55b": "2026-08-27", + "together_ai/pearl-ai/gemma-4-31b-it": "2026-08-27", + "together_ai/deepseek-ai/DeepSeek-V4-Pro": "2026-08-27", + "together_ai/moonshotai/Kimi-K2.7-Code": "2026-08-27", "together_ai/google/gemma-3n-E4B-it": "2026-08-25", "together_ai/meta-llama/Llama-Guard-4-12B": "2026-08-25", "together_ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": "2026-07-10", From a0689f04c46d7db2d1ba11ca0a91026dc2609c55 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 27 Aug 2026 09:48:19 -0700 Subject: [PATCH 17/29] fix(model_prices): cap ministral-3-3b at Mistral API's 131072 and mirror Anthropic family flags on new DeepInfra Claude rows --- litellm/model_prices_and_context_window_backup.json | 9 ++++++--- model_prices_and_context_window.json | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 0d867955544..b5d40e2800c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -31558,9 +31558,9 @@ "mistral/ministral-3-3b-2512": { "input_cost_per_token": 1e-07, "litellm_provider": "mistral", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://mistral.ai/pricing", @@ -53130,10 +53130,12 @@ "output_cost_per_token": 1.5e-05, "litellm_provider": "deepinfra", "mode": "chat", + "prompt_cache_min_tokens": 1024, "supports_function_calling": true, "supports_tool_choice": true, "supports_response_schema": true, "supports_reasoning": true, + "supports_adaptive_thinking": true, "supports_vision": true, "source": "https://deepinfra.com/pricing" }, @@ -53920,6 +53922,7 @@ "output_cost_per_token": 5e-06, "litellm_provider": "deepinfra", "mode": "chat", + "prompt_cache_min_tokens": 4096, "supports_function_calling": true, "supports_tool_choice": true, "supports_response_schema": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0d867955544..b5d40e2800c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -31558,9 +31558,9 @@ "mistral/ministral-3-3b-2512": { "input_cost_per_token": 1e-07, "litellm_provider": "mistral", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://mistral.ai/pricing", @@ -53130,10 +53130,12 @@ "output_cost_per_token": 1.5e-05, "litellm_provider": "deepinfra", "mode": "chat", + "prompt_cache_min_tokens": 1024, "supports_function_calling": true, "supports_tool_choice": true, "supports_response_schema": true, "supports_reasoning": true, + "supports_adaptive_thinking": true, "supports_vision": true, "source": "https://deepinfra.com/pricing" }, @@ -53920,6 +53922,7 @@ "output_cost_per_token": 5e-06, "litellm_provider": "deepinfra", "mode": "chat", + "prompt_cache_min_tokens": 4096, "supports_function_calling": true, "supports_tool_choice": true, "supports_response_schema": true, From 452254963e99f3946c154033397ec0c224fdab2a Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 27 Aug 2026 12:25:56 -0700 Subject: [PATCH 18/29] feat(health): opt-in model-group allowlist for background health checks and health-check routing (#38539) * feat(health): opt-in model-group allowlist for background health checks and health-check routing * fix(health): merge shared health states per writer scope instead of replacing * refactor(health): drop restating comment and parameterize test scope annotations * chore: remove stray generated prisma migration file * fix(health): merge health states against the Redis snapshot, not the pod-local copy * fix(health): fall back to the pod-local snapshot when the Redis read returns nothing --- litellm/proxy/_types.py | 12 +++ litellm/proxy/health_check.py | 38 ++++++++- litellm/proxy/proxy_server.py | 15 +++- litellm/router.py | 33 ++++++-- litellm/router_utils/health_state_cache.py | 27 ++++++- .../proxy_server/test_background_health.py | 70 ++++++++++++++++ .../proxy/test_health_check_functions.py | 48 +++++++++++ ..._health_check_allowed_fails_integration.py | 62 ++++++++++++++ .../router_utils/test_health_state_cache.py | 81 +++++++++++++++++++ .../test_router_health_check_routing.py | 73 ++++++++++++++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 ++ 11 files changed, 452 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index bb26350e1b1..ed49ca2caa9 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2510,6 +2510,18 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): "are skipped for on-demand GET /health as well as the background health loop." ), ) + background_health_check_model_groups: tuple[str, ...] | None = Field( + None, + description=( + "Opt-in allowlist of model group names for background health checks and " + "health-check routing. When set, the background loop probes only deployments " + "whose model_name is listed, and enable_health_check_routing filters unhealthy " + "deployments only within the listed groups; every other group, including newly " + "added deployments, is skipped and keeps its configured routing strategy. " + "When unset, all deployments participate (opt out per deployment via " + "model_info.disable_background_health_check)." + ), + ) model_list_healthy_only: bool | None = Field( None, description=( diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index 9b60595838d..219f6f270ed 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -7,8 +7,11 @@ import sys import threading import time from collections.abc import Mapping, Sequence +from collections.abc import Set as AbstractSet from types import MappingProxyType -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING, Final, TypeVar + +from pydantic import TypeAdapter, ValidationError import litellm @@ -16,6 +19,7 @@ if TYPE_CHECKING: from litellm.router import Router logger: Final = logging.getLogger(__name__) +_DeploymentT: Final = TypeVar("_DeploymentT", bound=Mapping[str, object]) from litellm.constants import ( BACKGROUND_HEALTH_CHECK_MAX_TOKENS, BACKGROUND_HEALTH_CHECK_MAX_TOKENS_REASONING, @@ -167,6 +171,38 @@ def health_check_filter_kwargs_from_general_settings( } +def parse_background_health_check_model_groups( + general_settings: Mapping[str, object] | None, +) -> frozenset[str] | None: + """ + Read ``general_settings.background_health_check_model_groups``. + + ``None`` means the allowlist is unset and every deployment participates + (legacy behavior). A list scopes background health checks and health-check + routing to deployments whose ``model_name`` is listed. A malformed value + raises so the proxy fails at startup instead of silently probing everything. + """ + raw: Final = (general_settings or {}).get("background_health_check_model_groups") + if raw is None: + return None + try: + return frozenset(TypeAdapter(list[str]).validate_python(raw)) + except ValidationError as e: + raise ValueError( + "general_settings.background_health_check_model_groups must be a list of model group names" + ) from e + + +def filter_deployments_to_model_groups( + model_list: Sequence[_DeploymentT], + model_groups: AbstractSet[str] | None, +) -> tuple[_DeploymentT, ...]: + """Deployments whose ``model_name`` is in ``model_groups``; all of them when unset.""" + if model_groups is None: + return tuple(model_list) + return tuple(x for x in model_list if x.get("model_name") in model_groups) + + def filter_deployments_by_id( model_list: Sequence[Mapping[str, object]], ) -> list: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 990682f10a5..99c3ccd915f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -411,7 +411,9 @@ from litellm.proxy.guardrails.init_guardrails import ( initialize_guardrails, ) from litellm.proxy.health_check import ( + filter_deployments_to_model_groups, health_check_filter_kwargs_from_general_settings, + parse_background_health_check_model_groups, perform_health_check, ) from litellm.proxy.health_endpoints._health_endpoints import router as health_router @@ -3660,6 +3662,13 @@ async def _run_background_health_check(): _llm_model_list = [ m for m in _llm_model_list if not m.get("model_info", {}).get("disable_background_health_check", False) ] + scoped_model_groups = llm_router.background_health_check_model_groups if llm_router is not None else None + _llm_model_list = list(filter_deployments_to_model_groups(_llm_model_list, scoped_model_groups)) + if scoped_model_groups is not None and not _llm_model_list: + verbose_proxy_logger.warning( + "background_health_check_model_groups matched no deployments; groups=%s", + sorted(scoped_model_groups), + ) model_count_enabled = len(_llm_model_list) expected_peak_in_flight = model_count_enabled if isinstance(health_check_concurrency, int) and health_check_concurrency > 0 and model_count_enabled > 0: @@ -5239,6 +5248,7 @@ class ProxyConfig: general_settings = config.get("general_settings", {}) if general_settings is None: general_settings = {} + _bg_hc_model_groups: Final = parse_background_health_check_model_groups(general_settings) _enable_hc_routing = False _hc_staleness = None _hc_ignore_transient = False @@ -5434,13 +5444,14 @@ class ProxyConfig: _hc_staleness = general_settings.get("health_check_staleness_threshold", None) _hc_ignore_transient = general_settings.get("health_check_ignore_transient_errors", False) verbose_proxy_logger.info( - "background_health_check_config enabled=%s shared=%s interval_seconds=%s max_concurrency=%s details=%s health_check_routing=%s", + "background_health_check_config enabled=%s shared=%s interval_seconds=%s max_concurrency=%s details=%s health_check_routing=%s model_groups=%s", use_background_health_checks, use_shared_health_check, health_check_interval, health_check_concurrency, health_check_details, _enable_hc_routing, + sorted(_bg_hc_model_groups) if _bg_hc_model_groups is not None else None, ) ### RBAC ### @@ -5472,6 +5483,8 @@ class ProxyConfig: router_params["health_check_staleness_threshold"] = _hc_staleness if _hc_ignore_transient: router_params["health_check_ignore_transient_errors"] = True + if _bg_hc_model_groups is not None: + router_params["background_health_check_model_groups"] = sorted(_bg_hc_model_groups) ## MODEL LIST model_list: Final = config.get("model_list", None) if model_list: diff --git a/litellm/router.py b/litellm/router.py index f0ebb539bb7..94a6498d490 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -602,6 +602,7 @@ class Router: enable_health_check_routing: bool = False, health_check_staleness_threshold: int | None = None, health_check_ignore_transient_errors: bool = False, + background_health_check_model_groups: Sequence[str] | None = None, enable_weighted_failover: bool = False, ) -> None: """ @@ -811,6 +812,11 @@ class Router: self.enable_health_check_routing = enable_health_check_routing self.enable_weighted_failover = enable_weighted_failover self.health_check_ignore_transient_errors = health_check_ignore_transient_errors + self.background_health_check_model_groups: frozenset[str] | None = ( + frozenset(background_health_check_model_groups) + if background_health_check_model_groups is not None + else None + ) _staleness: Final = health_check_staleness_threshold or ( DEFAULT_HEALTH_CHECK_INTERVAL * DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER ) @@ -12719,6 +12725,10 @@ class Router: """ Filter out deployments marked unhealthy by background health checks. No-op when enable_health_check_routing is False. + When background_health_check_model_groups is set, only deployments in the + listed model groups are filtered; every other group keeps its configured + routing strategy untouched, and a router-level allowed_fails_policy no + longer disables the filter for the listed groups. Returns all deployments if health state is unavailable, stale, or would exclude every candidate (safety net). """ @@ -12727,8 +12737,10 @@ class Router: # When allowed_fails_policy is set, cooldown is the sole routing exclusion # mechanism -- skip the binary health check filter so the policy threshold - # is respected before any deployment is excluded. - if self.allowed_fails_policy is not None: + # is respected before any deployment is excluded. With a model-group + # allowlist the filter is already scoped, so listed groups keep it. + scoped_groups: Final = self.background_health_check_model_groups + if self.allowed_fails_policy is not None and scoped_groups is None: return healthy_deployments unhealthy_ids: Final = await self.health_state_cache.async_get_unhealthy_deployment_ids( @@ -12737,7 +12749,12 @@ class Router: if not unhealthy_ids: return healthy_deployments - filtered: Final = [d for d in healthy_deployments if d["model_info"]["id"] not in unhealthy_ids] + filtered: Final = [ + d + for d in healthy_deployments + if d["model_info"]["id"] not in unhealthy_ids + or (scoped_groups is not None and d["model_name"] not in scoped_groups) + ] if not filtered: verbose_router_logger.warning("All deployments marked unhealthy by health checks, bypassing health filter") @@ -12754,14 +12771,20 @@ class Router: if not self.enable_health_check_routing: return healthy_deployments - if self.allowed_fails_policy is not None: + scoped_groups: Final = self.background_health_check_model_groups + if self.allowed_fails_policy is not None and scoped_groups is None: return healthy_deployments unhealthy_ids: Final = self.health_state_cache.get_unhealthy_deployment_ids(parent_otel_span=parent_otel_span) if not unhealthy_ids: return healthy_deployments - filtered: Final = [d for d in healthy_deployments if d["model_info"]["id"] not in unhealthy_ids] + filtered: Final = [ + d + for d in healthy_deployments + if d["model_info"]["id"] not in unhealthy_ids + or (scoped_groups is not None and d["model_name"] not in scoped_groups) + ] if not filtered: verbose_router_logger.warning("All deployments marked unhealthy by health checks, bypassing health filter") diff --git a/litellm/router_utils/health_state_cache.py b/litellm/router_utils/health_state_cache.py index 95094f7abfa..22d816e13e9 100644 --- a/litellm/router_utils/health_state_cache.py +++ b/litellm/router_utils/health_state_cache.py @@ -43,12 +43,33 @@ class DeploymentHealthCache: self.staleness_threshold = staleness_threshold def set_deployment_health_states(self, states: dict[str, DeploymentHealthStateValue]) -> None: - """Bulk-write all deployment health states as a single cache entry.""" + """Merge the given states into the shared cache entry, pruning expired ones. + + Merging instead of replacing lets writers probing different deployment + scopes (e.g. pods with different background health check allowlists) + coexist on the one shared entry without erasing each other's results. + The snapshot is read from Redis when available, since a pod-local read + would only ever see this writer's own previous merge. When the Redis + read comes back empty (a miss, or a swallowed connection error), the + pod-local copy of the last merge is used so peers are not erased. + """ try: + redis_raw: Final = ( + self.cache.redis_cache.get_cache(self.CACHE_KEY) if self.cache.redis_cache is not None else None + ) + raw: Final = redis_raw if isinstance(redis_raw, dict) else self.cache.get_cache(key=self.CACHE_KEY) + existing: Final = raw if isinstance(raw, dict) else {} + expiry_seconds: Final = self.staleness_threshold * 1.5 + now: Final = time.time() + merged: Final = { + model_id: state + for model_id, state in {**existing, **states}.items() + if isinstance(state, dict) and (now - state.get("timestamp", 0)) < expiry_seconds + } self.cache.set_cache( key=self.CACHE_KEY, - value=states, - ttl=int(self.staleness_threshold * 1.5), + value=merged, + ttl=int(expiry_seconds), ) except Exception as e: verbose_logger.error( diff --git a/tests/test_litellm/proxy/proxy_server/test_background_health.py b/tests/test_litellm/proxy/proxy_server/test_background_health.py index d5a97c0a087..990844369f7 100644 --- a/tests/test_litellm/proxy/proxy_server/test_background_health.py +++ b/tests/test_litellm/proxy/proxy_server/test_background_health.py @@ -581,3 +581,73 @@ async def test_run_background_health_check_runs_one_cycle_then_cancels(monkeypat "unhealthy_count": 1, "sleep_invoked": True, } + + +@pytest.mark.asyncio +async def test_run_background_health_check_probes_only_listed_model_groups(monkeypatch): + monkeypatch.setattr(proxy_server, "health_check_interval", 60) + monkeypatch.setattr(proxy_server, "health_check_concurrency", 1) + monkeypatch.setattr(proxy_server, "health_check_details", True) + monkeypatch.setattr(proxy_server, "use_shared_health_check", False) + monkeypatch.setattr(proxy_server, "redis_usage_cache", None) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr(proxy_server, "background_health_check_loop_active", False) + monkeypatch.setattr( + proxy_server, + "llm_router", + SimpleNamespace(background_health_check_model_groups=frozenset({"prod-openai"})), + ) + monkeypatch.setattr( + proxy_server, + "llm_model_list", + [ + {"model_name": "prod-openai", "model_info": {"id": "listed-1"}}, + {"model_name": "prod-openai", "model_info": {"id": "listed-2"}}, + {"model_name": "internal-claude", "model_info": {"id": "unlisted-1"}}, + { + "model_name": "prod-openai", + "model_info": { + "id": "listed-disabled", + "disable_background_health_check": True, + }, + }, + ], + ) + monkeypatch.setattr( + proxy_server, + "health_check_results", + {"healthy_endpoints": [], "unhealthy_endpoints": []}, + ) + + probed = {} + + async def _fake_direct(model_list, *_a, **_kw): + probed["ids"] = [m["model_info"]["id"] for m in model_list] + return ([], [], {}) + + monkeypatch.setattr( + proxy_server, + "_run_direct_health_check_with_instrumentation", + _fake_direct, + ) + monkeypatch.setattr( + proxy_server, "_schedule_background_health_check_db_save", lambda *a, **kw: None + ) + monkeypatch.setattr( + proxy_server, "_write_health_state_to_router_cache", lambda *a, **kw: None + ) + monkeypatch.setattr( + proxy_server, + "health_check_filter_kwargs_from_general_settings", + lambda _gs: {}, + ) + + async def _stop_sleep(_seconds): + raise asyncio.CancelledError() + + monkeypatch.setattr(proxy_server.asyncio, "sleep", _stop_sleep) + + with pytest.raises(asyncio.CancelledError): + await _run_background_health_check() + + assert probed["ids"] == ["listed-1", "listed-2"] diff --git a/tests/test_litellm/proxy/test_health_check_functions.py b/tests/test_litellm/proxy/test_health_check_functions.py index f2d95131e5e..fdae11d517a 100644 --- a/tests/test_litellm/proxy/test_health_check_functions.py +++ b/tests/test_litellm/proxy/test_health_check_functions.py @@ -623,5 +623,53 @@ async def test_perform_health_check_and_save_forwards_skip_disabled_background_f assert call_kwargs["health_check_skip_disabled_background_models"] is True +def test_parse_background_health_check_model_groups_unset_returns_none(): + from litellm.proxy.health_check import parse_background_health_check_model_groups + + assert parse_background_health_check_model_groups(None) is None + assert parse_background_health_check_model_groups({}) is None + assert ( + parse_background_health_check_model_groups( + {"background_health_check_model_groups": None} + ) + is None + ) + + +def test_parse_background_health_check_model_groups_list_returns_frozenset(): + from litellm.proxy.health_check import parse_background_health_check_model_groups + + parsed = parse_background_health_check_model_groups( + {"background_health_check_model_groups": ["prod-openai", "prod-claude"]} + ) + assert parsed == frozenset({"prod-openai", "prod-claude"}) + + +@pytest.mark.parametrize("bad_value", ["prod-openai", 42, {"a": 1}, [1, 2], [None]]) +def test_parse_background_health_check_model_groups_malformed_raises(bad_value): + from litellm.proxy.health_check import parse_background_health_check_model_groups + + with pytest.raises(ValueError, match="must be a list of model group names"): + parse_background_health_check_model_groups( + {"background_health_check_model_groups": bad_value} + ) + + +def test_filter_deployments_to_model_groups(): + from litellm.proxy.health_check import filter_deployments_to_model_groups + + model_list = [ + {"model_name": "prod-openai", "model_info": {"id": "a"}}, + {"model_name": "internal-claude", "model_info": {"id": "b"}}, + {"model_name": "prod-openai", "model_info": {"id": "c"}}, + ] + + assert filter_deployments_to_model_groups(model_list, None) == tuple(model_list) + assert filter_deployments_to_model_groups( + model_list, frozenset({"prod-openai"}) + ) == (model_list[0], model_list[2]) + assert filter_deployments_to_model_groups(model_list, frozenset()) == () + + if __name__ == "__main__": pytest.main([__file__]) diff --git a/tests/test_litellm/router_utils/test_health_check_allowed_fails_integration.py b/tests/test_litellm/router_utils/test_health_check_allowed_fails_integration.py index 64239f33966..6effbc5fa7f 100644 --- a/tests/test_litellm/router_utils/test_health_check_allowed_fails_integration.py +++ b/tests/test_litellm/router_utils/test_health_check_allowed_fails_integration.py @@ -502,6 +502,68 @@ class TestHealthCheckFilterBypassWithPolicy: ) assert len(result) == 2 + def _make_scoped_router_with_unhealthy(self, policy) -> Router: + import time + + from litellm.caching.caching import DualCache + from litellm.router_utils.health_state_cache import DeploymentHealthCache + + router = Router( + model_list=[ + _make_model("bad-listed"), + _make_model("ok-listed"), + _make_model("bad-unlisted", "gpt-5"), + ], + allowed_fails_policy=policy, + enable_health_check_routing=True, + background_health_check_model_groups=["gpt-4"], + ) + cache = DualCache() + health_cache = DeploymentHealthCache(cache=cache, staleness_threshold=60.0) + health_cache.set_deployment_health_states( + { + model_id: { + "is_healthy": False, + "timestamp": time.time(), + "reason": "test", + } + for model_id in ("bad-listed", "bad-unlisted") + } + ) + router.health_state_cache = health_cache + return router + + def test_filter_with_policy_still_applies_to_listed_groups(self): + """A model-group allowlist keeps the filter active for listed groups even with a policy set.""" + router = self._make_scoped_router_with_unhealthy( + AllowedFailsPolicy(AuthenticationErrorAllowedFails=3) + ) + deployments = [ + _make_model("bad-listed"), + _make_model("ok-listed"), + _make_model("bad-unlisted", "gpt-5"), + ] + + result = router._filter_health_check_unhealthy_deployments(deployments) + assert [d["model_info"]["id"] for d in result] == ["ok-listed", "bad-unlisted"] + + @pytest.mark.asyncio + async def test_async_filter_with_policy_still_applies_to_listed_groups(self): + """Async version: listed groups stay filtered with a policy set, unlisted stay untouched.""" + router = self._make_scoped_router_with_unhealthy( + AllowedFailsPolicy(TimeoutErrorAllowedFails=2) + ) + deployments = [ + _make_model("bad-listed"), + _make_model("ok-listed"), + _make_model("bad-unlisted", "gpt-5"), + ] + + result = await router._async_filter_health_check_unhealthy_deployments( + deployments + ) + assert [d["model_info"]["id"] for d in result] == ["ok-listed", "bad-unlisted"] + class TestAllDeploymentsInCooldownSafetyNet: """ diff --git a/tests/test_litellm/router_utils/test_health_state_cache.py b/tests/test_litellm/router_utils/test_health_state_cache.py index 1af61e899be..ffd031f9b7d 100644 --- a/tests/test_litellm/router_utils/test_health_state_cache.py +++ b/tests/test_litellm/router_utils/test_health_state_cache.py @@ -111,3 +111,84 @@ def test_malformed_state_entries_are_skipped(health_cache): health_cache.set_deployment_health_states(states) result = health_cache.get_unhealthy_deployment_ids() assert result == {"deploy-1"} + + +def test_set_merges_states_from_scoped_writers(health_cache): + """A writer covering one scope must not erase another scope's fresh states.""" + now = time.time() + health_cache.set_deployment_health_states( + {"listed-bad": {"is_healthy": False, "timestamp": now, "reason": "check_failed"}} + ) + health_cache.set_deployment_health_states( + {"other-ok": {"is_healthy": True, "timestamp": now, "reason": ""}} + ) + assert health_cache.get_unhealthy_deployment_ids() == {"listed-bad"} + + +def test_set_prunes_expired_entries(health_cache, cache): + """Entries older than 1.5x the staleness threshold are dropped on write.""" + expired_time = time.time() - 100 # threshold 60s, prune horizon 90s + health_cache.set_deployment_health_states( + {"gone": {"is_healthy": False, "timestamp": expired_time, "reason": "check_failed"}} + ) + now = time.time() + health_cache.set_deployment_health_states( + {"fresh": {"is_healthy": False, "timestamp": now, "reason": "check_failed"}} + ) + stored = cache.get_cache(key=DeploymentHealthCache.CACHE_KEY) + assert set(stored.keys()) == {"fresh"} + + +class _SharedRedisFake: + """Shared get/set key-value store standing in for the Redis layer of a DualCache.""" + + def __init__(self): + self.store = {} + self.fail_get = False + + def get_cache(self, key, parent_otel_span=None, **kwargs): + if self.fail_get: + return None # RedisCache.get_cache swallows connection errors and returns None + return self.store.get(key) + + def set_cache(self, key, value, **kwargs): + self.store[key] = value + + +def test_scoped_writers_on_shared_redis_preserve_each_other(): + """Pods with different allowlists share one Redis entry; each merge must keep the peer's scope.""" + redis_fake = _SharedRedisFake() + pod_a = DeploymentHealthCache(cache=DualCache(redis_cache=redis_fake), staleness_threshold=60.0) + pod_b = DeploymentHealthCache(cache=DualCache(redis_cache=redis_fake), staleness_threshold=60.0) + pod_a.set_deployment_health_states( + {"prod-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "check_failed"}} + ) + pod_b.set_deployment_health_states( + {"internal-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "timeout"}} + ) + pod_a.set_deployment_health_states( + {"prod-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "check_failed"}} + ) + assert set(redis_fake.store[DeploymentHealthCache.CACHE_KEY]) == {"prod-bad", "internal-bad"} + assert pod_a.get_unhealthy_deployment_ids() == {"prod-bad", "internal-bad"} + + +def test_failed_redis_read_falls_back_to_local_copy(): + """A swallowed Redis GET error must not make a writer erase peer scopes it already saw.""" + redis_fake = _SharedRedisFake() + pod_a = DeploymentHealthCache(cache=DualCache(redis_cache=redis_fake), staleness_threshold=60.0) + pod_b = DeploymentHealthCache(cache=DualCache(redis_cache=redis_fake), staleness_threshold=60.0) + pod_a.set_deployment_health_states( + {"prod-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "check_failed"}} + ) + pod_b.set_deployment_health_states( + {"internal-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "timeout"}} + ) + pod_a.set_deployment_health_states( + {"prod-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "check_failed"}} + ) + redis_fake.fail_get = True + pod_a.set_deployment_health_states( + {"prod-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "check_failed"}} + ) + assert set(redis_fake.store[DeploymentHealthCache.CACHE_KEY]) == {"prod-bad", "internal-bad"} diff --git a/tests/test_litellm/router_utils/test_router_health_check_routing.py b/tests/test_litellm/router_utils/test_router_health_check_routing.py index b87a39ac1de..46ed679f746 100644 --- a/tests/test_litellm/router_utils/test_router_health_check_routing.py +++ b/tests/test_litellm/router_utils/test_router_health_check_routing.py @@ -43,7 +43,12 @@ def _make_health_cache( class TestFilterHealthCheckUnhealthyDeployments: """Test the sync filter method.""" - def _make_router_like(self, enable: bool, health_cache: DeploymentHealthCache): + def _make_router_like( + self, + enable: bool, + health_cache: DeploymentHealthCache, + model_groups: frozenset[str] | None = None, + ): """Create a minimal object that behaves like Router for filter testing.""" class FakeRouter: @@ -51,6 +56,7 @@ class TestFilterHealthCheckUnhealthyDeployments: self.enable_health_check_routing = enable self.health_state_cache = health_cache self.allowed_fails_policy = None + self.background_health_check_model_groups = model_groups # Import the actual method and bind it from litellm.router import Router @@ -115,11 +121,50 @@ class TestFilterHealthCheckUnhealthyDeployments: result = router._filter_health_check_unhealthy_deployments(deployments) assert len(result) == 2 + def test_filter_scoped_to_listed_model_groups(self): + """With an allowlist, only deployments in listed groups are filtered on health.""" + health_cache = _make_health_cache(unhealthy_ids={"bad-listed", "bad-unlisted"}) + router = self._make_router_like( + enable=True, health_cache=health_cache, model_groups=frozenset({"prod"}) + ) + + deployments = [ + _make_deployment("bad-listed", model_name="prod"), + _make_deployment("ok-listed", model_name="prod"), + _make_deployment("bad-unlisted", model_name="other"), + _make_deployment("ok-unlisted", model_name="other"), + ] + result = router._filter_health_check_unhealthy_deployments(deployments) + assert [d["model_info"]["id"] for d in result] == [ + "ok-listed", + "bad-unlisted", + "ok-unlisted", + ] + + def test_filter_unscoped_when_model_groups_unset(self): + """Without an allowlist, unhealthy deployments in every group are filtered.""" + health_cache = _make_health_cache(unhealthy_ids={"bad-listed", "bad-unlisted"}) + router = self._make_router_like(enable=True, health_cache=health_cache) + + deployments = [ + _make_deployment("bad-listed", model_name="prod"), + _make_deployment("ok-listed", model_name="prod"), + _make_deployment("bad-unlisted", model_name="other"), + _make_deployment("ok-unlisted", model_name="other"), + ] + result = router._filter_health_check_unhealthy_deployments(deployments) + assert [d["model_info"]["id"] for d in result] == ["ok-listed", "ok-unlisted"] + class TestAsyncFilterHealthCheckUnhealthyDeployments: """Test the async filter method.""" - def _make_router_like(self, enable: bool, health_cache: DeploymentHealthCache): + def _make_router_like( + self, + enable: bool, + health_cache: DeploymentHealthCache, + model_groups: frozenset[str] | None = None, + ): from litellm.router import Router class FakeRouter: @@ -127,6 +172,7 @@ class TestAsyncFilterHealthCheckUnhealthyDeployments: self.enable_health_check_routing = enable self.health_state_cache = health_cache self.allowed_fails_policy = None + self.background_health_check_model_groups = model_groups fake = FakeRouter() fake._async_filter_health_check_unhealthy_deployments = ( @@ -168,6 +214,29 @@ class TestAsyncFilterHealthCheckUnhealthyDeployments: ) assert len(result) == 2 # safety net + @pytest.mark.asyncio + async def test_async_filter_scoped_to_listed_model_groups(self): + """Async version: only deployments in listed groups are filtered on health.""" + health_cache = _make_health_cache(unhealthy_ids={"bad-listed", "bad-unlisted"}) + router = self._make_router_like( + enable=True, health_cache=health_cache, model_groups=frozenset({"prod"}) + ) + + deployments = [ + _make_deployment("bad-listed", model_name="prod"), + _make_deployment("ok-listed", model_name="prod"), + _make_deployment("bad-unlisted", model_name="other"), + _make_deployment("ok-unlisted", model_name="other"), + ] + result = await router._async_filter_health_check_unhealthy_deployments( + healthy_deployments=deployments + ) + assert [d["model_info"]["id"] for d in result] == [ + "ok-listed", + "bad-unlisted", + "ok-unlisted", + ] + class TestBuildDeploymentHealthStates: """Test the build_deployment_health_states function.""" diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 9ac49fa96e1..c124cc2e9c8 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -24878,6 +24878,11 @@ export interface components { * @description If True, a user's personal max_budget is enforced on every request they make, including requests made with a team-scoped key. Defaults to False, where a team-scoped key is governed only by the team and team-member budgets and the key owner's personal max_budget does not apply (see GitHub issue #12905). */ apply_user_budget_to_team_keys?: boolean | null; + /** + * Background Health Check Model Groups + * @description Opt-in allowlist of model group names for background health checks and health-check routing. When set, the background loop probes only deployments whose model_name is listed, and enable_health_check_routing filters unhealthy deployments only within the listed groups; every other group, including newly added deployments, is skipped and keeps its configured routing strategy. When unset, all deployments participate (opt out per deployment via model_info.disable_background_health_check). + */ + background_health_check_model_groups?: string[] | null; /** * Background Health Checks * @description run health checks in background From c251703e8bed8f310eb0452858497affa670e555 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 27 Aug 2026 12:43:28 -0700 Subject: [PATCH 19/29] fix(realtime): bill trailing audio when a Gemini transcribe Live session closes --- .../litellm_core_utils/realtime_streaming.py | 19 ++++ .../llms/base_llm/realtime/transformation.py | 4 + .../llms/gemini/realtime/transformation.py | 3 + .../test_realtime_streaming.py | 92 +++++++++++++++++++ .../test_gemini_realtime_transformation.py | 24 +++++ 5 files changed, 142 insertions(+) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 2da63554b75..9125ed6e70a 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -330,6 +330,24 @@ class RealTimeStreaming: except (AttributeError, TypeError): pass + def _flush_unbilled_transcription_usage(self) -> None: + if self.provider_config is None: + return + usage: Final = self.provider_config.unbilled_usage_on_session_close(self.model) + if usage is None: + return + flush_event: Final = ( + cast( # cast-ok: usage-only partial event, the same shape _capture_transcription_usage logs + OpenAIRealtimeEvents, + { + "type": "conversation.item.input_audio_transcription.completed", + "usage": usage, + }, + ) + ) + self.store_message(flush_event) + self._capture_transcription_usage(flush_event) + def _collect_tool_calls_from_response_done(self, event_obj: dict | OpenAIRealtimeEvents) -> None: """Extract function_call items from response.done events for spend logging.""" try: @@ -1069,6 +1087,7 @@ class RealTimeStreaming: except Exception as e: verbose_logger.exception("Error in backend to client send messages: %s", e) finally: + self._flush_unbilled_transcription_usage() await self.log_messages() @staticmethod diff --git a/litellm/llms/base_llm/realtime/transformation.py b/litellm/llms/base_llm/realtime/transformation.py index 26c189504df..cfcde7c6e9e 100644 --- a/litellm/llms/base_llm/realtime/transformation.py +++ b/litellm/llms/base_llm/realtime/transformation.py @@ -5,6 +5,7 @@ import httpx from litellm.types.llms.openai import OpenAIRealtimeStreamSessionEvents from litellm.types.realtime import ( + RealtimeInputAudioTranscriptionUsage, RealtimeResponseTransformInput, RealtimeResponseTypedDict, ) @@ -70,6 +71,9 @@ class BaseRealtimeConfig(ABC): def session_configuration_request(self, model: str) -> str | None: # message sent to setup the realtime session return None + def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None: + return None + def transform_session_created_event( self, model: str, diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index a3b6381306e..367619db37d 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -1191,6 +1191,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): } return usage + def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None: + return self._consume_input_transcription_usage_estimate(model) + def transform_realtime_response( self, message: str | bytes, diff --git a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py index 1b71c2f1f9b..52e88db753a 100644 --- a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py +++ b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py @@ -3019,3 +3019,95 @@ async def test_provider_config_path_captures_transcription_usage(): and message.get("usage") == usage ) assert len(usage_events) == 1 + + +@pytest.mark.asyncio +async def test_session_close_flushes_unbilled_transcription_usage(): + """Trailing audio appended after the last transcript frame must still be billed: + on session close the provider's unbilled estimate is flushed into the logged + messages before log_messages runs, and never forwarded to the client.""" + from typing import Final + + from litellm.types.realtime import RealtimeInputAudioTranscriptionUsage + + client_ws: Final = MagicMock() + client_ws.send_text = AsyncMock() + backend_ws: Final = MagicMock() + backend_ws.recv = AsyncMock(side_effect=ConnectionClosed(None, None)) + logging_obj: Final = MagicMock() + logging_obj.async_success_handler = AsyncMock() + logging_obj.success_handler = MagicMock() + + usage: Final[RealtimeInputAudioTranscriptionUsage] = { + "type": "tokens", + "input_tokens": 153, + "output_tokens": 18, + "total_tokens": 171, + "input_token_details": {"text_tokens": 0, "audio_tokens": 153}, + } + provider_config: Final = MagicMock() + provider_config.unbilled_usage_on_session_close = MagicMock(return_value=usage) + + streaming: Final = RealTimeStreaming( + client_ws, + backend_ws, + logging_obj, + provider_config=provider_config, + model="gemini-3.5-transcribe-live", + ) + logged_snapshots: Final[list[tuple]] = [] + + original_log_messages: Final = streaming.log_messages + + async def _snapshot_then_log(): + logged_snapshots.append(tuple(streaming.messages)) + await original_log_messages() + + streaming.log_messages = _snapshot_then_log + + await streaming.backend_to_client_send_messages() + + provider_config.unbilled_usage_on_session_close.assert_called_once_with("gemini-3.5-transcribe-live") + flushed: Final = tuple( + message + for message in streaming.messages + if isinstance(message, dict) + and message.get("type") == "conversation.item.input_audio_transcription.completed" + and message.get("usage") == usage + ) + assert len(flushed) == 1 + assert flushed[0] in logged_snapshots[0] + assert not client_ws.send_text.called + + +@pytest.mark.asyncio +async def test_session_close_flush_noop_without_unbilled_usage(): + """Everything already billed mid-stream: the session-close flush must not append + a duplicate transcription event.""" + from typing import Final + + client_ws: Final = MagicMock() + client_ws.send_text = AsyncMock() + backend_ws: Final = MagicMock() + backend_ws.recv = AsyncMock(side_effect=ConnectionClosed(None, None)) + logging_obj: Final = MagicMock() + logging_obj.async_success_handler = AsyncMock() + logging_obj.success_handler = MagicMock() + + provider_config: Final = MagicMock() + provider_config.unbilled_usage_on_session_close = MagicMock(return_value=None) + + streaming: Final = RealTimeStreaming( + client_ws, + backend_ws, + logging_obj, + provider_config=provider_config, + model="gemini-3.5-transcribe-live", + ) + + await streaming.backend_to_client_send_messages() + + assert not any( + isinstance(message, dict) and message.get("type") == "conversation.item.input_audio_transcription.completed" + for message in streaming.messages + ) diff --git a/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py b/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py index c362efbfffa..42994dbd2af 100644 --- a/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py +++ b/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py @@ -2119,3 +2119,27 @@ def test_non_transcription_live_model_completed_event_has_no_usage(patch_gemini_ ) assert len(completed) == 1 assert "usage" not in completed[0] + + +def test_unbilled_usage_on_session_close_flushes_trailing_audio(patch_gemini_transcribe_live_cost_map_entry): + """Audio appended after the last transcript frame is still unbilled when the + session closes; the session-close hook must hand back the estimate exactly once + so the streaming layer can bill it (144000 pcm16 bytes = 3s -> 75 in / 9 out).""" + from typing import Final + + from litellm.types.realtime import RealtimeInputAudioTranscriptionUsage + + config: Final = GeminiRealtimeConfig() + config.transform_realtime_request(_input_audio_append_message(144000), "gemini-3.5-transcribe-live") + + usage: Final = config.unbilled_usage_on_session_close("gemini-3.5-transcribe-live") + + expected: Final[RealtimeInputAudioTranscriptionUsage] = { + "type": "tokens", + "input_tokens": 75, + "output_tokens": 9, + "total_tokens": 84, + "input_token_details": {"text_tokens": 0, "audio_tokens": 75}, + } + assert usage == expected + assert config.unbilled_usage_on_session_close("gemini-3.5-transcribe-live") is None From e16aa9f5126318d3dff269005df086fedaa98f00 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2026 12:44:36 -0700 Subject: [PATCH 20/29] fix(mcp): keep upstream OAuth Authorization when jwt signer hook injects one on tools/call (#38555) * fix(mcp): keep upstream OAuth Authorization when jwt signer hook injects one on tools/call Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(mcp): only treat server credential as occupying Authorization when it maps to that header Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../mcp_server/mcp_server_manager.py | 45 ++-- .../mcp_server/test_mcp_hook_extra_headers.py | 252 +++++++++++++++++- 2 files changed, 268 insertions(+), 29 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 308813039ca..6a1b6851d3e 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -5256,7 +5256,9 @@ class MCPServerManager: proxy_logging_obj: Optional ProxyLogging object for hook integration host_progress_callback: Optional callback for progress updates hook_extra_headers: Optional headers injected by pre_mcp_call guardrail - hooks. Merged last (highest priority) into outbound request headers. + hooks. Merged last into outbound request headers, except a hook + Authorization header is dropped when an upstream credential already + occupies the Authorization slot. Returns: CallToolResult from the MCP server @@ -5347,27 +5349,26 @@ class MCPServerManager: if hook_extra_headers: if extra_headers is None: extra_headers = {} - if "Authorization" in hook_extra_headers: - if "Authorization" in extra_headers: - verbose_logger.warning( - "MCPServerManager: hook_extra_headers 'Authorization' will overwrite " - "the existing Authorization header from static_headers. " - "The hook JWT will take precedence." - ) - elif server_auth_header is not None: - # server_auth_header is passed separately to _create_mcp_client as - # auth_value. Both will reach the upstream server — warn so admins - # know two Authorization credentials are being sent. - verbose_logger.warning( - "MCPServerManager: hook_extra_headers injects 'Authorization' while " - "server '%s' already has a configured authentication_token. " - "Both credentials will be sent; the hook header is in extra_headers " - "and the server token is in auth_value — the upstream server decides " - "which one wins. Consider unsetting authentication_token if you want " - "the hook JWT to be the sole credential.", - mcp_server.server_name or mcp_server.name, - ) - extra_headers.update(hook_extra_headers) + hook_has_authorization: Final = any(k.lower() == "authorization" for k in hook_extra_headers) + existing_has_authorization: Final = any(k.lower() == "authorization" for k in extra_headers) + server_auth_occupies_authorization: Final = ( + any(k.lower() == "authorization" for k in server_auth_header) + if isinstance(server_auth_header, dict) + else server_auth_header is not None and mcp_server.auth_type != MCPAuth.api_key + ) + if hook_has_authorization and (existing_has_authorization or server_auth_occupies_authorization): + # Mirror the tools/list signer guard: an upstream credential (user OAuth, + # static header, or configured authentication_token) already occupies the + # Authorization slot, so the hook must not replace it. + verbose_logger.warning( + "MCPServerManager: dropping hook-injected 'Authorization' header for " + "server '%s' because an upstream credential already occupies the " + "Authorization slot; the existing credential is kept.", + mcp_server.server_name or mcp_server.name, + ) + extra_headers.update({k: v for k, v in hook_extra_headers.items() if k.lower() != "authorization"}) + else: + extra_headers.update(hook_extra_headers) # Reset to None if no headers were actually added if extra_headers is not None and len(extra_headers) == 0: diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py index 4081681daef..56851d31241 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py @@ -5,7 +5,8 @@ Validates that: 1. _convert_mcp_hook_response_to_kwargs extracts extra_headers from hook response 2. pre_call_tool_check returns hook-provided extra_headers AND modified arguments 3. call_tool flows hook headers and modified arguments downstream -4. Hook-provided headers take highest priority (merge after static_headers) +4. Hook-provided headers merge after static_headers, but a hook Authorization + header never displaces an existing upstream Authorization credential 5. OpenAPI-backed servers log a warning and continue (skip injection) when hook headers are present 6. JWT claims are propagated in both standard and virtual-key fast paths 7. Backward compatibility: hooks without extra_headers continue to work @@ -487,8 +488,8 @@ class TestHookHeaderMergePriority: ) @pytest.mark.asyncio - async def test_hook_headers_override_static_headers(self): - """Hook headers should take precedence over static_headers.""" + async def test_hook_authorization_does_not_override_static_authorization(self): + """A hook Authorization must not displace a static_headers Authorization (LIT-6321).""" manager = MCPServerManager() server = self._make_server(static_headers={"Authorization": "Bearer static-token", "X-Static": "yes"}) @@ -521,7 +522,7 @@ class TestHookHeaderMergePriority: pass headers = captured_extra_headers.get("value", {}) - assert headers["Authorization"] == "Bearer hook-signed-jwt" + assert headers["Authorization"] == "Bearer static-token" assert headers["X-Static"] == "yes" @pytest.mark.asyncio @@ -560,8 +561,8 @@ class TestHookHeaderMergePriority: assert headers == {"X-Static": "static-value"} @pytest.mark.asyncio - async def test_hook_headers_merge_with_oauth2(self): - """Hook headers merge on top of OAuth2 headers.""" + async def test_hook_authorization_does_not_override_oauth2_authorization(self): + """tools/call keeps the user's OAuth Authorization; only non-auth hook headers merge (LIT-6321).""" manager = MCPServerManager() server = MCPServer( server_id="test-id", @@ -570,6 +571,8 @@ class TestHookHeaderMergePriority: url="https://example.com", transport=MCPTransport.http, auth_type=MCPAuth.oauth2, + oauth2_flow="authorization_code", + delegate_auth_to_upstream=True, ) captured_extra_headers: Dict[str, Any] = {} @@ -605,10 +608,245 @@ class TestHookHeaderMergePriority: pass headers = captured_extra_headers.get("value", {}) - assert headers["Authorization"] == "Bearer hook-jwt" + assert headers["Authorization"] == "Bearer oauth2-token" assert headers["X-OAuth"] == "yes" assert headers["X-Trace-Id"] == "trace-123" + @pytest.mark.asyncio + async def test_hook_authorization_used_when_no_upstream_credential(self): + """With no upstream credential, the signer JWT is still injected.""" + manager = MCPServerManager() + server = self._make_server() + + captured_extra_headers: Dict[str, Optional[Dict[str, str]]] = {} + + async def fake_create_mcp_client(server, mcp_auth_header=None, extra_headers=None, stdio_env=None, **kwargs): + captured_extra_headers["value"] = extra_headers + mock_client = MagicMock() + mock_client.call_tool = AsyncMock(return_value=MagicMock()) + return mock_client + + with patch.object(manager, "_create_mcp_client", side_effect=fake_create_mcp_client): + with patch.object(manager, "_build_stdio_env", return_value=None): + try: + await manager._call_regular_mcp_tool( + mcp_server=server, + original_tool_name="test_tool", + arguments={"key": "val"}, + tasks=[], + mcp_auth_header=None, + mcp_server_auth_headers=None, + oauth2_headers=None, + raw_headers=None, + proxy_logging_obj=None, + hook_extra_headers={"Authorization": "Bearer hook-jwt"}, + ) + except Exception: + pass + + headers = captured_extra_headers.get("value") or {} + assert headers["Authorization"] == "Bearer hook-jwt" + + @pytest.mark.asyncio + async def test_hook_authorization_dropped_when_server_auth_header_present(self): + """With a configured authentication_token (auth_value), the hook Authorization is dropped.""" + manager = MCPServerManager() + server = self._make_server() + + captured: Dict[str, object] = {} + + async def fake_create_mcp_client(server, mcp_auth_header=None, extra_headers=None, stdio_env=None, **kwargs): + captured["extra_headers"] = extra_headers + captured["mcp_auth_header"] = mcp_auth_header + mock_client = MagicMock() + mock_client.call_tool = AsyncMock(return_value=MagicMock()) + return mock_client + + with patch.object(manager, "_create_mcp_client", side_effect=fake_create_mcp_client): + with patch.object(manager, "_build_stdio_env", return_value=None): + try: + await manager._call_regular_mcp_tool( + mcp_server=server, + original_tool_name="test_tool", + arguments={"key": "val"}, + tasks=[], + mcp_auth_header="server-static-token", + mcp_server_auth_headers=None, + oauth2_headers=None, + raw_headers=None, + proxy_logging_obj=None, + hook_extra_headers={ + "Authorization": "Bearer hook-jwt", + "X-Trace-Id": "trace-123", + }, + ) + except Exception: + pass + + headers = captured.get("extra_headers") or {} + assert isinstance(headers, dict) + assert "Authorization" not in headers + assert headers.get("X-Trace-Id") == "trace-123" + assert captured.get("mcp_auth_header") == "server-static-token" + + @pytest.mark.asyncio + async def test_hook_authorization_case_insensitive_conflict(self): + """Authorization conflicts are matched case-insensitively.""" + manager = MCPServerManager() + server = self._make_server(static_headers={"authorization": "Bearer static-token"}) + + captured_extra_headers: Dict[str, Optional[Dict[str, str]]] = {} + + async def fake_create_mcp_client(server, mcp_auth_header=None, extra_headers=None, stdio_env=None, **kwargs): + captured_extra_headers["value"] = extra_headers + mock_client = MagicMock() + mock_client.call_tool = AsyncMock(return_value=MagicMock()) + return mock_client + + with patch.object(manager, "_create_mcp_client", side_effect=fake_create_mcp_client): + with patch.object(manager, "_build_stdio_env", return_value=None): + try: + await manager._call_regular_mcp_tool( + mcp_server=server, + original_tool_name="test_tool", + arguments={"key": "val"}, + tasks=[], + mcp_auth_header=None, + mcp_server_auth_headers=None, + oauth2_headers=None, + raw_headers=None, + proxy_logging_obj=None, + hook_extra_headers={"Authorization": "Bearer hook-jwt"}, + ) + except Exception: + pass + + headers = captured_extra_headers.get("value") or {} + assert headers.get("authorization") == "Bearer static-token" + assert "Authorization" not in headers + + @pytest.mark.asyncio + async def test_hook_authorization_kept_with_api_key_server_credential(self): + """An api_key credential maps to X-API-Key, so the hook Authorization is kept.""" + manager = MCPServerManager() + server = MCPServer( + server_id="test-id", + name="Test Server", + server_name="test_server", + url="https://example.com", + transport=MCPTransport.http, + auth_type=MCPAuth.api_key, + ) + + captured: Dict[str, object] = {} + + async def fake_create_mcp_client(server, mcp_auth_header=None, extra_headers=None, stdio_env=None, **kwargs): + captured["extra_headers"] = extra_headers + captured["mcp_auth_header"] = mcp_auth_header + mock_client = MagicMock() + mock_client.call_tool = AsyncMock(return_value=MagicMock()) + return mock_client + + with patch.object(manager, "_create_mcp_client", side_effect=fake_create_mcp_client): + with patch.object(manager, "_build_stdio_env", return_value=None): + try: + await manager._call_regular_mcp_tool( + mcp_server=server, + original_tool_name="test_tool", + arguments={"key": "val"}, + tasks=[], + mcp_auth_header="server-api-key", + mcp_server_auth_headers=None, + oauth2_headers=None, + raw_headers=None, + proxy_logging_obj=None, + hook_extra_headers={"Authorization": "Bearer hook-jwt"}, + ) + except Exception: + pass + + headers = captured.get("extra_headers") or {} + assert isinstance(headers, dict) + assert headers.get("Authorization") == "Bearer hook-jwt" + assert captured.get("mcp_auth_header") == "server-api-key" + + @pytest.mark.asyncio + async def test_hook_authorization_kept_with_non_authorization_server_header_dict(self): + """A per-server header dict without Authorization does not block the hook JWT.""" + manager = MCPServerManager() + server = self._make_server() + + captured: Dict[str, object] = {} + + async def fake_create_mcp_client(server, mcp_auth_header=None, extra_headers=None, stdio_env=None, **kwargs): + captured["extra_headers"] = extra_headers + captured["mcp_auth_header"] = mcp_auth_header + mock_client = MagicMock() + mock_client.call_tool = AsyncMock(return_value=MagicMock()) + return mock_client + + with patch.object(manager, "_create_mcp_client", side_effect=fake_create_mcp_client): + with patch.object(manager, "_build_stdio_env", return_value=None): + try: + await manager._call_regular_mcp_tool( + mcp_server=server, + original_tool_name="test_tool", + arguments={"key": "val"}, + tasks=[], + mcp_auth_header=None, + mcp_server_auth_headers={"test_server": {"X-API-Key": "per-server-key"}}, + oauth2_headers=None, + raw_headers=None, + proxy_logging_obj=None, + hook_extra_headers={"Authorization": "Bearer hook-jwt"}, + ) + except Exception: + pass + + headers = captured.get("extra_headers") or {} + assert isinstance(headers, dict) + assert headers.get("Authorization") == "Bearer hook-jwt" + assert captured.get("mcp_auth_header") == {"X-API-Key": "per-server-key"} + + @pytest.mark.asyncio + async def test_hook_authorization_dropped_with_authorization_server_header_dict(self): + """A per-server header dict carrying Authorization blocks the hook JWT.""" + manager = MCPServerManager() + server = self._make_server() + + captured: Dict[str, object] = {} + + async def fake_create_mcp_client(server, mcp_auth_header=None, extra_headers=None, stdio_env=None, **kwargs): + captured["extra_headers"] = extra_headers + captured["mcp_auth_header"] = mcp_auth_header + mock_client = MagicMock() + mock_client.call_tool = AsyncMock(return_value=MagicMock()) + return mock_client + + with patch.object(manager, "_create_mcp_client", side_effect=fake_create_mcp_client): + with patch.object(manager, "_build_stdio_env", return_value=None): + try: + await manager._call_regular_mcp_tool( + mcp_server=server, + original_tool_name="test_tool", + arguments={"key": "val"}, + tasks=[], + mcp_auth_header=None, + mcp_server_auth_headers={"test_server": {"authorization": "Bearer per-server-token"}}, + oauth2_headers=None, + raw_headers=None, + proxy_logging_obj=None, + hook_extra_headers={"Authorization": "Bearer hook-jwt", "X-Trace-Id": "trace-123"}, + ) + except Exception: + pass + + headers = captured.get("extra_headers") or {} + assert isinstance(headers, dict) + assert "Authorization" not in headers + assert headers.get("X-Trace-Id") == "trace-123" + assert captured.get("mcp_auth_header") == {"authorization": "Bearer per-server-token"} + @pytest.mark.asyncio async def test_m2m_oauth2_does_not_forward_litellm_caller_authorization(self): """M2M must not put caller Bearer (LiteLLM API key) into extra_headers (#23652).""" From f864908cd70a904278d0a3b7f274e44ead6e3d5e Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2026 12:45:30 -0700 Subject: [PATCH 21/29] fix: suppress misleading register_model unresolved-cost warnings for entries without custom pricing (#38542) * fix: suppress misleading register_model unresolved-cost warnings for entries without custom pricing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix: do not warn about zero cache costs for tiered pricing entries Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/main.py | 1 + litellm/router.py | 6 +- litellm/utils.py | 20 ++- .../test_register_model_custom_pricing.py | 145 ++++++++++++++++++ 4 files changed, 168 insertions(+), 4 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 8ee102f5d07..583c5b3f92a 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1219,6 +1219,7 @@ def _register_custom_pricing_for_request( shared_key: CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry), }, persist_across_reloads=False, + warning_display_name=shared_key, ) diff --git a/litellm/router.py b/litellm/router.py index 94a6498d490..021dafa9791 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9216,7 +9216,11 @@ class Router: } if model_id is not None: - litellm.register_model(model_cost={model_id: model_info}, persist_across_reloads=False) + litellm.register_model( + model_cost={model_id: model_info}, + persist_across_reloads=False, + warning_display_name=model, + ) ## OLD MODEL REGISTRATION ## Kept to prevent breaking changes backend_keys: Final = Router._backend_cost_map_keys(model=model, custom_llm_provider=custom_llm_provider) diff --git a/litellm/utils.py b/litellm/utils.py index a26b2c5b440..b164a9c4671 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2948,7 +2948,12 @@ def reapply_runtime_model_cost_registrations() -> None: register_model(model_cost=dict(_runtime_registered_model_cost)) # mutable-ok: snapshot, replay rewrites it -def register_model(model_cost: str | dict, *, persist_across_reloads: bool = True): +def register_model( + model_cost: str | dict, + *, + persist_across_reloads: bool = True, + warning_display_name: str | None = None, +): """ Register new / Override existing models (and their pricing) to specific providers. Provide EITHER a model cost dictionary or a url to a hosted json blob @@ -2968,6 +2973,10 @@ def register_model(model_cost: str | dict, *, persist_across_reloads: bool = Tru registering a model is declaring durable intent. Pass False for a registration that only describes one request, so it is dropped rather than re-asserted over every future catalog. + + ``warning_display_name`` names the model in the missing-cache-pricing + warning instead of the registered key, for callers that register under an + opaque key (e.g. the router's hashed deployment ids). """ loaded_model_cost = {} @@ -3014,10 +3023,15 @@ def register_model(model_cost: str | dict, *, persist_across_reloads: bool = Tru elif ( value.get("cache_creation_input_token_cost") is None and value.get("cache_read_input_token_cost") is None + and value.get("tiered_pricing") is None + and ( + value.get("input_cost_per_token") is not None + or value.get("output_cost_per_token") is not None + ) ): verbose_logger.warning( - "register_model: model=%s not in built-in cost map and no prefix/region variant matched; cache cost fields will default to 0. To track cache cost, add cache_creation_input_token_cost and cache_read_input_token_cost to model_info", - key, + "register_model: model=%s has custom pricing but not in built-in cost map and no prefix/region variant matched; cache_creation_input_token_cost and cache_read_input_token_cost will default to 0 for this model (input/output cost tracking is unaffected). To track cache cost, add them to model_info", + warning_display_name or key, ) # ``get_model_info`` returns ``litellm_provider: None`` when the # provider is unknown (e.g. custom deployments registered via diff --git a/tests/test_litellm/test_register_model_custom_pricing.py b/tests/test_litellm/test_register_model_custom_pricing.py index e3f6a1a0f40..39f498b4e58 100644 --- a/tests/test_litellm/test_register_model_custom_pricing.py +++ b/tests/test_litellm/test_register_model_custom_pricing.py @@ -435,6 +435,151 @@ def test_register_model_warns_when_no_builtin_match_for_cache_pricing(caplog): litellm.model_cost.pop(registered_key, None) +def test_register_model_no_warning_without_custom_pricing(caplog): + """LIT-6318: an entry with no custom pricing (e.g. router deployment + metadata) never drives cost calculation, so registering it under an + unmatched key must not emit the missing-cache-pricing warning. + """ + import logging + + from litellm._logging import verbose_logger + + registered_key = "azure/lit6318-deployment-without-pricing" + litellm.model_cost.pop(registered_key, None) + + try: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + litellm.register_model( + { + registered_key: { + "litellm_provider": "azure", + "base_model": "azure/text-embedding-3-large", + } + } + ) + + assert not any("register_model" in record.message for record in caplog.records), ( + "entry without custom pricing must register silently" + ) + finally: + litellm.model_cost.pop(registered_key, None) + + +def test_register_model_no_warning_for_tiered_pricing_without_cache_costs(caplog): + """LIT-6318: tiered pricing bills cache reads at the tier's input rate when + cache costs are omitted, so a tiered entry must not trigger the + cache-defaults-to-0 warning. + """ + import logging + + from litellm._logging import verbose_logger + + registered_key = "bedrock/lit6318-tiered-priced-model" + litellm.model_cost.pop(registered_key, None) + + try: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + litellm.register_model( + { + registered_key: { + "litellm_provider": "bedrock", + "tiered_pricing": [ + { + "range": [0, 200000], + "input_cost_per_token": 1e-06, + "output_cost_per_token": 5e-06, + } + ], + } + } + ) + + assert not any("register_model" in record.message for record in caplog.records), ( + "tiered pricing entry must register silently" + ) + finally: + litellm.model_cost.pop(registered_key, None) + + +def test_router_deployment_without_custom_pricing_registers_silently(caplog): + """LIT-6318: the router registers every deployment under its hashed id and + its backend key. Deployments without custom pricing are costed at request + time from the underlying model name, so startup must not warn about them. + """ + import logging + + from litellm import Router + from litellm._logging import verbose_logger + + deployment_model = "azure/lit6318-my-deployment-name" + deployment_id = "lit6318-no-pricing-deployment" + snapshot = _snapshot_model_cost_entries([deployment_model, deployment_id]) + + try: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + Router( + model_list=[ + { + "model_name": "indexing", + "litellm_params": { + "model": deployment_model, + "api_base": "https://example.openai.azure.com", + "api_key": "fake-key", + }, + "model_info": { + "id": deployment_id, + "base_model": "azure/text-embedding-3-large", + }, + } + ] + ) + + register_warnings = [record.message for record in caplog.records if "register_model" in record.message] + assert not register_warnings, register_warnings + finally: + _restore_model_cost_entries(snapshot) + + +def test_router_custom_priced_deployment_warning_names_model_not_hash(caplog): + """LIT-6318: when a custom-priced deployment genuinely lacks cache pricing + and no built-in entry matches, the warning must name the deployment's + model rather than its opaque hashed id. + """ + import logging + + from litellm import Router + from litellm._logging import verbose_logger + + deployment_model = "bedrock/lit6318-totally-made-up-model" + deployment_id = "lit6318-custom-priced-deployment-hash" + snapshot = _snapshot_model_cost_entries([deployment_model, deployment_id]) + + try: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + Router( + model_list=[ + { + "model_name": "made-up", + "litellm_params": { + "model": deployment_model, + "aws_region_name": "us-east-1", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 5e-06, + }, + "model_info": {"id": deployment_id}, + } + ] + ) + + register_warnings = [record.message for record in caplog.records if "register_model" in record.message] + assert register_warnings, "expected a warning for missing cache pricing" + for message in register_warnings: + assert deployment_id not in message, message + assert deployment_model in message, message + finally: + _restore_model_cost_entries(snapshot) + + def test_register_model_router_add_deployment_custom_pricing_applies(): """End-to-end regression for https://github.com/BerriAI/litellm/issues/28336. From de532833566a8c2b3718a7451c5855bb4239d7ae Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2026 12:46:09 -0700 Subject: [PATCH 22/29] feat(proxy): opt-in budget rollover carrying overage into the next window (#38514) * feat(proxy): opt-in budget rollover carrying overage into the next window Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): zero under-cap rows before decrementing over-cap rows in cascade resets Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/__init__.py | 1 + litellm/constants.py | 1 + .../proxy/common_utils/reset_budget_job.py | 199 ++++++++++-- litellm/proxy/proxy_server.py | 7 + litellm/repositories/unit_of_work.py | 45 ++- .../common_utils/test_reset_budget_job.py | 302 +++++++++++++++++- 6 files changed, 515 insertions(+), 40 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index eebd2dad91e..ec2960c196e 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -445,6 +445,7 @@ max_ui_session_budget: Optional[float] = ( 1.0 # USD budget for each dashboard login session (playground, test connection) ) internal_user_budget_duration: Optional[str] = None +budget_rollover: bool = False # carry spend beyond max_budget into the next window instead of zeroing it tag_budget_config: Optional[Dict[str, "BudgetConfig"]] = None max_end_user_budget: Optional[float] = None max_end_user_budget_id: Optional[str] = None diff --git a/litellm/constants.py b/litellm/constants.py index b2f59bc667c..ed89474600e 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1652,6 +1652,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [ "enable_anthropic_prompt_caching", "anthropic_prompt_caching_ttl", "max_ui_session_budget", + "budget_rollover", ] SPECIAL_LITELLM_AUTH_TOKEN: Final = ["ui-token"] DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60)) diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index 4ebcc549cdd..b8b9500ff63 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -1,5 +1,6 @@ import asyncio import json +import math import time from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence from dataclasses import dataclass @@ -45,6 +46,7 @@ from litellm.repositories.table_repositories import ( ) from litellm.repositories.team_repository import TeamRepository from litellm.repositories.unit_of_work import ( + LinkedSpendResetWrites, budget_cascade_unit_of_work, spend_reset_unit_of_work, ) @@ -59,7 +61,15 @@ _LINKED_KEYS_WHERE: Final[Mapping[str, object]] = MappingProxyType({"budget_dura _SPENT_ROWS_WHERE: Final[Mapping[str, object]] = MappingProxyType({"spend": {"gt": 0}}) -class _TeamMembershipRow(Protocol): +class _BudgetLinkedRow(Protocol): + @property + def spend(self) -> float | None: ... + + @property + def budget_id(self) -> str | None: ... + + +class _TeamMembershipRow(_BudgetLinkedRow, Protocol): @property def user_id(self) -> str: ... @@ -67,26 +77,48 @@ class _TeamMembershipRow(Protocol): def team_id(self) -> str: ... -class _KeyRow(Protocol): +class _KeyRow(_BudgetLinkedRow, Protocol): @property def token(self) -> str: ... -class _OrgRow(Protocol): +class _OrgRow(_BudgetLinkedRow, Protocol): @property def organization_id(self) -> str: ... -class _TagRow(Protocol): +class _TagRow(_BudgetLinkedRow, Protocol): @property def tag_name(self) -> str: ... -class _EndUserRow(Protocol): +class _EndUserRow(_BudgetLinkedRow, Protocol): @property def user_id(self) -> str: ... +def _rollover_enabled() -> bool: + return litellm.budget_rollover is True + + +def _rollover_cap(max_budget: float | None) -> float | None: + if max_budget is None or not math.isfinite(max_budget): + return None + return max_budget + + +def _carried_spend(spend: float | None, cap: float | None) -> float: + if cap is None: + return 0.0 + return max(0.0, (spend or 0.0) - cap) + + +def _row_carried_spend(row: _BudgetLinkedRow, caps: Mapping[str, float]) -> float: + if not caps: + return 0.0 + return _carried_spend(row.spend, caps.get(row.budget_id) if row.budget_id is not None else None) + + def _team_membership_counter_key(row: _TeamMembershipRow) -> str: return f"spend:team_member:{row.user_id}:{row.team_id}" @@ -129,6 +161,59 @@ def _budget_link_where( return {"budget_id": {"in": list(budget_ids)}, **extra} +def _queue_budget_linked_resets( + writes: LinkedSpendResetWrites, + cascade: "_BudgetCascade", + extra: Mapping[str, object] = MappingProxyType({}), +) -> None: + """Reset one linked table's spend for every expiring tier: tiers with a + rollover cap keep spend beyond the cap (decrement preserves writes racing + the reset), everything else is zeroed as before. Zero the under-cap rows + BEFORE decrementing the over-cap ones: the statements run sequentially in + one transaction, so the reverse order lets the zero re-match a row the + decrement just moved into the (0, cap] range and erase its carried spend.""" + for budget_id, cap in cascade.rollover_caps.items(): + writes.queue_spend_zero( + where={"budget_id": budget_id, **extra, "spend": {"gt": 0, "lte": cap}} + ) # mutable-ok: prisma where filter must be a dict + writes.queue_spend_decrement( + where={"budget_id": budget_id, **extra, "spend": {"gt": cap}}, amount=cap + ) # mutable-ok: prisma where filter must be a dict + plain_ids: Final = tuple(bid for bid in cascade.budget_ids if bid not in cascade.rollover_caps) + if plain_ids: + writes.queue_spend_zero(where=_budget_link_where(plain_ids, extra)) + + +def _queue_enduser_resets(writes: LinkedSpendResetWrites, cascade: "_BudgetCascade") -> None: + """End users are matched by id rather than budget link: rows with no + budget_id ride the default budget tier (litellm.max_end_user_budget_id). + Zero-before-decrement ordering matters here too (see + _queue_budget_linked_resets).""" + if not cascade.rollover_caps: + if cascade.endusers: + writes.queue_spend_zero( + where={"user_id": {"in": [row.user_id for row in cascade.endusers]}} + ) # mutable-ok: prisma where filter must be a dict + return + tiered: Final = tuple((row.budget_id or litellm.max_end_user_budget_id, row.user_id) for row in cascade.endusers) + for budget_id, cap in cascade.rollover_caps.items(): + if not ( + user_ids := [uid for bid, uid in tiered if bid == budget_id] + ): # mutable-ok: prisma "in" filter takes a list + continue + writes.queue_spend_zero( + where={"user_id": {"in": user_ids}, "spend": {"lte": cap}} + ) # mutable-ok: prisma where filter must be a dict + writes.queue_spend_decrement( + where={"user_id": {"in": user_ids}, "spend": {"gt": cap}}, amount=cap + ) # mutable-ok: prisma where filter must be a dict + plain: Final = [ + uid for bid, uid in tiered if bid is None or bid not in cascade.rollover_caps + ] # mutable-ok: prisma "in" filter takes a list + if plain: + writes.queue_spend_zero(where={"user_id": {"in": plain}}) # mutable-ok: prisma where filter must be a dict + + @dataclass(frozen=True, slots=True) class _BudgetCascade: """Everything one budget-tier reset touches, resolved before any write.""" @@ -137,8 +222,9 @@ class _BudgetCascade: budget_ids: tuple[str, ...] = () budget_resets: tuple[tuple[str, datetime], ...] = () endusers: tuple[_EndUserRow, ...] = () - counter_keys: tuple[str, ...] = () + counter_resets: tuple[tuple[str, float], ...] = () cache_keys: tuple[str, ...] = () + rollover_caps: Mapping[str, float] = MappingProxyType({}) @dataclass(frozen=True, slots=True) @@ -404,8 +490,10 @@ class ResetBudgetJob: ) @staticmethod - async def _invalidate_spend_counter(counter_key: str) -> None: - """Zero a spend counter so a DB-row reset takes effect immediately. + async def _invalidate_spend_counter(counter_key: str, new_spend: float = 0.0) -> None: + """Overwrite a spend counter with the post-reset value (0, or the carried + overage when budget rollover is enabled) so a DB-row reset takes effect + immediately. Call AFTER the DB write commits. Clearing Redis before the DB commit opens a window where get_current_spend reads 0 from Redis @@ -414,10 +502,10 @@ class ResetBudgetJob: try: from litellm.proxy.proxy_server import spend_counter_cache - spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=0.0, ttl=60) + spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=new_spend, ttl=60) if spend_counter_cache.redis_cache is not None: try: - await spend_counter_cache.redis_cache.async_set_cache(key=counter_key, value=0.0, ttl=60) + await spend_counter_cache.redis_cache.async_set_cache(key=counter_key, value=new_spend, ttl=60) except Exception as redis_err: verbose_proxy_logger.warning( "Failed to reset spend counter %s in Redis: %s. " @@ -522,6 +610,15 @@ class ResetBudgetJob: where=_budget_link_where(budget_ids, _SPENT_ROWS_WHERE), log_subject="tags", ) + rollover_caps: Final[Mapping[str, float]] = MappingProxyType( + { # mutable-ok: MappingProxyType wraps a one-shot dict comprehension + b.budget_id: cap + for b in budgets_to_reset + if b.budget_id is not None and (cap := _rollover_cap(b.max_budget)) is not None + } + if _rollover_enabled() + else {} # mutable-ok: empty sentinel immediately frozen by MappingProxyType + ) return _BudgetCascade( budgets=tuple(budgets_to_reset), budget_ids=budget_ids, @@ -534,12 +631,16 @@ class ResetBudgetJob: if b.budget_id is not None and b.budget_duration is not None ), endusers=await self._collect_endusers_to_reset(budget_ids), - counter_keys=( - *(_team_membership_counter_key(row) for row in team_memberships), - *(_key_counter_key(row) for row in keys), - *(_org_counter_key(row) for row in orgs), - *(_tag_counter_key(row) for row in tags), + counter_resets=( + *( + (_team_membership_counter_key(row), _row_carried_spend(row, rollover_caps)) + for row in team_memberships + ), + *((_key_counter_key(row), _row_carried_spend(row, rollover_caps)) for row in keys), + *((_org_counter_key(row), _row_carried_spend(row, rollover_caps)) for row in orgs), + *((_tag_counter_key(row), _row_carried_spend(row, rollover_caps)) for row in tags), ), + rollover_caps=rollover_caps, cache_keys=( *(key for row in team_memberships for key in _team_membership_cache_keys(row)), *(key for row in keys for key in _key_cache_keys(row)), @@ -565,20 +666,18 @@ class ResetBudgetJob: ) async def _commit_budget_cascade_once(self, cascade: _BudgetCascade) -> None: - enduser_ids: Final = tuple(row.user_id for row in cascade.endusers) async with budget_cascade_unit_of_work(self.prisma_client.db.batch_) as uow: - uow.team_memberships.queue_spend_zero(where=_budget_link_where(cascade.budget_ids)) - uow.keys.queue_spend_zero(where=_budget_link_where(cascade.budget_ids, _LINKED_KEYS_WHERE)) - uow.organizations.queue_spend_zero(where=_budget_link_where(cascade.budget_ids, _SPENT_ROWS_WHERE)) - uow.tags.queue_spend_zero(where=_budget_link_where(cascade.budget_ids, _SPENT_ROWS_WHERE)) - if enduser_ids: - uow.endusers.queue_spend_zero(where={"user_id": {"in": list(enduser_ids)}}) + _queue_budget_linked_resets(uow.team_memberships, cascade) + _queue_budget_linked_resets(uow.keys, cascade, extra=_LINKED_KEYS_WHERE) + _queue_budget_linked_resets(uow.organizations, cascade, extra=_SPENT_ROWS_WHERE) + _queue_budget_linked_resets(uow.tags, cascade, extra=_SPENT_ROWS_WHERE) + _queue_enduser_resets(uow.endusers, cascade) for budget_id, budget_reset_at in cascade.budget_resets: uow.budgets.queue_window_advance(budget_id=budget_id, budget_reset_at=budget_reset_at) async def _invalidate_budget_cascade_caches(self, cascade: _BudgetCascade) -> None: - for counter_key in cascade.counter_keys: - await self._invalidate_spend_counter(counter_key) + for counter_key, new_spend in cascade.counter_resets: + await self._invalidate_spend_counter(counter_key, new_spend=new_spend) for cache_key in cascade.cache_keys: await self._invalidate_user_api_key_cache_entry(cache_key) @@ -708,7 +807,11 @@ class ResetBudgetJob: for k in updated_keys: if k.token is None: continue - uow.keys.queue_spend_reset(token=k.token, budget_reset_at=k.budget_reset_at) + uow.keys.queue_spend_reset( + token=k.token, + budget_reset_at=k.budget_reset_at, + spend_decrement=k.max_budget if (k.spend or 0.0) > 0.0 else None, + ) async def _write_user_reset_updates(self, updated_users: list[LiteLLM_UserTable]) -> None: """ @@ -726,7 +829,11 @@ class ResetBudgetJob: async def _write_user_reset_updates_once(self, updated_users: list[LiteLLM_UserTable]) -> None: async with spend_reset_unit_of_work(self.prisma_client.db.batch_) as uow: for u in updated_users: - uow.users.queue_spend_reset(user_id=u.user_id, budget_reset_at=u.budget_reset_at) + uow.users.queue_spend_reset( + user_id=u.user_id, + budget_reset_at=u.budget_reset_at, + spend_decrement=u.max_budget if (u.spend or 0.0) > 0.0 else None, + ) async def _write_team_reset_updates(self, updated_teams: list[LiteLLM_TeamTable]) -> None: """ @@ -744,7 +851,11 @@ class ResetBudgetJob: async def _write_team_reset_updates_once(self, updated_teams: list[LiteLLM_TeamTable]) -> None: async with spend_reset_unit_of_work(self.prisma_client.db.batch_) as uow: for t in updated_teams: - uow.teams.queue_spend_reset(team_id=t.team_id, budget_reset_at=t.budget_reset_at) + uow.teams.queue_spend_reset( + team_id=t.team_id, + budget_reset_at=t.budget_reset_at, + spend_decrement=t.max_budget if (t.spend or 0.0) > 0.0 else None, + ) def _emit_phase_failure( self, @@ -820,7 +931,7 @@ class ResetBudgetJob: for k in updated_keys: token = getattr(k, "token", None) if token: - await self._invalidate_spend_counter(f"spend:key:{token}") + await self._invalidate_spend_counter(f"spend:key:{token}", new_spend=k.spend or 0.0) end_time = time.time() outcome: Final = _ChunkOutcome( @@ -925,7 +1036,7 @@ class ResetBudgetJob: for u in updated_users: user_id = getattr(u, "user_id", None) if user_id: - await self._invalidate_spend_counter(f"spend:user:{user_id}") + await self._invalidate_spend_counter(f"spend:user:{user_id}", new_spend=u.spend or 0.0) if user_id == LITELLM_PROXY_BUDGET_NAME: await self._invalidate_global_proxy_spend_cache() @@ -1034,7 +1145,7 @@ class ResetBudgetJob: for t in updated_teams: team_id = getattr(t, "team_id", None) if team_id: - await self._invalidate_spend_counter(f"spend:team:{team_id}") + await self._invalidate_spend_counter(f"spend:team:{team_id}", new_spend=t.spend or 0.0) end_time = time.time() outcome: Final = _ChunkOutcome( @@ -1107,10 +1218,11 @@ class ResetBudgetJob: reset_at: Final = datetime.fromisoformat(reset_at_str.replace("Z", "+00:00")).replace(tzinfo=None) if reset_at > now: return False - spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=0.0) + new_value: Final = await ResetBudgetJob._window_carried_spend(window, counter_key, spend_counter_cache) + spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=new_value) if spend_counter_cache.redis_cache is not None: try: - await spend_counter_cache.redis_cache.async_set_cache(key=counter_key, value=0.0) + await spend_counter_cache.redis_cache.async_set_cache(key=counter_key, value=new_value) except Exception as redis_err: verbose_proxy_logger.warning("Failed to reset Redis counter %s: %s", counter_key, redis_err) window["reset_at"] = compute_budget_reset_at( @@ -1118,6 +1230,27 @@ class ResetBudgetJob: ).isoformat() return True + @staticmethod + async def _window_carried_spend( + window: Mapping[str, object], counter_key: str, spend_counter_cache: DualCache + ) -> float: + """Per-window spend lives only in the counter, so the carried overage is + read from it before the reset overwrites it.""" + if not _rollover_enabled(): + return 0.0 + window_max: Final = window.get("max_budget") + cap: Final = _rollover_cap(window_max) if isinstance(window_max, (int, float)) else None + if cap is None: + return 0.0 + try: + current: Final = await spend_counter_cache.async_get_cache(key=counter_key) + except Exception as e: # noqa: BLE001 # an unreadable counter falls back to a plain zero reset + verbose_proxy_logger.warning("Failed to read spend counter %s for rollover: %s", counter_key, e) + return 0.0 + if not isinstance(current, (int, float)): + return 0.0 + return _carried_spend(float(current), cap) + async def reset_budget_windows(self) -> None: """ For keys and teams with budget_limits, reset any individual windows where @@ -1222,7 +1355,7 @@ class ResetBudgetJob: still holds the pre-reset value, admitting requests past the cap. """ try: - item.spend = 0.0 + item.spend = _carried_spend(item.spend, _rollover_cap(item.max_budget)) if _rollover_enabled() else 0.0 if hasattr(item, "budget_duration") and item.budget_duration is not None: item.budget_reset_at = compute_budget_reset_at( budget_duration=item.budget_duration, settings=reset_settings diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 99c3ccd915f..1d33c9ce393 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -16415,6 +16415,13 @@ _GENERAL_SETTINGS_UI_LITELLM_FIELDS: Final[dict[str, GeneralSettingsUILiteLLMFie "tab": "prompt_caching", "description": "Empty uses Anthropic's 5m default. 1h suits long sessions but doubles the cache write cost.", }, + "budget_rollover": { # mutable-ok: registry literal, frozen with its siblings below + "type": "Boolean", + "description": ( + "Carry spend beyond max_budget into the next window when budgets reset, instead of " + "forgiving it. Applies to key, user, team, team member, org, tag and end-user budgets." + ), + }, "max_ui_session_budget": { "type": "Dollar", "default": 1.0, diff --git a/litellm/repositories/unit_of_work.py b/litellm/repositories/unit_of_work.py index e504baceb9f..eb11ebe3b9c 100644 --- a/litellm/repositories/unit_of_work.py +++ b/litellm/repositories/unit_of_work.py @@ -19,32 +19,57 @@ from collections.abc import AsyncGenerator, Callable, Mapping from contextlib import asynccontextmanager from dataclasses import dataclass from datetime import datetime +from typing import Final from litellm.repositories.prisma_protocols import BatchTable, PrismaBatch +def _spend_reset_data(budget_reset_at: datetime | None, spend_decrement: float | None) -> Mapping[str, object]: + spend: Final[object] = ( + {"decrement": spend_decrement} # mutable-ok: prisma update payload must be a dict + if spend_decrement is not None + else 0 + ) + return {"spend": spend, "budget_reset_at": budget_reset_at} # mutable-ok: prisma update payload must be a dict + + @dataclass(frozen=True, slots=True) class KeySpendResetWrites: table: BatchTable - def queue_spend_reset(self, token: str, budget_reset_at: datetime | None) -> None: - self.table.update(where={"token": token}, data={"spend": 0, "budget_reset_at": budget_reset_at}) + def queue_spend_reset( + self, token: str, budget_reset_at: datetime | None, spend_decrement: float | None = None + ) -> None: + self.table.update( + where={"token": token}, # mutable-ok: prisma where filter must be a dict + data=_spend_reset_data(budget_reset_at, spend_decrement), + ) @dataclass(frozen=True, slots=True) class UserSpendResetWrites: table: BatchTable - def queue_spend_reset(self, user_id: str, budget_reset_at: datetime | None) -> None: - self.table.update(where={"user_id": user_id}, data={"spend": 0, "budget_reset_at": budget_reset_at}) + def queue_spend_reset( + self, user_id: str, budget_reset_at: datetime | None, spend_decrement: float | None = None + ) -> None: + self.table.update( + where={"user_id": user_id}, # mutable-ok: prisma where filter must be a dict + data=_spend_reset_data(budget_reset_at, spend_decrement), + ) @dataclass(frozen=True, slots=True) class TeamSpendResetWrites: table: BatchTable - def queue_spend_reset(self, team_id: str, budget_reset_at: datetime | None) -> None: - self.table.update(where={"team_id": team_id}, data={"spend": 0, "budget_reset_at": budget_reset_at}) + def queue_spend_reset( + self, team_id: str, budget_reset_at: datetime | None, spend_decrement: float | None = None + ) -> None: + self.table.update( + where={"team_id": team_id}, # mutable-ok: prisma where filter must be a dict + data=_spend_reset_data(budget_reset_at, spend_decrement), + ) @dataclass(frozen=True, slots=True) @@ -54,6 +79,14 @@ class LinkedSpendResetWrites: def queue_spend_zero(self, where: Mapping[str, object]) -> None: self.table.update_many(where=where, data={"spend": 0}) + def queue_spend_decrement(self, where: Mapping[str, object], amount: float) -> None: + """``decrement`` rather than a read-then-set, so spend written between the + cascade's read and its commit survives the reset instead of being erased.""" + self.table.update_many( + where=where, + data={"spend": {"decrement": amount}}, # mutable-ok: prisma update payload must be a dict + ) + @dataclass(frozen=True, slots=True) class BudgetWindowWrites: diff --git a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py index aa844304604..5d3afd95a55 100644 --- a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py +++ b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py @@ -1458,7 +1458,7 @@ def test_budget_cascade_writes_land_in_a_single_transaction(reset_budget_job, mo budget = _budget_row(budget_id="budget-1", budget_duration="7d") mock_prisma_client.data["budget"] = [budget] mock_prisma_client.data["enduser"] = [ - type("EndUser", (), {"spend": 5.0, "litellm_budget_table": budget, "user_id": "enduser-1"}) + type("EndUser", (), {"spend": 5.0, "litellm_budget_table": budget, "user_id": "enduser-1", "budget_id": "budget-1"}) ] asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table()) @@ -2588,3 +2588,303 @@ def test_ambiguous_commit_replay_does_not_erase_newly_accrued_spend( assert client.key_spend == expected_spend assert client.commit_attempts == expected_commits assert client.reconnect_reasons == expected_reconnects + + +# --------------------------------------------------------------------------- +# Budget rollover (LIT-3085): overage beyond max_budget carries into the next +# window instead of being forgiven +# --------------------------------------------------------------------------- + + +@pytest.fixture +def rollover_enabled(monkeypatch): + import litellm + + monkeypatch.setattr(litellm, "budget_rollover", True) + + +@pytest.mark.parametrize( + "run_phase, table, id_field, id_value, row_factory", + [ + ( + lambda job: job.reset_budget_for_litellm_keys(), + "key", + "token", + "tok-roll", + lambda now: type( + "Key", + (), + { + "spend": 150.0, + "max_budget": 100.0, + "budget_duration": "1d", + "budget_reset_at": now, + "token": "tok-roll", + }, + ), + ), + ( + lambda job: job.reset_budget_for_litellm_users(), + "user", + "user_id", + "user-roll", + lambda now: type( + "User", + (), + { + "spend": 150.0, + "max_budget": 100.0, + "budget_duration": "30d", + "budget_reset_at": now, + "user_id": "user-roll", + }, + ), + ), + ( + lambda job: job.reset_budget_for_litellm_teams(), + "team", + "team_id", + "team-roll", + lambda now: type( + "Team", + (), + { + "spend": 150.0, + "max_budget": 100.0, + "budget_duration": "1mo", + "budget_reset_at": now, + "team_id": "team-roll", + }, + ), + ), + ], +) +def test_direct_reset_carries_overage_when_rollover_enabled( + rollover_enabled, reset_budget_job, mock_prisma_client, monkeypatch, run_phase, table, id_field, id_value, row_factory +): + """spend=150 against max_budget=100 must decrement by the cap (leaving 50) + rather than zero the row, and the spend counter must be seeded with 50.""" + counter_cache = _make_counter_invalidation_job(monkeypatch) + now = datetime.now(timezone.utc) + mock_prisma_client.data[table] = [row_factory(now)] + + asyncio.run(run_phase(reset_budget_job)) + + writes = _batch_writes(mock_prisma_client, table) + assert len(writes) == 1 + assert writes[0]["where"] == {id_field: id_value} + assert writes[0]["data"]["spend"] == {"decrement": 100.0} + assert writes[0]["data"]["budget_reset_at"] > now + counter_prefix = {"key": "spend:key", "user": "spend:user", "team": "spend:team"}[table] + counter_cache.in_memory_cache.set_cache.assert_any_call(key=f"{counter_prefix}:{id_value}", value=50.0, ttl=60) + + +def test_direct_reset_zeroes_under_budget_row_even_with_rollover( + rollover_enabled, reset_budget_job, mock_prisma_client, monkeypatch +): + counter_cache = _make_counter_invalidation_job(monkeypatch) + now = datetime.now(timezone.utc) + mock_prisma_client.data["key"] = [ + type( + "Key", + (), + {"spend": 40.0, "max_budget": 100.0, "budget_duration": "1d", "budget_reset_at": now, "token": "tok-under"}, + ) + ] + + asyncio.run(reset_budget_job.reset_budget_for_litellm_keys()) + + assert _batch_writes(mock_prisma_client, "key")[0]["data"]["spend"] == 0 + counter_cache.in_memory_cache.set_cache.assert_any_call(key="spend:key:tok-under", value=0.0, ttl=60) + + +def test_direct_reset_zeroes_row_without_max_budget_even_with_rollover( + rollover_enabled, reset_budget_job, mock_prisma_client, monkeypatch +): + """No cap means nothing to carry against: reset to zero as before.""" + _make_counter_invalidation_job(monkeypatch) + now = datetime.now(timezone.utc) + mock_prisma_client.data["key"] = [ + type( + "Key", + (), + {"spend": 150.0, "max_budget": None, "budget_duration": "1d", "budget_reset_at": now, "token": "tok-nocap"}, + ) + ] + + asyncio.run(reset_budget_job.reset_budget_for_litellm_keys()) + + assert _batch_writes(mock_prisma_client, "key")[0]["data"]["spend"] == 0 + + +def test_budget_cascade_carries_overage_per_tier_when_rollover_enabled( + rollover_enabled, reset_budget_job, mock_prisma_client, monkeypatch +): + """A team member 5 over the tier cap keeps a spend of 5 in the next window: + the cascade decrements over-cap rows by the cap, zeroes the rest, and seeds + the spend counter with the carried amount.""" + counter_cache = _make_counter_invalidation_job(monkeypatch) + budget = _budget_row(budget_id="budget-roll", budget_duration="7d", max_budget=10.0) + mock_prisma_client.data["budget"] = [budget] + membership = type( + "Membership", + (), + {"user_id": "member-1", "team_id": "team-1", "spend": 15.0, "budget_id": "budget-roll"}, + ) + mock_prisma_client.db.litellm_teammembership.set_find_many_results([membership]) + + asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table()) + + membership_writes = _batch_writes(mock_prisma_client, "team_membership") + assert { + "table": "team_membership", + "op": "update_many", + "where": {"budget_id": "budget-roll", "spend": {"gt": 10.0}}, + "data": {"spend": {"decrement": 10.0}}, + } in membership_writes + assert { + "table": "team_membership", + "op": "update_many", + "where": {"budget_id": "budget-roll", "spend": {"gt": 0, "lte": 10.0}}, + "data": {"spend": 0}, + } in membership_writes + counter_cache.in_memory_cache.set_cache.assert_any_call(key="spend:team_member:member-1:team-1", value=5.0, ttl=60) + + +def test_budget_cascade_carries_enduser_overage_when_rollover_enabled( + rollover_enabled, reset_budget_job, mock_prisma_client, monkeypatch +): + _make_counter_invalidation_job(monkeypatch) + budget = _budget_row(budget_id="budget-roll", budget_duration="1d", max_budget=10.0) + mock_prisma_client.data["budget"] = [budget] + mock_prisma_client.data["enduser"] = [ + type( + "EndUser", + (), + {"spend": 15.0, "litellm_budget_table": budget, "user_id": "enduser-roll", "budget_id": "budget-roll"}, + ) + ] + + asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table()) + + enduser_writes = _batch_writes(mock_prisma_client, "enduser") + assert { + "table": "enduser", + "op": "update_many", + "where": {"user_id": {"in": ["enduser-roll"]}, "spend": {"gt": 10.0}}, + "data": {"spend": {"decrement": 10.0}}, + } in enduser_writes + assert { + "table": "enduser", + "op": "update_many", + "where": {"user_id": {"in": ["enduser-roll"]}, "spend": {"lte": 10.0}}, + "data": {"spend": 0}, + } in enduser_writes + + +def _replay_spend_writes(writes, spend): + """Apply the queued update_many statements in order, the way the DB + transaction executes them, and return the row's final spend.""" + for write in writes: + condition = write["where"].get("spend") + if isinstance(condition, dict): + if "gt" in condition and not spend > condition["gt"]: + continue + if "lte" in condition and not spend <= condition["lte"]: + continue + payload = write["data"]["spend"] + spend = payload if not isinstance(payload, dict) else spend - payload["decrement"] + return spend + + +@pytest.mark.parametrize("table", ["team_membership", "enduser"]) +def test_cascade_rollover_writes_survive_sequential_execution( + rollover_enabled, reset_budget_job, mock_prisma_client, monkeypatch, table +): + """The statements run one after another inside a transaction, so a + decrement-then-zero order would re-match the decremented row (now in the + 0..cap range) and erase the carried spend. Replaying the writes in queue + order must leave the overage, for any spend between cap and twice the cap.""" + _make_counter_invalidation_job(monkeypatch) + budget = _budget_row(budget_id="budget-roll", budget_duration="7d", max_budget=10.0) + mock_prisma_client.data["budget"] = [budget] + membership = type( + "Membership", + (), + {"user_id": "member-1", "team_id": "team-1", "spend": 15.0, "budget_id": "budget-roll"}, + ) + mock_prisma_client.db.litellm_teammembership.set_find_many_results([membership]) + mock_prisma_client.data["enduser"] = [ + type( + "EndUser", + (), + {"spend": 15.0, "litellm_budget_table": budget, "user_id": "enduser-roll", "budget_id": "budget-roll"}, + ) + ] + + asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table()) + + writes = _batch_writes(mock_prisma_client, table) + assert _replay_spend_writes(writes, 15.0) == 5.0 + assert _replay_spend_writes(writes, 8.0) == 0 + assert _replay_spend_writes(writes, 25.0) == 15.0 + + +def test_budget_cascade_zeroes_everything_when_rollover_disabled(reset_budget_job, mock_prisma_client, monkeypatch): + """Control: with the flag off the cascade keeps the plain zeroing writes.""" + _make_counter_invalidation_job(monkeypatch) + budget = _budget_row(budget_id="budget-off", budget_duration="7d", max_budget=10.0) + mock_prisma_client.data["budget"] = [budget] + + asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table()) + + membership_writes = _batch_writes(mock_prisma_client, "team_membership") + assert membership_writes == [ + { + "table": "team_membership", + "op": "update_many", + "where": {"budget_id": {"in": ["budget-off"]}}, + "data": {"spend": 0}, + } + ] + + +def test_window_reset_carries_counter_overage_when_rollover_enabled(rollover_enabled, monkeypatch): + """A per-window counter at 130 against a 100 cap restarts the window at 30.""" + now = datetime.utcnow() + expired = (now - timedelta(minutes=5)).isoformat() + "Z" + key_rows = [ + { + "token": "sk-roll", + "budget_limits": [{"budget_duration": "1d", "reset_at": expired, "max_budget": 100.0}], + } + ] + job, prisma_client, spend_counter_cache = _make_reset_budget_windows_job( + monkeypatch, key_rows=key_rows, team_rows=[] + ) + spend_counter_cache.async_get_cache = AsyncMock(return_value=130.0) + + asyncio.run(job.reset_budget_windows()) + + prisma_client.db.litellm_verificationtoken.update.assert_awaited_once() + spend_counter_cache.in_memory_cache.set_cache.assert_any_call(key="spend:key:sk-roll:window:1d", value=30.0) + + +def test_window_reset_zeroes_counter_when_rollover_disabled(monkeypatch): + now = datetime.utcnow() + expired = (now - timedelta(minutes=5)).isoformat() + "Z" + key_rows = [ + { + "token": "sk-off", + "budget_limits": [{"budget_duration": "1d", "reset_at": expired, "max_budget": 100.0}], + } + ] + job, prisma_client, spend_counter_cache = _make_reset_budget_windows_job( + monkeypatch, key_rows=key_rows, team_rows=[] + ) + spend_counter_cache.async_get_cache = AsyncMock(return_value=130.0) + + asyncio.run(job.reset_budget_windows()) + + spend_counter_cache.in_memory_cache.set_cache.assert_any_call(key="spend:key:sk-off:window:1d", value=0.0) + spend_counter_cache.async_get_cache.assert_not_awaited() From 390595c626db37c231b830f23746f3ae0f473b5a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2026 12:48:11 -0700 Subject: [PATCH 23/29] fix(auth): skip guaranteed-miss team lookup for the litellm-dashboard sentinel (#38471) * fix(auth): skip guaranteed-miss team lookup for the litellm-dashboard sentinel Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style: ruff format Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: assert builder result instead of swallowing exceptions; drop redundant comment Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/auth/user_api_key_auth.py | 15 +- .../proxy/auth/test_user_api_key_auth.py | 137 ++++++++++++++++++ 2 files changed, 149 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 90a16052b71..e92d090a2fb 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -1769,7 +1769,12 @@ async def _user_api_key_auth_builder( return valid_token - if valid_token is not None and isinstance(valid_token, UserAPIKeyAuth) and valid_token.team_id is not None: + if ( + valid_token is not None + and isinstance(valid_token, UserAPIKeyAuth) + and valid_token.team_id is not None + and valid_token.team_id != UI_TEAM_ID + ): ## UPDATE TEAM VALUES BASED ON CACHED TEAM OBJECT - allows `/team/update` values to work for cached token try: team_obj: Final[LiteLLM_TeamTableCachedObj] = await get_team_object( @@ -2149,6 +2154,8 @@ async def _user_api_key_auth_builder( # Check 6: Additional Common Checks across jwt + key auth if valid_token.team_id is not None: try: + if valid_token.team_id == UI_TEAM_ID: + raise TeamNotFoundError(team_id=UI_TEAM_ID) with tracer.trace("litellm.proxy.auth.get_team_object"): _team_obj = await get_team_object( team_id=valid_token.team_id, @@ -2443,7 +2450,7 @@ async def _run_centralized_common_checks( ) fetch_coros: Final = [] - if user_api_key_auth_obj.team_id is not None: + if user_api_key_auth_obj.team_id is not None and user_api_key_auth_obj.team_id != UI_TEAM_ID: fetch_coros.append( _safe_fetch( "team", @@ -2567,7 +2574,9 @@ async def _run_centralized_common_checks( else: raise team_result else: - team_object = team_result + team_object = ( + _team_obj_from_token(user_api_key_auth_obj) if user_api_key_auth_obj.team_id == UI_TEAM_ID else team_result + ) user_object: LiteLLM_UserTable | None = None if isinstance(user_result, BaseException) else user_result project_object: Final[LiteLLM_ProjectTableCachedObj | None] = ( diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index 6a117985820..d44f96d95bf 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -4972,6 +4972,143 @@ async def test_centralized_common_checks_ui_sentinel_team_vouches_despite_absent setattr(_proxy_server_mod, k, v) +@pytest.mark.asyncio +async def test_centralized_common_checks_ui_sentinel_team_skips_db_lookup(): + """LIT-6297 / GH#28775: ``UI_TEAM_ID`` never has a team row and the + not-found path bypasses the DB throttle, so building the team fetch for it + cost one guaranteed-miss ``LiteLLM_TeamTable.find_unique`` plus a 404 debug + log on every dashboard request. The gate must not call ``get_team_object`` + for the sentinel at all, while the token-derived team object still reaches + ``common_checks``.""" + import litellm.proxy.proxy_server as _proxy_server_mod + from fastapi import Request + from starlette.datastructures import URL + + from litellm.proxy._types import UI_TEAM_ID, LiteLLM_TeamTableCachedObj + + token = UserAPIKeyAuth( + api_key="sk-test", + user_id="ui-session-user", + team_id=UI_TEAM_ID, + models=[], + team_models=[], + ) + request = Request(scope={"type": "http"}) + request._url = URL(url="/user/info") + request._body = b"{}" + + received_team_objects: list[LiteLLM_TeamTableCachedObj | None] = [] + + async def _capturing_common_checks(*_args, **kwargs) -> bool: + received_team_objects.append(kwargs.get("team_object")) + return True + + attrs = _proxy_attrs_for_centralized_checks(user_custom_auth=None) + originals = {a: getattr(_proxy_server_mod, a, None) for a in attrs} + try: + for k, v in attrs.items(): + setattr(_proxy_server_mod, k, v) + with ( + patch( # test-quality-ok: the regression IS that this DB lookup is never made for the sentinel + "litellm.proxy.auth.user_api_key_auth.get_team_object", + new_callable=AsyncMock, + ) as mock_get_team_object, + patch( # test-quality-ok: capture the team_object the consumer receives without a DB + "litellm.proxy.auth.user_api_key_auth.common_checks", + _capturing_common_checks, + ), + ): + await _run_centralized_common_checks( + user_api_key_auth_obj=token, + request=request, + request_data={}, + route="/user/info", + ) + mock_get_team_object.assert_not_awaited() + assert len(received_team_objects) == 1 + received_team_object = received_team_objects[0] + assert received_team_object is not None + assert received_team_object.team_id == UI_TEAM_ID + finally: + for k, v in originals.items(): + setattr(_proxy_server_mod, k, v) + + +@pytest.mark.asyncio +async def test_builder_ui_sentinel_team_never_hits_get_team_object(): # test-quality-ok: absence of the guaranteed-miss DB call is the observable being pinned + """Companion to the centralized-gate test for the builder path: the cached + UI session token's team refresh and the post-validation team fetch must + both skip ``get_team_object`` for ``UI_TEAM_ID`` instead of 404ing on + every request.""" + import litellm.proxy.proxy_server as _proxy_server_mod + from fastapi import Request + from starlette.datastructures import URL + + from litellm.proxy._types import UI_TEAM_ID + from litellm.proxy.auth.user_api_key_auth import _user_api_key_auth_builder + from litellm.proxy.proxy_server import hash_token + + api_key = "sk-test-ui-session-key" + cached_token = UserAPIKeyAuth( + api_key=api_key, + token=hash_token(api_key), + user_id="ui-session-user", + user_role=LitellmUserRoles.INTERNAL_USER, + team_id=UI_TEAM_ID, + ) + + mock_proxy_logging_obj = MagicMock() + mock_proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) + + attrs = { + "prisma_client": MagicMock(), + "user_api_key_cache": DualCache(), + "proxy_logging_obj": mock_proxy_logging_obj, + "master_key": "sk-master-key", + "general_settings": {}, + "llm_model_list": [], + "llm_router": None, + "open_telemetry_logger": None, + "model_max_budget_limiter": MagicMock(), + "user_custom_auth": None, + "jwt_handler": None, + "litellm_proxy_admin_name": "admin", + } + originals = {a: getattr(_proxy_server_mod, a, None) for a in attrs} + try: + for k, v in attrs.items(): + setattr(_proxy_server_mod, k, v) + + request = Request(scope={"type": "http"}) + request._url = URL(url="/user/info") + + with ( + patch( # test-quality-ok: seed the cached UI session token without a DB + "litellm.proxy.auth.resolvers.store.IdentityStore._resolve_key", + new_callable=AsyncMock, + return_value=cached_token, + ), + patch( # test-quality-ok: the regression IS that this DB lookup is never made for the sentinel + "litellm.proxy.auth.user_api_key_auth.get_team_object", + new_callable=AsyncMock, + ) as mock_get_team_object, + ): + result = await _user_api_key_auth_builder( + request=request, + api_key=f"Bearer {api_key}", + azure_api_key_header="", + anthropic_api_key_header=None, + google_ai_studio_api_key_header=None, + azure_apim_header=None, + request_data={}, + ) + assert result.team_id == UI_TEAM_ID + mock_get_team_object.assert_not_awaited() + finally: + for k, v in originals.items(): + setattr(_proxy_server_mod, k, v) + + @pytest.mark.asyncio async def test_centralized_common_checks_user_http_exception_isolates_to_user_only(): """Per-fetch isolation, mirror of the team case: an HTTPException From 6b1844442cfac06cc7266d67c3bde4a9cc50701c Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2026 12:51:17 -0700 Subject: [PATCH 24/29] fix(key_management): allow /key/update to keep or shrink MCP server grants the key already holds (#38463) * fix(key_management): allow /key/update to keep or shrink MCP server grants the key already holds Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(key_management): reuse key row's included object_permission instead of a second lookup Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../key_management_endpoints.py | 4 +- .../object_permission_utils.py | 42 ++++++- .../test_key_management_endpoints.py | 64 +++++++++- .../test_object_permission_utils.py | 118 ++++++++++++++++++ 4 files changed, 225 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index d1c08352919..97999cbb6d7 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -2190,7 +2190,7 @@ async def _get_and_validate_existing_key( existing_key_row: Final[LiteLLM_VerificationToken | None] = await _prisma_table( VerificationTokenRepository(prisma_client) - ).find_unique(where={"token": hashed_token}) + ).find_unique(where={"token": hashed_token}, include={"object_permission": True}) if existing_key_row is None: raise ProxyException( @@ -2442,11 +2442,13 @@ async def _validate_mcp_servers_for_key_update( check_db_only=True, ) object_permission_dict: Final = _object_permission_to_dict(data.object_permission) + team_unchanged: Final = data.team_id is None or data.team_id == existing_key_row.team_id normalized_object_permission: Final = await validate_key_mcp_servers_against_team( object_permission=object_permission_dict, team_obj=effective_team_obj, prisma_client=prisma_client, is_proxy_admin=is_proxy_admin, + existing_key_object_permission=existing_key_row.object_permission if team_unchanged else None, ) await validate_key_search_tools_against_team( object_permission=object_permission_dict, diff --git a/litellm/proxy/management_helpers/object_permission_utils.py b/litellm/proxy/management_helpers/object_permission_utils.py index fb64914f6f5..13080a6cf83 100644 --- a/litellm/proxy/management_helpers/object_permission_utils.py +++ b/litellm/proxy/management_helpers/object_permission_utils.py @@ -447,6 +447,36 @@ async def enforce_all_proxy_mcp_servers_grant_is_admin_only( ) +async def _get_grandfathered_key_mcp_server_ids( + existing_object_permission: Optional["LiteLLM_ObjectPermissionTable"], + prisma_client: PrismaClient | None, +) -> frozenset[str]: + """ + Resolve the canonical MCP server IDs a key's stored object_permission already + grants. Updates that keep or shrink those grants stay valid even when the + team allowlist has since changed; sentinels are excluded so they cannot + grandfather anything. + """ + if existing_object_permission is None or prisma_client is None: + return frozenset() + raw_tool_perms: Final = existing_object_permission.mcp_tool_permissions or {} + tool_perm_keys: Final[frozenset[str]] = frozenset( + json.loads(raw_tool_perms).keys() if isinstance(raw_tool_perms, str) else raw_tool_perms.keys() + ) + identifiers: Final = (frozenset(existing_object_permission.mcp_servers or []) | tool_perm_keys) - { + SpecialMCPServerNames.no_mcp_servers.value, + SpecialMCPServerName.all_proxy_servers.value, + } + return frozenset( + _flatten_resolved_mcp_server_ids( + await _resolve_mcp_server_identifiers_to_ids( + identifiers=set(identifiers), + prisma_client=prisma_client, + ) + ) + ) + + async def _get_team_allowed_mcp_servers( team_obj: Optional["LiteLLM_TeamTableCachedObj"], prisma_client: PrismaClient | None = None, @@ -527,10 +557,16 @@ async def validate_key_mcp_servers_against_team( team_obj: Optional["LiteLLM_TeamTableCachedObj"], prisma_client: PrismaClient | None = None, is_proxy_admin: bool = False, + existing_key_object_permission: Optional["LiteLLM_ObjectPermissionTable"] = None, ) -> ObjectPermissionDict | None: """ Validate that MCP servers requested on a key are within the allowed scope. + When ``existing_key_object_permission`` is provided (key updates), servers + the key already holds are grandfathered: keeping or removing them stays valid + even if the team allowlist has since shrunk, while adding new servers outside + the allowlist is still rejected. + Rules: - If key is in a team: key's mcp_servers must be a subset of (team's allowed servers + allow_all_keys servers) @@ -589,7 +625,11 @@ async def validate_key_mcp_servers_against_team( if teamless_admin_assignment: allowed_servers = all_allowed_servers | active_requested_servers - disallowed_servers: Final = active_requested_servers - allowed_servers + grandfathered_servers: Final = await _get_grandfathered_key_mcp_server_ids( + existing_object_permission=existing_key_object_permission, + prisma_client=prisma_client, + ) + disallowed_servers: Final = active_requested_servers - allowed_servers - grandfathered_servers if disallowed_servers: if team_obj is not None: team_id = team_obj.team_id diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index a37c4f72b3d..6045b64023d 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -730,6 +730,68 @@ async def test_update_key_personal_non_admin_denied_vector_stores(monkeypatch): assert "Vector stores" in str(exc.value.detail) +@pytest.mark.asyncio +async def test_update_key_grandfathers_existing_mcp_servers(monkeypatch): + """/key/update on a team key that already holds MCP servers outside the + team allowlist must accept re-sent or shrunk grants (LIT-6062). The wrapper + must pass the existing key's object_permission row into the validator when + the team is unchanged.""" + from unittest.mock import AsyncMock, MagicMock + + from litellm.proxy._types import ( + LiteLLM_ObjectPermissionBase, + UpdateKeyRequest, + ) + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _validate_mcp_servers_for_key_update, + ) + + existing_row = MagicMock() + existing_row.mcp_servers = ["server-a", "server-b"] + existing_row.mcp_tool_permissions = {} + mock_prisma = MagicMock() + mock_prisma.db.litellm_mcpservertable.find_many = AsyncMock(return_value=[]) + + team_obj = MagicMock() + team_obj.team_id = "team-1" + team_obj.object_permission = None + + existing_key_row = MagicMock( + team_id="team-1", + object_permission_id="perm-1", + object_permission=existing_row, + ) + + mock_server_a = MagicMock() + mock_server_a.server_id = "server-a" + mock_server_b = MagicMock() + mock_server_b.server_id = "server-b" + mock_mgr = MagicMock() + mock_mgr.get_registry.return_value = { + "server-a": mock_server_a, + "server-b": mock_server_b, + } + mock_mgr.get_allow_all_keys_server_ids.return_value = [] + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + mock_mgr, + ) + + result = await _validate_mcp_servers_for_key_update( + data=UpdateKeyRequest( + key="sk-team-key", + object_permission=LiteLLM_ObjectPermissionBase(mcp_servers=["server-a"]), + ), + team_obj=team_obj, + existing_key_row=existing_key_row, + prisma_client=mock_prisma, + user_api_key_cache=MagicMock(), + is_proxy_admin=False, + ) + assert result is not None + assert result["mcp_servers"] == ["server-a"] + + @pytest.mark.asyncio async def test_update_key_personal_non_admin_denied_access_groups( monkeypatch, @@ -6552,7 +6614,7 @@ async def test_get_and_validate_existing_key(): assert result == mock_key mock_prisma_client.db.litellm_verificationtoken.find_unique.assert_called_once_with( - where={"token": "hashed-test-key-123"} + where={"token": "hashed-test-key-123"}, include={"object_permission": True} ) # Test Case 2: Key not found raises ProxyException diff --git a/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py b/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py index b129ad0f659..5ef83344c1a 100644 --- a/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py +++ b/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py @@ -1213,6 +1213,124 @@ async def test_empty_object_permission_passes_for_personal_non_admin(): ) +# ---- Tests for grandfathering existing key MCP servers on /key/update (LIT-6062) ---- + + +def _make_grandfather_fixtures(mcp_servers=None, mcp_tool_permissions=None): + """Mock prisma client plus the key's existing object permission row.""" + existing_row = MagicMock() + existing_row.mcp_servers = mcp_servers or [] + existing_row.mcp_tool_permissions = mcp_tool_permissions or {} + mock_prisma = MagicMock() + mock_prisma.db.litellm_mcpservertable.find_many = AsyncMock(return_value=[]) + return mock_prisma, existing_row + + +def _patch_grandfather_env(monkeypatch, mock_mgr): + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + mock_mgr, + ) + monkeypatch.setattr( + "litellm.proxy.management_helpers.object_permission_utils._get_allow_all_keys_server_ids", + lambda: set(), + ) + + +@pytest.mark.asyncio +async def test_validate_key_update_grandfathers_existing_servers(monkeypatch): + """A key already holding servers outside the team allowlist can re-send or + shrink those grants on /key/update without a 403 (LIT-6062).""" + _patch_grandfather_env(monkeypatch, _make_mock_mcp_manager("server-a", "server-b")) + team_obj = _make_team_obj(mcp_servers=[]) + mock_prisma, existing_row = _make_grandfather_fixtures(mcp_servers=["server-a", "server-b"]) + resend = await validate_key_mcp_servers_against_team( + object_permission={"mcp_servers": ["server-a", "server-b"]}, + team_obj=team_obj, + prisma_client=mock_prisma, + existing_key_object_permission=existing_row, + ) + assert sorted(resend["mcp_servers"]) == ["server-a", "server-b"] + shrink = await validate_key_mcp_servers_against_team( + object_permission={"mcp_servers": ["server-a"]}, + team_obj=team_obj, + prisma_client=mock_prisma, + existing_key_object_permission=existing_row, + ) + assert shrink["mcp_servers"] == ["server-a"] + + +@pytest.mark.asyncio +async def test_validate_key_update_grandfather_does_not_allow_new_servers(monkeypatch): + """Grandfathering only covers servers the key already holds; adding a new + server outside the team allowlist still raises 403.""" + _patch_grandfather_env(monkeypatch, _make_mock_mcp_manager("server-a", "server-new")) + team_obj = _make_team_obj(mcp_servers=[]) + mock_prisma, existing_row = _make_grandfather_fixtures(mcp_servers=["server-a"]) + with pytest.raises(HTTPException) as exc_info: + await validate_key_mcp_servers_against_team( + object_permission={"mcp_servers": ["server-a", "server-new"]}, + team_obj=team_obj, + prisma_client=mock_prisma, + existing_key_object_permission=existing_row, + ) + assert exc_info.value.status_code == 403 + assert "server-new" in str(exc_info.value.detail) + + +@pytest.mark.asyncio +async def test_validate_key_update_without_existing_permission_still_raises(monkeypatch): + """Without an existing permission row (new grants or team change) the + subset check stays strict.""" + _patch_grandfather_env(monkeypatch, _make_mock_mcp_manager("server-a")) + team_obj = _make_team_obj(mcp_servers=[]) + mock_prisma, _ = _make_grandfather_fixtures(mcp_servers=["server-a"]) + with pytest.raises(HTTPException) as exc_info: + await validate_key_mcp_servers_against_team( + object_permission={"mcp_servers": ["server-a"]}, + team_obj=team_obj, + prisma_client=mock_prisma, + existing_key_object_permission=None, + ) + assert exc_info.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_validate_key_update_grandfathers_tool_permission_keys(monkeypatch): + """Servers granted only via mcp_tool_permissions keys on the existing row + (stored as a JSON string) are grandfathered too.""" + _patch_grandfather_env(monkeypatch, _make_mock_mcp_manager("server-a")) + team_obj = _make_team_obj(mcp_servers=[]) + mock_prisma, existing_row = _make_grandfather_fixtures( + mcp_tool_permissions=json.dumps({"server-a": ["tool1"]}) + ) + result = await validate_key_mcp_servers_against_team( + object_permission={"mcp_servers": ["server-a"]}, + team_obj=team_obj, + prisma_client=mock_prisma, + existing_key_object_permission=existing_row, + ) + assert result["mcp_servers"] == ["server-a"] + + +@pytest.mark.asyncio +async def test_validate_key_update_sentinels_do_not_grandfather(monkeypatch): + """Sentinels stored on the existing row must not grandfather anything.""" + _patch_grandfather_env(monkeypatch, _make_mock_mcp_manager("server-a")) + team_obj = _make_team_obj(mcp_servers=[]) + mock_prisma, existing_row = _make_grandfather_fixtures( + mcp_servers=[SpecialMCPServerName.all_proxy_servers.value, "no-mcp-servers"] + ) + with pytest.raises(HTTPException) as exc_info: + await validate_key_mcp_servers_against_team( + object_permission={"mcp_servers": ["server-a"]}, + team_obj=team_obj, + prisma_client=mock_prisma, + existing_key_object_permission=existing_row, + ) + assert exc_info.value.status_code == 403 + + def test_object_permission_dict_mirrors_pydantic_model(): """ObjectPermissionDict must stay field-for-field aligned with LiteLLM_ObjectPermissionBase. If a new field is added to the Pydantic From fe87b187c6a0e93878910dcb15ba95f3ffb4da7f Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2026 12:52:52 -0700 Subject: [PATCH 25/29] fix: keep schema reconciliation from fighting a partitioned LiteLLM_SpendLogs (#38452) * fix: keep schema reconciliation from fighting a partitioned LiteLLM_SpendLogs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix: scope partitioned SpendLogs detection to Prisma's target schema Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix: default partition detection to Prisma's public schema, not current_schema() Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- db_scripts/partition_spend_logs.sql | 5 + .../litellm_proxy_extras/utils.py | 141 +++++++++++- litellm/proxy/db/prisma_client.py | 17 ++ litellm/proxy/proxy_cli.py | 8 +- .../test_litellm_proxy_extras_utils.py | 208 +++++++++++++++++- .../proxy/db/test_prisma_client.py | 38 ++++ 6 files changed, 411 insertions(+), 6 deletions(-) diff --git a/db_scripts/partition_spend_logs.sql b/db_scripts/partition_spend_logs.sql index 08fcbddb6f8..4e4a93539d7 100644 --- a/db_scripts/partition_spend_logs.sql +++ b/db_scripts/partition_spend_logs.sql @@ -10,6 +10,11 @@ -- partitioned, so existing installs are unaffected until you run this. -- -- IMPORTANT +-- * After partitioning, `prisma db push` (including the proxy's +-- --use_prisma_db_push startup mode) is NOT supported: it tries to rewrite +-- the primary key back to ("request_id"), which Postgres rejects on a +-- partitioned table. The proxy detects this and exits with guidance. +-- Use the default startup path (`prisma migrate deploy`) instead. -- * Test on a staging copy first and take a backup. -- * Postgres cannot convert a populated table to partitioned in place, so this -- renames the old table aside and creates a fresh partitioned table. diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index 5118865e43a..b27221c9beb 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -40,6 +40,65 @@ def _get_prisma_env() -> dict: _MIGRATION_TS_RE = re.compile(r"^(\d{14})_") +_SPEND_LOGS_ALTER_RE = re.compile(r'^ALTER\s+TABLE\s+"LiteLLM_SpendLogs"\s', re.IGNORECASE) +_SPEND_LOGS_ARTIFACT_DROP_RE = re.compile( + r'^DROP\s+TABLE\s+"LiteLLM_SpendLogs_[^"]*"', re.IGNORECASE +) +_SPEND_LOGS_PK_CLAUSE_RE = re.compile( + r'^(?:DROP\s+CONSTRAINT\s+"[^"]*_pkey"' + r'|ADD\s+(?:CONSTRAINT\s+"[^"]*"\s+)?PRIMARY\s+KEY\s*\([^)]*\))$', + re.IGNORECASE, +) + +PARTITIONED_SPEND_LOGS_PUSH_ERROR = ( + "LiteLLM_SpendLogs is a partitioned table (see db_scripts/partition_spend_logs.sql), " + "so its primary key must include the partition key (\"startTime\"). `prisma db push` " + "reconciles the database against schema.prisma, which declares the unpartitioned " + "primary key (\"request_id\"), and Postgres rejects that rewrite with: unique " + "constraint on partitioned table must include all partitioning columns. Start the " + "proxy without --use_prisma_db_push so it uses `prisma migrate deploy`, which only " + "applies shipped migrations and leaves the partitioned primary key alone." +) + + +def _without_sql_comments(statement: str) -> str: + return "\n".join( + line + for line in statement.splitlines() + if line.strip() and not line.strip().startswith("--") + ).strip() + + +def _without_spend_logs_pk_clauses(statement: str) -> Optional[str]: + prefix_match = _SPEND_LOGS_ALTER_RE.match(statement) + if not prefix_match: + return statement + kept = tuple( + clause.strip() + for clause in statement[prefix_match.end():].split(",\n") + if not _SPEND_LOGS_PK_CLAUSE_RE.match(clause.strip()) + ) + if not kept: + return None + return statement[: prefix_match.end()] + ",\n".join(kept) + + +def filter_partitioned_spend_logs_diff(diff_sql: str) -> str: + """Drop statements from a `prisma migrate diff` script that fight the + SpendLogs partitioning runbook (db_scripts/partition_spend_logs.sql): the + primary-key rewrite on "LiteLLM_SpendLogs", which Postgres rejects on a + partitioned table, and drops of runbook artifacts such as + "LiteLLM_SpendLogs_legacy".""" + kept = tuple( + filtered + for statement in diff_sql.split(";") + for bare in (_without_sql_comments(statement),) + if bare and not _SPEND_LOGS_ARTIFACT_DROP_RE.match(bare) + for filtered in (_without_spend_logs_pk_clauses(bare),) + if filtered is not None + ) + return "".join(f"{statement};\n\n" for statement in kept) + def _migration_timestamp(name: str) -> int: """Extract the leading `YYYYMMDDHHMMSS` timestamp from a migration name. @@ -355,7 +414,24 @@ class ProxyExtrasDBManager: return logger.info(f"Migration diff created at {diff_sql_path}") + if ProxyExtrasDBManager.spend_logs_is_partitioned(): + filtered_sql = filter_partitioned_spend_logs_diff( + diff_sql_path.read_text() + ) + diff_sql_path.write_text(filtered_sql) + logger.info( + "LiteLLM_SpendLogs is partitioned; removed its primary-key " + "rewrite and partitioning artifacts from the drift script" + ) + if not filtered_sql.strip(): + logger.info("Drift script is empty after filtering; nothing to apply") + if not mark_all_applied: + return + ProxyExtrasDBManager._mark_migrations_applied(migrations_dir) + return + # 2. Run prisma db execute to apply the migration + applied_ok = False try: logger.info("Running prisma db execute to apply the migration diff...") result = subprocess.run( @@ -376,6 +452,7 @@ class ProxyExtrasDBManager: ) logger.info(f"prisma db execute stdout: {result.stdout}") logger.info("✅ Migration diff applied successfully") + applied_ok = True except subprocess.CalledProcessError as e: logger.warning(f"Failed to apply migration diff: {e.stderr}") except subprocess.TimeoutExpired: @@ -384,6 +461,16 @@ class ProxyExtrasDBManager: # 3. Mark all migrations as applied if not mark_all_applied: return + if not applied_ok: + logger.warning( + "Drift script failed to apply; NOT marking migrations as " + "applied so a later migration run can retry them" + ) + return + ProxyExtrasDBManager._mark_migrations_applied(migrations_dir) + + @staticmethod + def _mark_migrations_applied(migrations_dir: str): migration_names = ProxyExtrasDBManager._get_migration_names(migrations_dir) logger.info(f"Resolving {len(migration_names)} migrations") for migration_name in migration_names: @@ -410,6 +497,55 @@ class ProxyExtrasDBManager: f"Failed to resolve migration {migration_name}: {e.stderr}" ) + @staticmethod + def spend_logs_is_partitioned() -> bool: + """True when the connected database's LiteLLM_SpendLogs is a + partitioned table in Prisma's target schema (the `schema` URL param, + falling back to Prisma's default target, public), i.e. the operator + ran db_scripts/partition_spend_logs.sql. Returns False when psycopg is + unavailable or the database cannot be reached, preserving the + pre-existing behavior in those cases.""" + database_url = os.getenv("DATABASE_URL") + if not database_url: + return False + + try: + import psycopg + except ImportError: + return False + + cleaned_url = ProxyExtrasDBManager._strip_prisma_query_params(database_url) + try: + with psycopg.connect( + cleaned_url, connect_timeout=10, autocommit=True + ) as conn: + row = conn.execute( + "SELECT 1 " + "FROM pg_partitioned_table pt " + "JOIN pg_class c ON c.oid = pt.partrelid " + "JOIN pg_namespace n ON n.oid = c.relnamespace " + "WHERE c.relname = 'LiteLLM_SpendLogs' " + " AND n.nspname = %s", + ( + ProxyExtrasDBManager._prisma_schema_param(database_url) + or "public", + ), + ).fetchone() + except (psycopg.OperationalError, psycopg.DatabaseError): + return False + return row is not None + + @staticmethod + def _prisma_schema_param(url: str) -> Optional[str]: + """The `schema` query param Prisma uses to pick its target schema, + or None when the URL does not set one.""" + from urllib.parse import urlparse, parse_qsl + + return next( + (v for k, v in parse_qsl(urlparse(url).query) if k == "schema"), + None, + ) + @staticmethod def _strip_prisma_query_params(url: str) -> str: """Remove Prisma-specific query params (connection_limit, pool_timeout, @@ -528,7 +664,8 @@ class ProxyExtrasDBManager: migrations_dir = ProxyExtrasDBManager._get_prisma_dir() if not use_migrate: - # Preserve `prisma db push` path unchanged. + if ProxyExtrasDBManager.spend_logs_is_partitioned(): + raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR) original_dir = os.getcwd() os.chdir(migrations_dir) try: @@ -972,6 +1109,8 @@ class ProxyExtrasDBManager: ) raise else: + if ProxyExtrasDBManager.spend_logs_is_partitioned(): + raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR) # Use prisma db push with increased timeout subprocess.run( [_get_prisma_command(), "db", "push", "--accept-data-loss"], diff --git a/litellm/proxy/db/prisma_client.py b/litellm/proxy/db/prisma_client.py index fc761fc1831..4bd007769b8 100644 --- a/litellm/proxy/db/prisma_client.py +++ b/litellm/proxy/db/prisma_client.py @@ -887,6 +887,22 @@ class PrismaManager: return ProxyExtrasDBManager.apply_replica_identity_full_if_requested() + @staticmethod + def _raise_if_partitioned_spend_logs() -> None: + """`prisma db push` rewrites a doc-partitioned LiteLLM_SpendLogs + primary key back to ("request_id"), which Postgres rejects. Fail fast + with guidance instead of retrying into that raw error. No-op when + litellm-proxy-extras is absent.""" + try: + from litellm_proxy_extras.utils import ( + PARTITIONED_SPEND_LOGS_PUSH_ERROR, + ProxyExtrasDBManager, + ) + except ImportError: + return + if ProxyExtrasDBManager.spend_logs_is_partitioned(): + raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR) + @staticmethod def setup_database(use_migrate: bool = False, use_v2_resolver: bool = False) -> bool: """ @@ -921,6 +937,7 @@ class PrismaManager: use_v2_resolver=use_v2_resolver, ) else: + PrismaManager._raise_if_partitioned_spend_logs() # Use prisma db push with increased timeout subprocess.run( [ diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 0449802abae..8ac63ba25c9 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -1321,10 +1321,10 @@ def run_server( use_v2_resolver=use_v2_migration_resolver, ) except RuntimeError as e: - # v2 resolver raises on unrecoverable migration errors - # (e.g. non-idempotent failures, permission issues). - # v1 never raises here, so this only fires when the - # operator opted into v2. + # Raised on unrecoverable migration errors: the v2 + # resolver's non-idempotent failures and permission + # issues, and any `prisma db push` against a + # partitioned LiteLLM_SpendLogs. print( f"\033[1;31mLiteLLM Proxy: Database migration cannot proceed. {e}\033[0m", file=sys.stderr, diff --git a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py index 09f3e0ba34f..498d0cb4723 100644 --- a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py +++ b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py @@ -12,7 +12,11 @@ sys.path.insert( ), ) -from litellm_proxy_extras.utils import ProxyExtrasDBManager +from litellm_proxy_extras.utils import ( + PARTITIONED_SPEND_LOGS_PUSH_ERROR, + ProxyExtrasDBManager, + filter_partitioned_spend_logs_diff, +) # Path to the migrations directory _MIGRATIONS_DIR = os.path.abspath( @@ -475,3 +479,205 @@ class TestMigrationGuardScope: if not self._run_rules([(TestMigrationGuardScope._NEW, by_name[name])]) ] assert not redundant, f"these no longer violate and should be removed: {redundant}" + + +_PARTITIONED_DRIFT_SQL = """-- AlterTable +ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN "updated_by" TEXT; + +-- AlterTable +ALTER TABLE "LiteLLM_SpendLogs" DROP CONSTRAINT "LiteLLM_SpendLogs_pkey", +ADD COLUMN "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, +ADD COLUMN "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, +ADD CONSTRAINT "LiteLLM_SpendLogs_pkey" PRIMARY KEY ("request_id"); + +-- DropTable +DROP TABLE "LiteLLM_SpendLogs_legacy"; +""" + + +class TestPartitionedSpendLogsDriftFilter: + """A doc-partitioned LiteLLM_SpendLogs (db_scripts/partition_spend_logs.sql) has a + composite primary key that schema.prisma cannot express, so `prisma migrate diff` + emits a primary-key rewrite that Postgres rejects, aborting the whole drift script + before its legitimate statements run.""" + + def test_pk_rewrite_and_runbook_artifact_drops_are_removed(self): + filtered = filter_partitioned_spend_logs_diff(_PARTITIONED_DRIFT_SQL) + assert 'DROP CONSTRAINT "LiteLLM_SpendLogs_pkey"' not in filtered + assert 'PRIMARY KEY ("request_id")' not in filtered + assert "LiteLLM_SpendLogs_legacy" not in filtered + + def test_legitimate_statements_in_the_same_script_are_kept(self): + filtered = filter_partitioned_spend_logs_diff(_PARTITIONED_DRIFT_SQL) + assert 'ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN "updated_by" TEXT;' in filtered + assert 'ADD COLUMN "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP' in filtered + assert 'ADD COLUMN "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP' in filtered + assert filtered.count('ALTER TABLE "LiteLLM_SpendLogs"') == 1 + + def test_an_alter_containing_only_the_pk_rewrite_is_dropped_entirely(self): + sql = ( + 'ALTER TABLE "LiteLLM_SpendLogs" DROP CONSTRAINT "LiteLLM_SpendLogs_pkey",\n' + 'ADD CONSTRAINT "LiteLLM_SpendLogs_pkey" PRIMARY KEY ("request_id");\n' + ) + assert filter_partitioned_spend_logs_diff(sql).strip() == "" + + def test_other_tables_pk_changes_are_untouched(self): + sql = ( + 'ALTER TABLE "LiteLLM_TeamTable" DROP CONSTRAINT "LiteLLM_TeamTable_pkey",\n' + 'ADD CONSTRAINT "LiteLLM_TeamTable_pkey" PRIMARY KEY ("team_id");\n' + ) + filtered = filter_partitioned_spend_logs_diff(sql) + assert 'DROP CONSTRAINT "LiteLLM_TeamTable_pkey"' in filtered + assert 'PRIMARY KEY ("team_id")' in filtered + + +class _FakeCompleted: + stdout = "" + stderr = "" + + +class TestResolveAllMigrationsLedger: + def _run(self, monkeypatch, tmp_path, partitioned, execute_fails): + import subprocess as subprocess_module + + import litellm_proxy_extras.utils as utils_module + + monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:5432/db") + monkeypatch.delenv("DIRECT_URL", raising=False) + monkeypatch.setattr( + ProxyExtrasDBManager, "spend_logs_is_partitioned", staticmethod(lambda: partitioned) + ) + monkeypatch.setattr( + ProxyExtrasDBManager, + "_get_migration_names", + staticmethod(lambda migrations_dir: ["20250326162113_baseline"]), + ) + calls = [] + + def fake_run(cmd, **kwargs): + calls.append(cmd) + if "diff" in cmd: + kwargs["stdout"].write(_PARTITIONED_DRIFT_SQL) + return _FakeCompleted() + if "execute" in cmd: + executed_sql = open(cmd[cmd.index("--file") + 1]).read() + calls.append(("executed_sql", executed_sql)) + if execute_fails: + raise subprocess_module.CalledProcessError(1, cmd, stderr="boom") + return _FakeCompleted() + return _FakeCompleted() + + monkeypatch.setattr(utils_module.subprocess, "run", fake_run) + ProxyExtrasDBManager._resolve_all_migrations(str(tmp_path), "schema.prisma") + return calls + + def _resolved(self, calls): + return [c for c in calls if isinstance(c, list) and "resolve" in c] + + def _executed_sql(self, calls): + return next(c[1] for c in calls if isinstance(c, tuple) and c[0] == "executed_sql") + + def test_failed_drift_apply_does_not_mark_migrations_applied(self, monkeypatch, tmp_path): + calls = self._run(monkeypatch, tmp_path, partitioned=False, execute_fails=True) + assert self._resolved(calls) == [] + + def test_successful_drift_apply_still_marks_migrations_applied(self, monkeypatch, tmp_path): + calls = self._run(monkeypatch, tmp_path, partitioned=False, execute_fails=False) + assert len(self._resolved(calls)) == 1 + + def test_partitioned_spend_logs_gets_the_filtered_drift_script(self, monkeypatch, tmp_path): + calls = self._run(monkeypatch, tmp_path, partitioned=True, execute_fails=False) + executed_sql = self._executed_sql(calls) + assert 'PRIMARY KEY ("request_id")' not in executed_sql + assert "LiteLLM_SpendLogs_legacy" not in executed_sql + assert 'ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN "updated_by" TEXT;' in executed_sql + assert 'ADD COLUMN "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP' in executed_sql + assert len(self._resolved(calls)) == 1 + + def test_unpartitioned_spend_logs_drift_script_is_untouched(self, monkeypatch, tmp_path): + calls = self._run(monkeypatch, tmp_path, partitioned=False, execute_fails=False) + assert self._executed_sql(calls) == _PARTITIONED_DRIFT_SQL + + +class TestPartitionedSpendLogsPushGuard: + def _forbid_subprocess(self, monkeypatch): + import litellm_proxy_extras.utils as utils_module + + def fail_run(cmd, **kwargs): + raise AssertionError(f"subprocess.run should not be called, got: {cmd}") + + monkeypatch.setattr(utils_module.subprocess, "run", fail_run) + + def test_v1_db_push_fails_fast_with_guidance(self, monkeypatch): + monkeypatch.setattr( + ProxyExtrasDBManager, "spend_logs_is_partitioned", staticmethod(lambda: True) + ) + self._forbid_subprocess(monkeypatch) + with pytest.raises(RuntimeError) as err: + ProxyExtrasDBManager._run_migrations(use_migrate=False, use_v2_resolver=False) + assert str(err.value) == PARTITIONED_SPEND_LOGS_PUSH_ERROR + + def test_v2_db_push_fails_fast_with_guidance(self, monkeypatch): + monkeypatch.setattr( + ProxyExtrasDBManager, "spend_logs_is_partitioned", staticmethod(lambda: True) + ) + self._forbid_subprocess(monkeypatch) + with pytest.raises(RuntimeError) as err: + ProxyExtrasDBManager._setup_database_v2(use_migrate=False) + assert str(err.value) == PARTITIONED_SPEND_LOGS_PUSH_ERROR + + +class _FakeCursor: + def fetchone(self): + return (1,) + + +class _FakePsycopgConn: + def __init__(self, executed): + self._executed = executed + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def execute(self, query, params): + self._executed.append((query, params)) + return _FakeCursor() + + +class TestSpendLogsPartitionDetectionSchemaScope: + """A same-named LiteLLM_SpendLogs in another schema must not trip the + detector: the catalog lookup has to be scoped to Prisma's target schema.""" + + def _detect(self, monkeypatch, database_url): + import sys + import types + + executed = [] + fake_psycopg = types.ModuleType("psycopg") + fake_psycopg.connect = lambda url, **kwargs: _FakePsycopgConn(executed) + fake_psycopg.OperationalError = type("OperationalError", (Exception,), {}) + fake_psycopg.DatabaseError = type("DatabaseError", (Exception,), {}) + monkeypatch.setitem(sys.modules, "psycopg", fake_psycopg) + monkeypatch.setenv("DATABASE_URL", database_url) + assert ProxyExtrasDBManager.spend_logs_is_partitioned() is True + return executed[0] + + def test_lookup_is_scoped_to_the_schema_url_param(self, monkeypatch): + query, params = self._detect( + monkeypatch, "postgresql://u:p@localhost:5432/db?schema=tenant_a" + ) + assert "pg_namespace" in query + assert "n.nspname = %s" in query + assert params == ("tenant_a",) + + def test_lookup_falls_back_to_public_without_a_schema_param(self, monkeypatch): + query, params = self._detect(monkeypatch, "postgresql://u:p@localhost:5432/db") + assert "n.nspname = %s" in query + assert params == ("public",) + + def test_only_partitioned_relations_match(self, monkeypatch): + query, _ = self._detect(monkeypatch, "postgresql://u:p@localhost:5432/db") + assert "pg_partitioned_table" in query diff --git a/tests/test_litellm/proxy/db/test_prisma_client.py b/tests/test_litellm/proxy/db/test_prisma_client.py index b1ecbfeff8e..f0983d6bf62 100644 --- a/tests/test_litellm/proxy/db/test_prisma_client.py +++ b/tests/test_litellm/proxy/db/test_prisma_client.py @@ -215,6 +215,44 @@ def test_db_push_applies_replica_identity_full_when_requested(monkeypatch): assert applied == [True] +def test_db_push_is_rejected_when_spend_logs_is_partitioned(monkeypatch): + """A doc-partitioned LiteLLM_SpendLogs makes `prisma db push` rewrite the + primary key back to ("request_id"), which Postgres rejects; the guard must + fail fast with guidance instead of running the push.""" + from litellm.proxy.db.prisma_client import PrismaManager + from litellm_proxy_extras.utils import ( + PARTITIONED_SPEND_LOGS_PUSH_ERROR, + ProxyExtrasDBManager, + ) + + monkeypatch.setattr( + ProxyExtrasDBManager, "spend_logs_is_partitioned", staticmethod(lambda: True) + ) + with patch( # test-quality-ok: subprocess.run is the external prisma CLI boundary, asserted never reached + "litellm.proxy.db.prisma_client.subprocess.run" + ) as mock_run: + with pytest.raises(RuntimeError) as err: + PrismaManager.setup_database(use_migrate=False) + + assert str(err.value) == PARTITIONED_SPEND_LOGS_PUSH_ERROR + mock_run.assert_not_called() + + +def test_db_push_proceeds_when_spend_logs_is_not_partitioned(monkeypatch): + from litellm.proxy.db.prisma_client import PrismaManager + from litellm_proxy_extras.utils import ProxyExtrasDBManager + + monkeypatch.setattr( + ProxyExtrasDBManager, "spend_logs_is_partitioned", staticmethod(lambda: False) + ) + with patch( # test-quality-ok: subprocess.run is the external prisma CLI boundary, not SDK logic + "litellm.proxy.db.prisma_client.subprocess.run" + ) as mock_run: + assert PrismaManager.setup_database(use_migrate=False) is True + + assert mock_run.call_args[0][0][:3] == ["prisma", "db", "push"] + + def _entra_jwt(expires_in_seconds: int) -> str: """A JWT shaped like a real Entra access token, expiring ``expires_in_seconds`` from now.""" import base64 From 71449b9c550c488f24f748d6f66428b2091b6b48 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Thu, 27 Aug 2026 13:08:13 -0700 Subject: [PATCH 26/29] fix(ui): open select popups below the trigger instead of over it (#38554) The shared SelectContent wrapper defaulted alignItemWithTrigger to true, which puts Base UI's positioner into item-aligned mode and places the popup so the active item sits on top of the trigger. In that mode the side and sideOffset the wrapper passes two lines above are ignored, and the popup reports data-side="none". The overlap only becomes visible once the items are tall enough to matter, which is why the autorouter Template picker shows it clearly: its options are three-line cards, so the popup covers both the select box and its own label. No call site in the dashboard asked for item-aligned mode. 21 of them across 15 files already passed alignItemWithTrigger={false} by hand to undo the default, and the remaining 127 inherited the bug. Flipping the default makes side and sideOffset live, so collision handling works and a select with no room below now flips above the trigger rather than covering it. The 21 hand-written opt-outs are deleted as redundant. --- .../autoRouterTemplateSelect.spec.ts | 59 +++++++++++++++++++ .../_components/TeamGuardrailsTab.tsx | 2 +- .../content_filter/CategoryTable.tsx | 4 +- .../CompetitorIntentConfiguration.tsx | 6 +- .../ContentCategoryConfiguration.tsx | 4 +- .../content_filter/CustomPatternModal.tsx | 2 +- .../content_filter/KeywordModal.tsx | 2 +- .../content_filter/KeywordTable.tsx | 2 +- .../content_filter/PatternModal.tsx | 2 +- .../content_filter/PatternTable.tsx | 2 +- .../custom_code/CustomCodeModal.tsx | 2 +- .../guardrails/_components/pii_components.tsx | 2 +- .../ToolPermissionRulesEditor.tsx | 6 +- .../_components/CreateVectorStore.tsx | 2 +- .../_components/VectorStoreForm.tsx | 2 +- .../_components/vector_store_info.tsx | 2 +- .../src/components/ui/select.test.tsx | 38 ++++++++++++ .../src/components/ui/select.tsx | 2 +- 18 files changed, 119 insertions(+), 22 deletions(-) create mode 100644 tests/e2e/ui/tests/modelsPage/autoRouterTemplateSelect.spec.ts diff --git a/tests/e2e/ui/tests/modelsPage/autoRouterTemplateSelect.spec.ts b/tests/e2e/ui/tests/modelsPage/autoRouterTemplateSelect.spec.ts new file mode 100644 index 00000000000..98bd1b84f11 --- /dev/null +++ b/tests/e2e/ui/tests/modelsPage/autoRouterTemplateSelect.spec.ts @@ -0,0 +1,59 @@ +import { expect, test, type Page as PlaywrightPage } from "@playwright/test"; +import { ADMIN_STORAGE_PATH } from "../../constants"; +import { navigateToPage } from "../../helpers/navigation"; +import { Page } from "../../fixtures/pages"; + +/** + * Opens Add Auto Router and returns the Template select's trigger, which is the + * shallowest real page that renders SelectContent with tall multi-line options. + */ +async function openTemplateSelect(page: PlaywrightPage) { + await navigateToPage(page, Page.Models); + await page.getByRole("tab", { name: "Auto-Routers" }).click(); + await page.getByRole("button", { name: "Add Auto Router" }).click(); + + const trigger = page.getByTestId("template-selector"); + await expect(trigger).toBeVisible(); + return trigger; +} + +test.describe("Auto Router template select anchoring", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test("opens the options below the trigger rather than over it", async ({ page }) => { + await page.setViewportSize({ width: 1280, height: 900 }); + const trigger = await openTemplateSelect(page); + const triggerBox = await trigger.boundingBox(); + + await trigger.click(); + const popup = page.locator('[data-slot="select-content"]'); + await expect(popup).toBeVisible(); + const popupBox = await popup.boundingBox(); + + expect(triggerBox).not.toBeNull(); + expect(popupBox).not.toBeNull(); + + // Item-aligned mode reports "none" and puts the active item over the trigger. + await expect(popup).toHaveAttribute("data-side", "bottom"); + expect(popupBox!.y).toBeGreaterThanOrEqual(triggerBox!.y + triggerBox!.height); + }); + + test("flips above the trigger instead of covering it when there is no room below", async ({ page }) => { + await page.setViewportSize({ width: 1280, height: 560 }); + const trigger = await openTemplateSelect(page); + await trigger.scrollIntoViewIfNeeded(); + const triggerBox = await trigger.boundingBox(); + + await trigger.click(); + const popup = page.locator('[data-slot="select-content"]'); + await expect(popup).toBeVisible(); + const popupBox = await popup.boundingBox(); + + expect(triggerBox).not.toBeNull(); + expect(popupBox).not.toBeNull(); + + const overlaps = + popupBox!.y < triggerBox!.y + triggerBox!.height && popupBox!.y + popupBox!.height > triggerBox!.y; + expect(overlaps).toBe(false); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx index dc1223264bb..1de4e697f64 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx @@ -1106,7 +1106,7 @@ export function TeamGuardrailsTab({ accessToken }: TeamGuardrailsTabProps) { > - + {GUARDRAIL_MODES.map((mode) => ( {mode.label} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/CategoryTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/CategoryTable.tsx index fa40ce6ab54..f012923d32f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/CategoryTable.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/CategoryTable.tsx @@ -64,7 +64,7 @@ const CategoryTable: React.FC = ({ - + {SEVERITY_ITEMS.map((item) => ( {item.label} @@ -93,7 +93,7 @@ const CategoryTable: React.FC = ({ - + {ACTION_ITEMS.map((item) => ( {item.label} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/CompetitorIntentConfiguration.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/CompetitorIntentConfiguration.tsx index 8445495246d..0632ec87f34 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/CompetitorIntentConfiguration.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/CompetitorIntentConfiguration.tsx @@ -194,7 +194,7 @@ const CompetitorIntentConfiguration: React.FC - + {INTENT_TYPES.map((type) => ( {type.label} @@ -268,7 +268,7 @@ const CompetitorIntentConfiguration: React.FC - + {COMPETITOR_COMPARISON_POLICIES.map((policy) => ( {policy.label} @@ -292,7 +292,7 @@ const CompetitorIntentConfiguration: React.FC - + {POSSIBLE_COMPETITOR_COMPARISON_POLICIES.map((policy) => ( {policy.label} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/ContentCategoryConfiguration.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/ContentCategoryConfiguration.tsx index 1924133a6cc..f2226a3bc6e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/ContentCategoryConfiguration.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/ContentCategoryConfiguration.tsx @@ -199,7 +199,7 @@ const ContentCategoryConfiguration: React.FC - + {ACTION_ITEMS.map((item) => ( {item.value} @@ -224,7 +224,7 @@ const ContentCategoryConfiguration: React.FC - + {SEVERITY_ITEMS.map((item) => ( {item.label} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/CustomPatternModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/CustomPatternModal.tsx index 2ead171d5e4..68eb7e138ab 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/CustomPatternModal.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/CustomPatternModal.tsx @@ -70,7 +70,7 @@ const CustomPatternModal: React.FC = ({ - + {ACTION_ITEMS.map((item) => ( {item.label} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/KeywordModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/KeywordModal.tsx index 504f35973fd..bf1b49dabd0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/KeywordModal.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/KeywordModal.tsx @@ -60,7 +60,7 @@ const KeywordModal: React.FC = ({ - + {ACTION_ITEMS.map((item) => ( {item.label} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/KeywordTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/KeywordTable.tsx index 5c7e3ef3ab8..5b69b04955f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/KeywordTable.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/KeywordTable.tsx @@ -38,7 +38,7 @@ const KeywordTable: React.FC = ({ keywords, onActionChange, o - + {ACTION_ITEMS.map((item) => ( {item.label} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/PatternModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/PatternModal.tsx index 4caa47217fe..aeeadedfbf1 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/PatternModal.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/PatternModal.tsx @@ -114,7 +114,7 @@ const PatternModal: React.FC = ({ - + {ACTION_ITEMS.map((item) => ( {item.label} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/PatternTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/PatternTable.tsx index 6dd266f07a0..f4e87119d7b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/PatternTable.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/content_filter/PatternTable.tsx @@ -58,7 +58,7 @@ const PatternTable: React.FC = ({ patterns, onActionChange, o - + {ACTION_ITEMS.map((item) => ( {item.label} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/custom_code/CustomCodeModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/custom_code/CustomCodeModal.tsx index 77bac8bb0aa..a69824f32d3 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/custom_code/CustomCodeModal.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/custom_code/CustomCodeModal.tsx @@ -556,7 +556,7 @@ const CustomCodeModal: React.FC = ({ visible, onClose, onS - + STANDARD {TEMPLATE_ITEMS.map((template) => ( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/pii_components.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/pii_components.tsx index 5f8e833af8d..0de7eb1c9ce 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/pii_components.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/pii_components.tsx @@ -179,7 +179,7 @@ export const PiiEntityList: React.FC = ({ - + {actions.map((action) => ( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/tool_permission/ToolPermissionRulesEditor.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/tool_permission/ToolPermissionRulesEditor.tsx index 8fe2bf5bf21..c154d102314 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/tool_permission/ToolPermissionRulesEditor.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/tool_permission/ToolPermissionRulesEditor.tsx @@ -280,7 +280,7 @@ const ToolPermissionRulesEditor: React.FC = ({ v - + {DECISION_ITEMS.map((item) => ( {item.label} @@ -313,7 +313,7 @@ const ToolPermissionRulesEditor: React.FC = ({ v - + {DECISION_ITEMS.map((item) => ( {item.label} @@ -350,7 +350,7 @@ const ToolPermissionRulesEditor: React.FC = ({ v - + {ON_DISALLOWED_ITEMS.map((item) => ( {item.label} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/CreateVectorStore.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/CreateVectorStore.tsx index 9d447090381..8f24e47340c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/CreateVectorStore.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/CreateVectorStore.tsx @@ -342,7 +342,7 @@ const CreateVectorStore: React.FC = ({ accessToken, onSu - + {providerItems.map((item) => ( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx index be0954a7d24..9d78b727768 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx @@ -298,7 +298,7 @@ const VectorStoreForm: React.FC = ({ }} - + {Object.entries(VectorStoreProviders).map(([providerEnum, providerDisplayName]) => ( = ({ }} - + {Object.entries(Providers) .filter(([providerEnum]) => providerEnum === "Bedrock") .map(([providerEnum, providerDisplayName]) => ( diff --git a/ui/litellm-dashboard/src/components/ui/select.test.tsx b/ui/litellm-dashboard/src/components/ui/select.test.tsx index 15dd266d38c..308b5ecec3b 100644 --- a/ui/litellm-dashboard/src/components/ui/select.test.tsx +++ b/ui/litellm-dashboard/src/components/ui/select.test.tsx @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; const ENVIRONMENTS = [ @@ -63,3 +64,40 @@ describe("SelectValue label resolution", () => { expect(screen.getByTestId("trigger")).toHaveTextContent("Any environment"); }); }); + +function renderOpenableSelect(contentProps?: React.ComponentProps) { + return render( + , + ); +} + +describe("SelectContent anchoring", () => { + it("anchors to the edge of the trigger rather than over it by default", async () => { + const user = userEvent.setup(); + renderOpenableSelect(); + + await user.click(screen.getByTestId("trigger")); + + expect(await screen.findByTestId("content")).toHaveAttribute("data-align-trigger", "false"); + }); + + it("still lets a caller opt into item-aligned anchoring", async () => { + const user = userEvent.setup(); + renderOpenableSelect({ alignItemWithTrigger: true }); + + await user.click(screen.getByTestId("trigger")); + + expect(await screen.findByTestId("content")).toHaveAttribute("data-align-trigger", "true"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/ui/select.tsx b/ui/litellm-dashboard/src/components/ui/select.tsx index 3c3f8c81c66..814695d3418 100644 --- a/ui/litellm-dashboard/src/components/ui/select.tsx +++ b/ui/litellm-dashboard/src/components/ui/select.tsx @@ -49,7 +49,7 @@ function SelectContent({ sideOffset = 4, align = "center", alignOffset = 0, - alignItemWithTrigger = true, + alignItemWithTrigger = false, ...props }: SelectPrimitive.Popup.Props & Pick) { From 17136e5b0b0fe03f25a283287e42165f90e60c2c Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 27 Aug 2026 13:19:51 -0700 Subject: [PATCH 27/29] bump: litellm-enterprise 0.1.60 -> 0.1.61, litellm-proxy-extras 0.4.89 -> 0.4.90 --- enterprise/pyproject.toml | 4 ++-- litellm-proxy-extras/pyproject.toml | 4 ++-- pyproject.toml | 4 ++-- uv.lock | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml index b7a62e52cf9..3653aba67ef 100644 --- a/enterprise/pyproject.toml +++ b/enterprise/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-enterprise" -version = "0.1.60" +version = "0.1.61" description = "Package for LiteLLM Enterprise features" readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.1.60" +version = "0.1.61" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-enterprise==", diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index 98a3d8d535e..0ef5cd1e856 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.89" +version = "0.4.90" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.89" +version = "0.4.90" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-proxy-extras==", diff --git a/pyproject.toml b/pyproject.toml index 1c3f5a4875c..eba9e5afc98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,8 +67,8 @@ proxy = [ "azure-identity>=1.25.2,<2.0", "azure-storage-blob>=12.28.0,<13.0", "mcp>=1.28.1,<2.0", - "litellm-proxy-extras==0.4.89", - "litellm-enterprise==0.1.60", + "litellm-proxy-extras==0.4.90", + "litellm-enterprise==0.1.61", "RestrictedPython>=8.1,<9.0", "rich>=13.9.4,<14.0", "InquirerPy>=0.3.4,<1.0", diff --git a/uv.lock b/uv.lock index ca5c4eb8c3c..f42c67079e6 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-08-23T20:15:58.934396Z" +exclude-newer = "2026-08-24T20:19:42.376246Z" exclude-newer-span = "P3D" [manifest] @@ -4665,12 +4665,12 @@ proxy-dev = [ [[package]] name = "litellm-enterprise" -version = "0.1.60" +version = "0.1.61" source = { editable = "enterprise" } [[package]] name = "litellm-proxy-extras" -version = "0.4.89" +version = "0.4.90" source = { editable = "litellm-proxy-extras" } [[package]] From 3746ba58d7b8406de1c22d0977fdce8f87c641cd Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 27 Aug 2026 13:32:20 -0700 Subject: [PATCH 28/29] fix(ui): let the paginated search select keep what the user types (#38475) * fix(ui): let the paginated search select keep what the user types The combobox handed Base UI a freshly built option object for the current selection every time a page of results came back. Base UI answers a changed value by rewriting the input with that option's label, so every search response wiped the query mid-typing and the list never narrowed. Once a user had been picked in the Usage page filter box, no other user could be reached. The component now owns the input text. It holds the query while the list is open, falls back to the selected option's label once the list closes, and remembers the picked option so its label survives later pages that no longer carry it, the way the multi-select sibling already does. * refactor(ui): name the paginated select's search state instead of commenting it * fix(ui): start a fresh query when typing lands on the selected label Focusing the filter box without clicking it leaves the caret at the end of the selected option's label, so the next keystroke extended that label into a query no server could match. Only a click cleared the box first. A keystroke that arrives while the box is showing a label is now read as the start of a new query, wherever in the label it landed. --- .../shared/PaginatedSearchSelect.test.tsx | 148 ++++++++++++++++++ .../shared/PaginatedSearchSelect.tsx | 40 ++++- .../components/shared/usePaginatedCombobox.ts | 21 ++- 3 files changed, 200 insertions(+), 9 deletions(-) diff --git a/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.test.tsx b/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.test.tsx index d36b414b726..310b5363b0b 100644 --- a/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.test.tsx @@ -142,6 +142,154 @@ describe("PaginatedSearchSelect", () => { expect(onValueChange).toHaveBeenCalledWith("alias-beta"); }); + it("keeps the typed query when a refreshed page of options arrives while a value is selected", async () => { + const user = userEvent.setup(); + const onSearchChange = vi.fn(); + + function ServerBacked() { + const [search, setSearch] = useState(""); + const [value, setValue] = useState("alias-alpha"); + const freshlyBuiltOptions = OPTIONS.filter((option) => option.label.includes(search)).map((option) => ({ + ...option, + })); + return ( + { + onSearchChange(query); + setSearch(query); + }} + onLoadMore={vi.fn()} + /> + ); + } + render(); + + const input = screen.getByRole("combobox"); + await user.click(input); + await user.type(input, "gamma"); + + await waitFor(() => expect(onSearchChange).toHaveBeenLastCalledWith("gamma")); + await waitFor(() => expect(input).toHaveValue("gamma")); + expect(await screen.findByText("gamma-key")).toBeInTheDocument(); + }); + + it("shows the selection again after the popup closes with the query abandoned", async () => { + const user = userEvent.setup(); + renderSelect({ value: "alias-alpha" }); + + const input = screen.getByRole("combobox"); + await user.click(input); + expect(input).toHaveValue(""); + + await user.type(input, "gamma"); + await user.keyboard("{Escape}"); + + await waitFor(() => expect(input).toHaveValue("alias-alpha")); + }); + + it("puts the unfiltered page back when a typed query is abandoned", async () => { + const user = userEvent.setup(); + const onSearchChange = vi.fn(); + renderSelect({ onSearchChange }); + + const input = screen.getByRole("combobox"); + await user.click(input); + await user.type(input, "gamma"); + await waitFor(() => expect(onSearchChange).toHaveBeenCalledWith("gamma")); + + await user.keyboard("{Escape}"); + + await waitFor(() => expect(onSearchChange).toHaveBeenLastCalledWith("")); + }); + + it("puts the unfiltered page back once an option found by typing is picked", async () => { + const user = userEvent.setup(); + const onSearchChange = vi.fn(); + renderSelect({ onSearchChange }); + + const input = screen.getByRole("combobox"); + await user.click(input); + await user.type(input, "gamma"); + await waitFor(() => expect(onSearchChange).toHaveBeenCalledWith("gamma")); + + await user.click(await screen.findByText("gamma-key")); + + await waitFor(() => expect(onSearchChange).toHaveBeenLastCalledWith("")); + }); + + it("keeps the first character when typing is what opened the list", async () => { + const user = userEvent.setup(); + const onSearchChange = vi.fn(); + renderSelect({ onSearchChange }); + + await user.tab(); + await user.keyboard("gamma"); + + expect(screen.getByRole("combobox")).toHaveValue("gamma"); + await waitFor(() => expect(onSearchChange).toHaveBeenLastCalledWith("gamma")); + }); + + it("keeps showing a picked option's label after it drops out of the loaded page", async () => { + const user = userEvent.setup(); + + function Refetching() { + const [options, setOptions] = useState([{ label: "Beta Team", value: "team-2" }]); + const [value, setValue] = useState(""); + return ( + <> + + + + ); + } + render(); + + await user.click(screen.getByRole("combobox")); + await user.click(await screen.findByText("Beta Team")); + await user.click(screen.getByRole("button", { name: "refetch" })); + + expect(screen.getByRole("combobox")).toHaveValue("Beta Team"); + }); + + it("starts a fresh query when typing lands after the selected label", async () => { + const user = userEvent.setup(); + const onSearchChange = vi.fn(); + renderSelect({ onSearchChange, value: "alias-alpha" }); + + const input = screen.getByRole("combobox") as HTMLInputElement; + input.focus(); + input.setSelectionRange(input.value.length, input.value.length); + await user.keyboard("gamma"); + + expect(input).toHaveValue("gamma"); + await waitFor(() => expect(onSearchChange).toHaveBeenLastCalledWith("gamma")); + }); + + it("starts a fresh query when typing lands inside the selected label", async () => { + const user = userEvent.setup(); + const onSearchChange = vi.fn(); + renderSelect({ onSearchChange, value: "alias-alpha" }); + + const input = screen.getByRole("combobox") as HTMLInputElement; + input.focus(); + input.setSelectionRange(3, 3); + await user.keyboard("g"); + + expect(input).toHaveValue("g"); + await waitFor(() => expect(onSearchChange).toHaveBeenLastCalledWith("g")); + }); + it("surfaces loading and fetching-more affordances", async () => { const user = userEvent.setup(); const { unmount } = render( diff --git a/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.tsx b/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.tsx index 6966c669187..bb1730941a6 100644 --- a/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.tsx +++ b/ui/litellm-dashboard/src/components/shared/PaginatedSearchSelect.tsx @@ -1,7 +1,7 @@ "use client"; import { Loader2 } from "lucide-react"; -import { useMemo } from "react"; +import { useMemo, useState } from "react"; import { Combobox, @@ -35,6 +35,19 @@ interface PaginatedSearchSelectProps { "aria-describedby"?: string; } +const typedInsertion = (previous: string, next: string): string => { + let start = 0; + while (start < previous.length && start < next.length && previous[start] === next[start]) start++; + let end = 0; + while ( + end < previous.length - start && + end < next.length - start && + previous[previous.length - 1 - end] === next[next.length - 1 - end] + ) + end++; + return next.slice(start, next.length - end); +}; + export function PaginatedSearchSelect({ options, value, @@ -54,10 +67,15 @@ export function PaginatedSearchSelect({ "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy, }: PaginatedSearchSelectProps) { + const [pickedOption, setPickedOption] = useState(null); + const selected = useMemo(() => { if (value === undefined || value === "") return null; - return options.find((option) => option.value === value) ?? { label: value, value }; - }, [options, value]); + return ( + options.find((option) => option.value === value) ?? + (pickedOption?.value === value ? pickedOption : { label: value, value }) + ); + }, [options, value, pickedOption]); const items = useMemo(() => { if (selected === null) return options; @@ -66,14 +84,24 @@ export function PaginatedSearchSelect({ }, [options, selected]); const pagination = { onSearchChange, onLoadMore, hasNextPage, isFetchingNextPage }; - const { handleInputValueChange, handleScroll } = usePaginatedCombobox(pagination); + const { typedQuery, handleInputValueChange, handleOpenChange, handleScroll } = usePaginatedCombobox(pagination); return ( onValueChange(item?.value ?? "")} - onInputValueChange={(next, eventDetails) => handleInputValueChange(next, eventDetails.reason)} + inputValue={typedQuery ?? selected?.label ?? ""} + onValueChange={(item: SearchSelectOption | null) => { + setPickedOption(item); + onValueChange(item?.value ?? ""); + }} + onInputValueChange={(next, eventDetails) => + handleInputValueChange( + typedQuery === null ? typedInsertion(selected?.label ?? "", next) : next, + eventDetails.reason, + ) + } + onOpenChange={(nextOpen, eventDetails) => handleOpenChange(nextOpen, eventDetails.reason)} isItemEqualToValue={(a: SearchSelectOption, b: SearchSelectOption) => a.value === b.value} itemToStringLabel={(item: SearchSelectOption) => item.label} filter={null} diff --git a/ui/litellm-dashboard/src/components/shared/usePaginatedCombobox.ts b/ui/litellm-dashboard/src/components/shared/usePaginatedCombobox.ts index 75171d7d75f..3a51d97cd44 100644 --- a/ui/litellm-dashboard/src/components/shared/usePaginatedCombobox.ts +++ b/ui/litellm-dashboard/src/components/shared/usePaginatedCombobox.ts @@ -1,7 +1,7 @@ "use client"; import { useDebouncedCallback } from "@tanstack/react-pacer/debouncer"; -import type { UIEvent } from "react"; +import { useState, type UIEvent } from "react"; import { DEBOUNCE_WAIT_MS } from "@/utils/debounceConstants"; @@ -23,12 +23,27 @@ export function usePaginatedCombobox({ isFetchingNextPage, }: PaginatedComboboxCallbacks) { const debouncedSearch = useDebouncedCallback(onSearchChange, { wait: DEBOUNCE_WAIT_MS }); + const [typedQuery, setTypedQuery] = useState(null); const handleInputValueChange = (next: string, reason: string) => { - if (!SEARCH_REASONS.has(reason)) return; + if (!SEARCH_REASONS.has(reason)) { + setTypedQuery(null); + return; + } + setTypedQuery(next); debouncedSearch(next); }; + const handleOpenChange = (open: boolean, reason: string) => { + if (!open) { + if (typedQuery) debouncedSearch(""); + setTypedQuery(null); + return; + } + const openedByTyping = SEARCH_REASONS.has(reason); + if (!openedByTyping) setTypedQuery(""); + }; + const handleScroll = (event: UIEvent) => { const target = event.currentTarget; if (target.scrollHeight === 0) return; @@ -38,5 +53,5 @@ export function usePaginatedCombobox({ } }; - return { handleInputValueChange, handleScroll }; + return { typedQuery, handleInputValueChange, handleOpenChange, handleScroll }; } From 88cb83484ba36d35f40f549525701f46c1769e34 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Thu, 27 Aug 2026 13:41:30 -0700 Subject: [PATCH 29/29] fix(otel): anchor MCP tool-call spans to the gateway's own trace, link the client's context (#38317) Under otel_v2, a client that propagates W3C trace context in params._meta (SEP-414) pulled the tools/call span out of the gateway's trace: resolve_mcp_span_context parented the MCP span to the client's remote context and demoted the gateway's own transport span to a span link. The gateway's tracing backend only ever receives the gateway's half of such a trace, so the span was unreachable from the trace view and the POST transaction showed a dangling link. Invert the anchoring: the MCP tool-call and tools/list spans now always nest under the transport span of the request carrying the message, and the client's propagated context is recorded as the span link instead, so the correlation survives while every trace stays renderable. With no transport at all the span roots its own trace and still carries the link, keeping a single shape for the event. Both returned contexts are built on an explicitly empty base so ambient session state can never leak in, and the span inherits the transport's sampling decision like every other request-level span. --- litellm/integrations/otel/emitter.py | 6 +- litellm/integrations/otel/logger.py | 14 +-- litellm/integrations/otel/model/spans.py | 44 ++++----- litellm/integrations/otel/plumbing/context.py | 63 ++++++------ .../proxy/_experimental/mcp_server/server.py | 11 ++- .../integrations/otel/test_otel_v2_logger.py | 96 +++++++++++++++---- .../otel/test_otel_v2_sources_of_truth.py | 25 +++-- 7 files changed, 158 insertions(+), 101 deletions(-) diff --git a/litellm/integrations/otel/emitter.py b/litellm/integrations/otel/emitter.py index 244e58eddf3..101dbc6538d 100644 --- a/litellm/integrations/otel/emitter.py +++ b/litellm/integrations/otel/emitter.py @@ -146,7 +146,7 @@ class SpanEmitter: For callers that own and manage their own span lifecycle. ``tracer`` overrides the bound tracer for this span only, used for per-request multi-tenant credential routing. ``links`` records related-but-not-parent - spans (e.g. the transport span of an MCP message, per MCP semconv). + spans (e.g. the trace context an MCP client propagated in ``params._meta``). """ return (tracer or self._tracer).start_span( name, @@ -196,8 +196,8 @@ class SpanEmitter: Return the span, or ``None`` if it was deduplicated away. ``tracer`` overrides the bound tracer for this span, used for per-request routing. - ``links`` records related-but-not-parent spans (the transport span of an - MCP message). + ``links`` records related-but-not-parent spans (e.g. the trace context an + MCP client propagated in ``params._meta``). """ # LLM-call and MCP tool-call spans carry a dedup key (their request's # call id), so a sync+async double-firing coalesces. ``isinstance`` narrows diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 4359b222d06..d2a32ef73b6 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -390,10 +390,10 @@ class OpenTelemetryV2(CustomLogger): MCP tool calls reach the success/failure callbacks like any other request (with ``call_type`` ``call_mcp_tool``), but they are not LLM calls and have - no ``pre_call`` carrier — so they get their own CLIENT span here. Per the MCP - semconv it parents to the trace context the client propagated in - ``params._meta`` (or starts a new root) and links the transport span, rather - than nesting under the HTTP/session span. Returns whether it handled the + no ``pre_call`` carrier — so they get their own CLIENT span here. It nests + under the transport span of the request carrying this message, and trace + context the client propagated in ``params._meta`` is recorded as a span + link (see ``resolve_mcp_span_context``). Returns whether it handled the event, so the caller skips the LLM-call path. The whole span is emitted at once (there is no boundary to open it at), deduped on the call id. """ @@ -436,9 +436,9 @@ class OpenTelemetryV2(CustomLogger): Like a tool call, listing reaches the success/failure callbacks (here with ``call_type`` ``list_mcp_tools``) with no ``pre_call`` carrier, so it gets its - own CLIENT span. Per the MCP semconv it parents to the ``params._meta`` trace - context (or starts a new root) and links the transport span, rather than - nesting under the HTTP/session span. Returns whether it handled the event so + own CLIENT span, nested under the transport span of the request carrying + this message with any ``params._meta`` trace context recorded as a span + link (see ``resolve_mcp_span_context``). Returns whether it handled the event so the caller skips the LLM-call path. """ raw_payload: Final = kwargs.get("standard_logging_object") diff --git a/litellm/integrations/otel/model/spans.py b/litellm/integrations/otel/model/spans.py index 08318f78b7c..35fc50a2a83 100644 --- a/litellm/integrations/otel/model/spans.py +++ b/litellm/integrations/otel/model/spans.py @@ -10,6 +10,8 @@ Canonical hierarchy:: │ └── DB_CALL (CLIENT) # its key/user/team lookups nest here ├── GUARDRAIL (INTERNAL) # request-lifecycle hook, sibling of LLM_CALL ├── LLM_CALL (CLIENT) + ├── MCP_TOOL_CALL (CLIENT) # nests under the POST carrying the message + ├── MCP_LIST_TOOLS (CLIENT) # (client-propagated context is a span link) └── DB_CALL (CLIENT) # e.g. the spend-log write Guardrails parent to PROXY_REQUEST, not LLM_CALL: pre/during/post-call guardrail @@ -18,14 +20,14 @@ before the LLM call even starts), so a guardrail is a sibling of the LLM call, not a child of it. The emitter parents every span to the ambient OTel context (the active server span), which matches this. -MCP spans (``MCP_TOOL_CALL``, ``MCP_LIST_TOOLS``) have two shapes, chosen at emit -time by :func:`resolve_mcp_span_context`. When the client propagates trace context -in ``params._meta`` MCP and the HTTP transport are independent contexts per the -OTel GenAI MCP semconv, so the span parents to that propagated context and records -the ``PROXY_REQUEST`` transport span as a span *link*, never a parent — the shape -this registry's ``parent=None, links=PROXY_REQUEST`` entry encodes. When nothing is -propagated (the common case) the span nests under the transport span of the request -carrying that message, so the tool call stays in one trace. +MCP spans (``MCP_TOOL_CALL``, ``MCP_LIST_TOOLS``) are parented at emit time by +:func:`resolve_mcp_span_context`: they nest under the ``PROXY_REQUEST`` transport +span of the request carrying that message, so the tool call stays in one trace. +Trace context the client propagated in ``params._meta`` (SEP-414) is recorded as +a span *link*, never the parent — a remote parent would root the span in a trace +whose root never reaches the gateway's tracing backend. Links always target that +remote client context, never a registry role, so ``SpanSpec`` declares no link +field; the concrete transport parent is resolved per message at emit time. Not every service call becomes a span — :func:`span_role_for_service` decides: @@ -85,25 +87,19 @@ class SpanSpec: role: SpanRole kind: LiteLLMSpanKind parent: SpanRole | None - links: SpanRole | None = None SPAN_REGISTRY: Final[dict[SpanRole, SpanSpec]] = { SpanRole.PROXY_REQUEST: SpanSpec(SpanRole.PROXY_REQUEST, LiteLLMSpanKind.SERVER, parent=None), SpanRole.LLM_CALL: SpanSpec(SpanRole.LLM_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST), # The proxy is an MCP client to the upstream server, so MCP spans are CLIENT - # spans. With trace context propagated in ``params._meta``, MCP and the HTTP - # transport are independent contexts (OTel GenAI MCP semconv): the span parents - # to the propagated context and records the PROXY_REQUEST transport span as a - # span *link*, never a parent — the shape ``parent=None, links=PROXY_REQUEST`` - # encodes. With nothing propagated, ``resolve_mcp_span_context`` nests the span - # under that message's transport span instead, keeping the call in one trace. - SpanRole.MCP_TOOL_CALL: SpanSpec( - SpanRole.MCP_TOOL_CALL, LiteLLMSpanKind.CLIENT, parent=None, links=SpanRole.PROXY_REQUEST - ), - SpanRole.MCP_LIST_TOOLS: SpanSpec( - SpanRole.MCP_LIST_TOOLS, LiteLLMSpanKind.CLIENT, parent=None, links=SpanRole.PROXY_REQUEST - ), + # spans. ``resolve_mcp_span_context`` nests them under the PROXY_REQUEST + # transport span of the request carrying that message (resolved per message at + # emit time), keeping the call in one trace. Trace context the client + # propagated in ``params._meta`` becomes a span *link* to that remote context, + # which is not a registry role, so ``SpanSpec`` has no link field. + SpanRole.MCP_TOOL_CALL: SpanSpec(SpanRole.MCP_TOOL_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST), + SpanRole.MCP_LIST_TOOLS: SpanSpec(SpanRole.MCP_LIST_TOOLS, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST), SpanRole.GUARDRAIL: SpanSpec(SpanRole.GUARDRAIL, LiteLLMSpanKind.INTERNAL, parent=SpanRole.PROXY_REQUEST), SpanRole.DB_CALL: SpanSpec(SpanRole.DB_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST), SpanRole.SERVICE: SpanSpec(SpanRole.SERVICE, LiteLLMSpanKind.INTERNAL, parent=SpanRole.PROXY_REQUEST), @@ -209,8 +205,8 @@ def service_span_name(data: "ServiceSpanData") -> str: def root_roles() -> list[SpanRole]: - """Roles with no in-process parent. They start a new trace unless they adopt a - remote parent (e.g. an MCP span joining the client's propagated context).""" + """Roles with no in-process parent, i.e. they start a new trace (only the + instrumentor-owned ``PROXY_REQUEST`` server span today).""" return [role for role, spec in SPAN_REGISTRY.items() if spec.parent is None] @@ -227,8 +223,6 @@ def validate_registry( raise ValueError(f"SPAN_REGISTRY[{role}] has mismatched role {spec.role}") if spec.parent is not None and spec.parent not in reg: raise ValueError(f"span role {role} declares unknown parent {spec.parent}") - if spec.links is not None and spec.links not in reg: - raise ValueError(f"span role {role} declares unknown link target {spec.links}") missing: Final = [role for role in SpanRole if role not in reg] if missing: raise ValueError(f"SPAN_REGISTRY is missing roles: {missing}") diff --git a/litellm/integrations/otel/plumbing/context.py b/litellm/integrations/otel/plumbing/context.py index 19b36c0b967..159a84b121f 100644 --- a/litellm/integrations/otel/plumbing/context.py +++ b/litellm/integrations/otel/plumbing/context.py @@ -57,8 +57,8 @@ def request_root_span() -> "Span | None": # The W3C trace-context carrier (``traceparent``/``tracestate``/``baggage``) the # MCP client propagated in the current request's ``params._meta``. The MCP gateway -# sets it per message so the MCP span can parent to the client's span rather than -# to the transport. A ``ContextVar`` because, like the root-span anchor, it must +# sets it per message so the MCP span can record the client's span as a span +# link. A ``ContextVar`` because, like the root-span anchor, it must # ride the request task and be readable by the inline success-logging callback. _mcp_message_trace_carrier: Final["ContextVar[Mapping[str, str] | None]"] = ContextVar( "litellm_otel_mcp_message_trace_carrier", default=None @@ -148,10 +148,10 @@ def _mcp_transport_span_context() -> "SpanContext | None": Prefers the transport the gateway published for this specific message; falls back to the ambient request anchor for paths that emit an MCP span on the - request task itself (the REST MCP endpoints, the SDK). Parenting and linking - only need the immutable context, and unlike ``mcp_message_transport_span`` they - stay correct against a transport that has already finished, so this does not - require the span to still be recording. + request task itself (the REST MCP endpoints). Parenting needs only the + immutable context, and unlike ``mcp_message_transport_span`` it stays correct + against a transport that has already finished, so this does not require the + span to still be recording. """ published: Final = _mcp_message_transport_span.get() if published is not None: @@ -222,25 +222,31 @@ def resolve_mcp_span_context( ) -> "tuple[Context, tuple[Link, ...]]": """Parent context + links for an MCP message span. + The span always nests under the transport span of the request carrying this + message, so a tool call and the ``POST`` that carried it stay in one trace. + The transport comes from :func:`_mcp_transport_span_context`, which is the + *current message's* POST rather than whatever request happened to open the + session, so a long-lived session does not glue every message under its first + request. + When the client propagates W3C trace context in the request's ``params._meta`` - (SEP-414), MCP and the underlying transport are independent lifecycles — one - streamable-HTTP session multiplexes many messages, and the client's own span is - the truthful parent. So, per the OTel GenAI MCP semconv: + (SEP-414), that remote context is recorded as a span *link*, never the parent. + The OTel GenAI MCP semconv prefers the inverse (remote parent, transport link), + but the gateway's tracing backend only ever receives the gateway's half of such + a trace: parenting into the client's trace id roots the span in a trace whose + root span never reaches the backend, so the span is unreachable from the trace + view and the transport transaction shows a dangling link (observed with + clients that propagate synthetic trace ids). Anchoring to the gateway's own + request and linking the client's context keeps every trace renderable while + preserving the client-side correlation. - * parent to the trace context the client propagated (a *remote* parent), and - * record the transport span as a *link*, never the parent. - - Almost no client implements SEP-414 yet, so in practice nothing is propagated. - Rooting the span there splits a single tool call into two disconnected traces - joined only by a link, which is how it surfaces in APM: the ``POST`` transaction - and the ``tools/call`` span share no trace. With no remote parent to honor, - parent to the transport span of the request carrying this message instead, so - the call stays in one trace; no link is added since the transport is now the - real parent. The transport comes from :func:`_mcp_transport_span_context`, which - is the *current message's* POST rather than whatever request happened to open - the session, so a long-lived session does not glue every message under its - first request. With neither a remote parent nor a transport the returned context - carries no span and the span legitimately starts its own root trace. + With no transport at all the span starts its own root trace, still carrying + the link — the client context is only ever a link, so this event keeps one + shape everywhere. Both returned contexts are built on an explicitly empty + base, so ambient (stale session) state can never leak in, and the span + inherits the transport's sampling decision exactly like every other + request-level span — a client's sampled flag neither forces nor suppresses + recording. Only trace context (``traceparent``/``tracestate``) is extracted, never the client's W3C Baggage: ``params._meta`` is caller-controlled, and the otel @@ -251,13 +257,12 @@ def resolve_mcp_span_context( never fall through to the ambient (stale session) span. """ source: Final = carrier if carrier is not None else _mcp_message_trace_carrier.get() - parent: Final = _PROPAGATOR.extract(dict(source or {}), context=Context()) + propagated: Final = get_current_span(_PROPAGATOR.extract(dict(source or {}), context=Context())) + links: Final = (Link(propagated.get_span_context()),) if is_recordable_span(propagated) else () transport: Final = _mcp_transport_span_context() - if is_recordable_span(get_current_span(parent)): - return parent, (Link(transport),) if transport is not None else () - if transport is not None: - return context_from_span(NonRecordingSpan(transport)), () - return parent, () + if transport is None: + return Context(), links + return context_from_span(NonRecordingSpan(transport), context=Context()), links def is_recordable_span(obj: object) -> bool: diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 3c6eb06bc71..57e59dab2d1 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -246,11 +246,12 @@ def _mcp_meta_trace_carrier(req_ctx: object) -> dict[str, str] | None: """The W3C trace context (``traceparent``/``tracestate``) the MCP client propagated in the request's ``params._meta`` (SEP-414), or ``None``. - When present, per the OTel MCP semconv the MCP span parents to this propagated - context rather than to the HTTP transport (which is recorded as a link instead). - When absent, the span nests under the transport span of the request carrying - this specific message, so a streamable-HTTP session that multiplexes many - messages still does not glue every message under the session's first request; + When present, the MCP span records this propagated context as a span *link*, + never the parent — a remote parent would root the span in a trace whose root + never reaches the gateway's tracing backend. The span itself nests under the + transport span of the request carrying this specific message, so a + streamable-HTTP session that multiplexes many messages still does not glue + every message under the session's first request; see ``resolve_mcp_span_context``. The client's W3C Baggage is deliberately excluded: it is caller-controlled, and the otel baggage processor stamps allowlisted baggage keys (``litellm.team.id``, ``litellm.metadata.*``, diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py index 455d84c764f..4973bda29e0 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py @@ -778,11 +778,15 @@ def test_mcp_span_roots_without_transport_or_propagated_context( @pytest.mark.parametrize("make_payload, span_name", _MCP_SPAN_CASES) -def test_mcp_span_parents_to_propagated_meta_trace_context(make_payload, span_name): +def test_mcp_span_links_propagated_meta_trace_context_and_nests_under_transport( + make_payload, span_name +): """When the client propagates W3C trace context in the request's - ``params._meta`` (SEP-414), the MCP span parents to it (one distributed trace) - and still links the transport span — never falling through to the - ambient/session span.""" + ``params._meta`` (SEP-414), the MCP span still nests under the gateway's own + transport span — one renderable trace — and records the client's context as a + span *link*. Parenting to the remote context instead would root the span in a + trace whose root span never reaches the gateway's tracing backend, leaving the + span unreachable from the trace view.""" logger, exporter = _logger() transport = logger._emitter.start_span( SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME @@ -801,12 +805,65 @@ def test_mcp_span_parents_to_propagated_meta_trace_context(make_payload, span_na reset_mcp_message_trace_carrier(token) transport.end() span = next(s for s in exporter.get_finished_spans() if s.name == span_name) - assert span.context.trace_id == 0x11111111111111111111111111111111 assert span.parent is not None - assert span.parent.span_id == 0x2222222222222222 - assert [link.context.span_id for link in span.links] == [ - transport.get_span_context().span_id + assert span.parent.span_id == transport.get_span_context().span_id + assert span.context.trace_id == transport.get_span_context().trace_id + assert [link.context.trace_id for link in span.links] == [ + 0x11111111111111111111111111111111 ] + assert [link.context.span_id for link in span.links] == [0x2222222222222222] + + +@pytest.mark.parametrize("make_payload, span_name", _MCP_SPAN_CASES) +def test_mcp_span_without_transport_roots_and_links_propagated_context( + make_payload, span_name +): + """With no transport span at all there is nothing of the gateway's to anchor + to, so the span starts its own root trace — and the client context stays a + span link there too, so the event keeps one shape everywhere.""" + logger, exporter = _logger() + token = set_mcp_message_trace_carrier( + {"traceparent": "00-11111111111111111111111111111111-2222222222222222-01"} + ) + try: + asyncio.run( + logger.async_log_success_event( + {"standard_logging_object": make_payload()}, None, None, None + ) + ) + finally: + reset_mcp_message_trace_carrier(token) + span = next(s for s in exporter.get_finished_spans() if s.name == span_name) + assert span.parent is None + assert span.context.trace_id != 0x11111111111111111111111111111111 + assert [link.context.span_id for link in span.links] == [0x2222222222222222] + + +def test_mcp_span_links_unsampled_client_traceparent(): + """A client traceparent with the sampled flag off ('-00') still yields a valid + remote context, so the link is recorded; the span's own recording follows the + transport's sampling decision, never the client's flag.""" + logger, exporter = _logger() + transport = logger._emitter.start_span( + SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME + ) + set_request_root_span(transport) + token = set_mcp_message_trace_carrier( + {"traceparent": "00-11111111111111111111111111111111-2222222222222222-00"} + ) + try: + asyncio.run( + logger.async_log_success_event( + {"standard_logging_object": _mcp_list_payload()}, None, None, None + ) + ) + finally: + reset_mcp_message_trace_carrier(token) + transport.end() + span = next(s for s in exporter.get_finished_spans() if s.name == "tools/list") + assert span.parent is not None + assert span.parent.span_id == transport.get_span_context().span_id + assert [link.context.span_id for link in span.links] == [0x2222222222222222] @pytest.mark.parametrize("make_payload, span_name", _MCP_SPAN_CASES) @@ -839,8 +896,11 @@ def test_mcp_span_ignores_client_supplied_baggage(make_payload, span_name): reset_mcp_message_trace_carrier(token) transport.end() span = next(s for s in exporter.get_finished_spans() if s.name == span_name) - # Trace context still honored: proves the carrier was processed, not dropped wholesale. - assert span.parent is not None and span.parent.span_id == 0x2222222222222222 + # Trace context still honored (as a link): proves the carrier was processed, + # not dropped wholesale. + assert [link.context.span_id for link in span.links] == [0x2222222222222222] + assert span.parent is not None + assert span.parent.span_id == transport.get_span_context().span_id # Identity is the authenticated payload's team, never the client's spoofed value. assert span.attributes[LiteLLM.TEAM_ID] == "t1" assert "litellm.metadata.user_api_key_user_id" not in span.attributes @@ -888,10 +948,10 @@ def test_mcp_span_malformed_traceparent_nests_under_transport(): assert span.links == () -def test_mcp_span_links_this_messages_transport_when_context_is_propagated(): - """On the semconv path the transport is recorded as a link, and that link must - point at the POST carrying this message too. Reading the stale session anchor - would attribute the tool call to whichever request opened the session.""" +def test_mcp_span_with_propagated_context_nests_under_this_messages_transport(): + """With client context propagated, the span must still anchor to the POST + carrying this message, not the stale session anchor — otherwise the tool call + is attributed to whichever request opened the session.""" logger, exporter = _logger() session_opener = logger._emitter.start_span( SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME @@ -916,10 +976,10 @@ def test_mcp_span_links_this_messages_transport_when_context_is_propagated(): session_opener.end() this_message.end() span = next(s for s in exporter.get_finished_spans() if s.name == "tools/list") - assert span.parent is not None and span.parent.span_id == 0x2222222222222222 - assert [link.context.span_id for link in span.links] == [ - this_message.get_span_context().span_id - ] + assert span.parent is not None + assert span.parent.span_id == this_message.get_span_context().span_id + assert span.context.trace_id == this_message.get_span_context().trace_id + assert [link.context.span_id for link in span.links] == [0x2222222222222222] def test_pre_call_idempotent_keeps_first_span(): diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py index cc9b311084e..baa72b5a7fe 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py @@ -107,32 +107,29 @@ def test_registry_parent_integrity_no_orphans(): def test_registry_hierarchy_shape(): - # MCP roles have no in-process parent: per the MCP semconv they root (or adopt - # the client's propagated _meta context), so they sit alongside PROXY_REQUEST. - assert set(root_roles()) == { - SpanRole.PROXY_REQUEST, - SpanRole.MCP_TOOL_CALL, - SpanRole.MCP_LIST_TOOLS, - } + assert set(root_roles()) == {SpanRole.PROXY_REQUEST} # Guardrails parent to the request span, not the LLM call: a pre-call - # guardrail runs before the LLM call exists, so it's a sibling of it. + # guardrail runs before the LLM call exists, so it's a sibling of it. MCP + # spans nest under the transport span of the request carrying that message. assert set(child_roles(SpanRole.PROXY_REQUEST)) == { SpanRole.LLM_CALL, SpanRole.GUARDRAIL, SpanRole.DB_CALL, SpanRole.SERVICE, + SpanRole.MCP_TOOL_CALL, + SpanRole.MCP_LIST_TOOLS, } assert SPAN_REGISTRY[SpanRole.LLM_CALL].kind is LiteLLMSpanKind.CLIENT # The proxy is an MCP client to the upstream tool server: CLIENT span. Listing # tools is the same client relationship, so it's a CLIENT span too. assert SPAN_REGISTRY[SpanRole.MCP_TOOL_CALL].kind is LiteLLMSpanKind.CLIENT assert SPAN_REGISTRY[SpanRole.MCP_LIST_TOOLS].kind is LiteLLMSpanKind.CLIENT - # MCP spans don't nest under the transport: they link the PROXY_REQUEST span - # instead of parenting to it (OTel GenAI MCP semconv). - assert SPAN_REGISTRY[SpanRole.MCP_TOOL_CALL].parent is None - assert SPAN_REGISTRY[SpanRole.MCP_LIST_TOOLS].parent is None - assert SPAN_REGISTRY[SpanRole.MCP_TOOL_CALL].links is SpanRole.PROXY_REQUEST - assert SPAN_REGISTRY[SpanRole.MCP_LIST_TOOLS].links is SpanRole.PROXY_REQUEST + # MCP spans nest under the transport span of the request carrying that + # message (resolved per message at emit time); a client-propagated context + # becomes a span link to that remote context, which is not a registry role + # (SpanSpec declares no link field at all). + assert SPAN_REGISTRY[SpanRole.MCP_TOOL_CALL].parent is SpanRole.PROXY_REQUEST + assert SPAN_REGISTRY[SpanRole.MCP_LIST_TOOLS].parent is SpanRole.PROXY_REQUEST assert SPAN_REGISTRY[SpanRole.PROXY_REQUEST].kind is LiteLLMSpanKind.SERVER assert SPAN_REGISTRY[SpanRole.GUARDRAIL].parent is SpanRole.PROXY_REQUEST # An outbound datastore call is a CLIENT span; an internal service is INTERNAL.